diff --git a/crates/tinymemory-api/src/host/composio.rs b/crates/tinymemory-api/src/host/composio.rs deleted file mode 100644 index b3267633..00000000 --- a/crates/tinymemory-api/src/host/composio.rs +++ /dev/null @@ -1,540 +0,0 @@ -//! Composio value types — connections, capabilities, execute responses. -//! -//! Moved here from the host's `integrations::composio::types` because the -//! extracted memory sync pipelines read these fields directly on every run, and -//! a trait accessor per field would be absurd. They are inert serde data with -//! no behaviour and no dependencies beyond `serde`, so the contract crate's -//! dependency-light guarantee is unaffected. -//! -//! The Composio *client* deliberately did not come with them — see -//! `tinymemory_core::composio_host`. Its `Direct` variant wraps a host agent -//! tool, and mode dispatch is host policy. -//! -//! Domain types for the Composio integration. -//! -//! These mirror the response envelopes emitted by the openhuman backend under -//! `/agent-integrations/composio/*`. See: -//! - `src/routes/agentIntegrations/composio.ts` -//! - `src/controllers/agentIntegrations/composio/*.ts` -//! in the backend repo for the authoritative shapes. - -use serde::{Deserialize, Deserializer, Serialize}; - -/// Accepts either a JSON string or an object whose first matching field -/// (`slug`/`id`/`name`/`key`) is a string. Lets us tolerate upstream -/// shape drift where a previously-stringy field is now nested in an -/// object — e.g. `"toolkit": {"slug": "gmail", "logo": "…"}`. -fn de_string_or_object<'de, D: Deserializer<'de>>(d: D) -> Result { - use serde::de::Error; - let v = serde_json::Value::deserialize(d)?; - match v { - serde_json::Value::String(s) => Ok(s), - serde_json::Value::Object(map) => { - for key in ["slug", "id", "name", "key"] { - if let Some(serde_json::Value::String(s)) = map.get(key) { - return Ok(s.clone()); - } - } - Err(D::Error::custom( - "expected string or object with slug/id/name/key field", - )) - } - other => Err(D::Error::custom(format!( - "expected string, got {}", - match other { - serde_json::Value::Null => "null", - serde_json::Value::Bool(_) => "bool", - serde_json::Value::Number(_) => "number", - serde_json::Value::Array(_) => "array", - _ => "unknown", - } - ))), - } -} - -/// Like [`de_string_or_object`] but optional and resilient: missing / -/// null / unrecognized object shapes return `None` instead of erroring. -fn de_opt_string_or_object<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { - let v = Option::::deserialize(d)?; - Ok(match v { - None | Some(serde_json::Value::Null) => None, - Some(serde_json::Value::String(s)) => Some(s), - Some(serde_json::Value::Object(map)) => { - let mut found = None; - for key in ["state", "value", "slug", "id", "name", "key"] { - if let Some(serde_json::Value::String(s)) = map.get(key) { - found = Some(s.clone()); - break; - } - } - found - } - _ => None, - }) -} - -// ── Toolkits ──────────────────────────────────────────────────────── - -/// One toolkit from the live Composio catalog, forwarded verbatim from the -/// backend (`GET /agent-integrations/composio/toolkits`). -/// -/// The core does not interpret these fields — it passes them straight through -/// to the desktop UI so the app no longer hardcodes toolkit display metadata -/// (see the workspace `COMPOSIO_DYNAMIC_CATALOG_PLAN.md`). Everything except -/// `slug` is best-effort; backends predating the dynamic catalog omit the -/// whole `catalog` array, in which case the UI falls back to local metadata. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioToolkitCatalogEntry { - /// Toolkit slug as Composio emits it, e.g. `"googlecalendar"`. - pub slug: String, - /// Human-readable name, e.g. `"Google Calendar"`. - #[serde(default)] - pub name: String, - /// Composio-hosted logo URL (`meta.logo`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub logo: Option, - /// Short description (`meta.description`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub description: Option, - /// Composio category names (`meta.categories`). - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub categories: Vec, - /// Whether the user can connect/use this toolkit (passed the backend gate). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub enabled: Option, -} - -/// Response body of `GET /agent-integrations/composio/toolkits`. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioToolkitsResponse { - /// Server-enforced toolkit allowlist, e.g. `["gmail", "notion"]`. - #[serde(default)] - pub toolkits: Vec, - /// Rich render model from the live Composio catalog. Optional — empty when - /// the backend predates the dynamic catalog. Forwarded as-is to the UI. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub catalog: Vec, -} - -/// One row in OpenHuman's local Composio capability matrix. -/// -/// Unlike `ComposioToolkitsResponse`, this is not tied to a signed-in -/// backend/direct Composio session. It describes what this core build knows -/// how to do for each toolkit: whether the toolkit has a native provider -/// implementation, a curated tool catalog, profile/sync hooks, and memory -/// ingestion support. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioCapability { - pub toolkit: String, - pub description: String, - pub native_provider: bool, - pub curated_tools: bool, - pub curated_tool_count: usize, - pub tool_execution: bool, - pub user_profile: bool, - pub initial_sync: bool, - pub periodic_sync: bool, - pub sync_interval_secs: Option, - pub trigger_webhooks: bool, - pub memory_ingest: bool, -} - -/// Response body of `composio.list_capabilities`. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioCapabilitiesResponse { - #[serde(default)] - pub capabilities: Vec, -} - -/// Response body of `composio.list_agent_ready_toolkits`. -/// -/// Sorted slugs that have a curated agent catalog — the frontend -/// uses this to decide whether to label a connected toolkit as -/// "preview / agent integration coming soon". See #2283. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioAgentReadyToolkitsResponse { - #[serde(default)] - pub toolkits: Vec, -} - -// ── Connections ───────────────────────────────────────────────────── - -/// One connected Composio account (OAuth integration instance). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioConnection { - /// Composio connection id (what you DELETE to disconnect). - pub id: String, - /// Toolkit slug, e.g. `"gmail"`. - pub toolkit: String, - /// Connection status — `"ACTIVE"`, `"CONNECTED"`, `"PENDING"`, … - pub status: String, - /// ISO timestamp (backend passes this through from Composio). - #[serde(rename = "createdAt", default, skip_serializing_if = "Option::is_none")] - pub created_at: Option, - /// Account email — populated from the cached provider profile when - /// the toolkit reports an email address (e.g. Gmail, Google Calendar, - /// Google Sheets). Lets the UI picker show "Gmail · user@example.com" - /// instead of a generic "Account N" label. - #[serde( - rename = "accountEmail", - default, - skip_serializing_if = "Option::is_none" - )] - pub account_email: Option, - /// Workspace or team display name — populated for workspace-based - /// services (e.g. Slack: user display name / team name, Notion: workspace - /// name). Used by the picker when no email is available. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub workspace: Option, - /// Screen name or handle — populated for username-based services - /// (e.g. GitHub login, Twitter handle). Used by the picker as a - /// last-resort identity hint after email and workspace. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub username: Option, -} - -impl ComposioConnection { - /// Return the toolkit slug in the canonical form used by provider - /// lookup, prompt injection, and tool-action prefix matching. - pub fn normalized_toolkit(&self) -> String { - self.toolkit.trim().to_ascii_lowercase() - } - - /// Whether this row represents a usable connection. - /// - /// The web UI already treats status case-insensitively. Keep the - /// core-side chat/runtime filters aligned so a backend spelling such - /// as `connected` cannot display as connected in Settings while - /// disappearing from the agent's integration surface. - pub fn is_active(&self) -> bool { - let status = self.status.trim(); - status.eq_ignore_ascii_case("ACTIVE") || status.eq_ignore_ascii_case("CONNECTED") - } -} - -/// Response body of `GET /agent-integrations/composio/connections`. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioConnectionsResponse { - #[serde(default)] - pub connections: Vec, -} - -/// Response body of `POST /agent-integrations/composio/authorize`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioAuthorizeResponse { - /// Composio-hosted OAuth URL the user opens in a browser. - #[serde(rename = "connectUrl")] - pub connect_url: String, - /// Composio connection id created by this authorize call. - #[serde(rename = "connectionId")] - pub connection_id: String, -} - -/// Response body of `DELETE /agent-integrations/composio/connections/:id`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioDeleteResponse { - #[serde(default)] - pub deleted: bool, - #[serde(default)] - pub memory_chunks_deleted: usize, -} - -// ── Tools ─────────────────────────────────────────────────────────── - -/// OpenAI function-calling schema returned by the backend for each tool. -/// -/// The backend wraps Composio's upstream shape; we keep the `type` + -/// `function` envelope so callers can forward directly into an LLM. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioToolSchema { - #[serde(rename = "type", default = "default_function_type")] - pub kind: String, - pub function: ComposioToolFunction, -} - -fn default_function_type() -> String { - "function".to_string() -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioToolFunction { - /// Composio action slug, e.g. `"GMAIL_SEND_EMAIL"`. - pub name: String, - /// Human-readable description shown to the model. - #[serde(default)] - pub description: Option, - /// JSON schema for the tool's INPUT parameters. - #[serde(default)] - pub parameters: Option, - /// JSON schema describing the tool's OUTPUT/return-value shape, when the - /// upstream listing publishes one. Composio's v3 `/tools` endpoint calls - /// this `output_parameters` — documented as "Schema definition of return - /// values from the tool" - /// () — - /// alongside `input_parameters`. `None` means "unknown" (not "empty"): - /// the backend-proxied `/agent-integrations/composio/tools` path is - /// opaque to this crate and may not forward it, and not every Composio - /// action publishes an output schema. - #[serde(default)] - pub output_parameters: Option, -} - -/// Response body of `GET /agent-integrations/composio/tools`. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioToolsResponse { - #[serde(default)] - pub tools: Vec, -} - -// ── Execute ───────────────────────────────────────────────────────── - -/// Response body of `POST /agent-integrations/composio/execute`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioExecuteResponse { - /// Raw result from the upstream provider. - #[serde(default)] - pub data: serde_json::Value, - /// Did the provider report success? - #[serde(default)] - pub successful: bool, - /// Provider error message if any. - #[serde(default)] - pub error: Option, - /// Amount charged to the caller (base + margin) in USD. - #[serde(rename = "costUsd", default)] - pub cost_usd: f64, - /// Backend-rendered compact markdown for known tools (set by - /// backend PR tinyhumansai/backend#683). When present and non-empty - /// callers should prefer this over `data` for LLM/CLI consumption. - #[serde(rename = "markdownFormatted", default)] - pub markdown_formatted: Option, -} - -// ── GitHub repos + triggers ───────────────────────────────────────── - -/// One repository returned by `GET /agent-integrations/composio/github/repos`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioGithubRepo { - pub owner: String, - pub repo: String, - #[serde(rename = "fullName")] - pub full_name: String, - #[serde(default)] - pub private: Option, - #[serde(rename = "defaultBranch", default)] - pub default_branch: Option, - #[serde(rename = "htmlUrl", default)] - pub html_url: Option, -} - -/// Response body of `GET /agent-integrations/composio/github/repos`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioGithubReposResponse { - #[serde(rename = "connectionId")] - pub connection_id: String, - #[serde(default, rename = "repositories")] - pub repositories: Vec, -} - -/// Response body of `POST /agent-integrations/composio/triggers`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioCreateTriggerResponse { - #[serde(rename = "triggerId")] - pub trigger_id: String, - #[serde(default)] - pub status: Option, -} - -// ── Trigger management (catalog + active list + enable/disable) ───── - -/// Per-repo descriptor used by GitHub-scoped available triggers. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioAvailableTriggerRepo { - pub owner: String, - pub repo: String, -} - -/// One entry in `GET /agent-integrations/composio/triggers/available`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioAvailableTrigger { - pub slug: String, - /// `"static"` or `"github_repo"`. - pub scope: String, - #[serde( - rename = "defaultConfig", - default, - skip_serializing_if = "Option::is_none" - )] - pub default_config: Option, - #[serde( - rename = "requiredConfigKeys", - default, - skip_serializing_if = "Option::is_none" - )] - pub required_config_keys: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub repo: Option, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioAvailableTriggersResponse { - #[serde(default)] - pub triggers: Vec, -} - -/// One entry in `GET /agent-integrations/composio/triggers`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioActiveTrigger { - #[serde(deserialize_with = "de_string_or_object")] - pub id: String, - #[serde(deserialize_with = "de_string_or_object")] - pub slug: String, - #[serde(deserialize_with = "de_string_or_object")] - pub toolkit: String, - #[serde(rename = "connectionId", deserialize_with = "de_string_or_object")] - pub connection_id: String, - #[serde( - rename = "triggerConfig", - default, - skip_serializing_if = "Option::is_none" - )] - pub trigger_config: Option, - #[serde( - default, - skip_serializing_if = "Option::is_none", - deserialize_with = "de_opt_string_or_object" - )] - pub state: Option, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioActiveTriggersResponse { - #[serde(default)] - pub triggers: Vec, -} - -/// Response body of `POST /agent-integrations/composio/triggers` (enable). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioEnableTriggerResponse { - #[serde(rename = "triggerId")] - pub trigger_id: String, - pub slug: String, - #[serde(rename = "connectionId")] - pub connection_id: String, -} - -/// Response body of `DELETE /agent-integrations/composio/triggers/:id`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioDisableTriggerResponse { - #[serde(default)] - pub deleted: bool, -} - -// ── Triggers ──────────────────────────────────────────────────────── - -/// Payload of the `composio:trigger` Socket.IO event emitted by the backend -/// when a Composio webhook is received, HMAC-verified, and delivered to the -/// user's active sockets. -/// -/// See `src/controllers/agentIntegrations/composio/handleWebhook.ts` in the -/// backend repo. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioTriggerEvent { - /// Toolkit slug, e.g. `"gmail"`. - #[serde(default)] - pub toolkit: String, - /// Trigger slug, e.g. `"GMAIL_NEW_GMAIL_MESSAGE"`. - #[serde(default)] - pub trigger: String, - /// Trigger-specific payload (provider-defined shape). - #[serde(default)] - pub payload: serde_json::Value, - /// Metadata the backend attaches: `{ id, uuid }`. - #[serde(default)] - pub metadata: ComposioTriggerMetadata, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioTriggerMetadata { - #[serde(default)] - pub id: String, - #[serde(default)] - pub uuid: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioTriggerHistoryEntry { - /// Unix timestamp in milliseconds when the trigger reached the core. - pub received_at_ms: u64, - /// Toolkit slug, e.g. `"gmail"`. - pub toolkit: String, - /// Trigger slug, e.g. `"GMAIL_NEW_GMAIL_MESSAGE"`. - pub trigger: String, - /// Backend metadata id for this event. - pub metadata_id: String, - /// Backend metadata UUID for this event. - pub metadata_uuid: String, - /// Raw provider payload as forwarded by the backend socket event. - pub payload: serde_json::Value, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioTriggerHistoryResult { - /// Directory containing daily JSONL archives. - pub archive_dir: String, - /// Today's JSONL file path. - pub current_day_file: String, - /// Recent triggers, newest first. - pub entries: Vec, -} - -/// Static overview of the Composio integrations this build supports. -/// -/// Deliberately does not consult the live Composio backend or a direct tenant: -/// it is an observability surface over what the code knows how to do, not over -/// what the signed-in user has authorized. Callers wanting the latter want -/// `composio.list_toolkits` / `composio.list_connections`. -/// -/// # Why this lives here and not in the engine crate -/// -/// It reads only [`ComposioCapability`] and the contract's curated catalogs -/// (OpenHuman#5560). The version it replaces sat in -/// `tinymemory_core::sync::composio::providers`, so a host rendering its own -/// capability matrix had to link the engine to spell a table of `&'static str`. -/// -/// The two provider-shaped facts it needs — which toolkits have a native -/// provider, and how often each syncs — are -/// [`catalogs::NATIVE_PROVIDERS`][crate::composio::catalogs::NATIVE_PROVIDERS], -/// a const table that carries the same defaults the provider impls pass to -/// their own interval resolver. -#[must_use] -pub fn capability_matrix() -> Vec { - use crate::composio::catalogs; - catalogs::CAPABILITY_TOOLKITS - .iter() - .map(|toolkit| { - let native_provider = catalogs::has_native_provider(toolkit); - let catalog = catalogs::catalog_for_toolkit(toolkit); - let sync_interval_secs = catalogs::native_provider_sync_interval_secs(toolkit); - ComposioCapability { - toolkit: (*toolkit).to_string(), - description: catalogs::toolkit_description(toolkit).to_string(), - native_provider, - curated_tools: catalog.is_some(), - curated_tool_count: catalog - .map_or(0, <[crate::composio::scopes::CuratedTool]>::len), - tool_execution: catalog.is_some(), - user_profile: native_provider, - initial_sync: native_provider, - periodic_sync: sync_interval_secs.is_some(), - sync_interval_secs, - trigger_webhooks: native_provider, - memory_ingest: native_provider, - } - }) - .collect() -} - -#[cfg(test)] -#[path = "composio_tests.rs"] -mod tests; diff --git a/crates/tinymemory-api/src/host/composio_tests.rs b/crates/tinymemory-api/src/host/composio_tests.rs deleted file mode 100644 index 7ab342a7..00000000 --- a/crates/tinymemory-api/src/host/composio_tests.rs +++ /dev/null @@ -1,387 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; -use serde_json::json; - -#[test] -fn connection_is_active_matches_ui_status_normalization() { - for status in ["ACTIVE", "CONNECTED", "active", "connected", " connected "] { - let conn = ComposioConnection { - id: "c1".into(), - toolkit: "slack".into(), - status: status.into(), - created_at: None, - account_email: None, - workspace: None, - username: None, - }; - assert!(conn.is_active(), "status {status:?} should be active"); - } - - for status in ["PENDING", "INITIATED", "FAILED", ""] { - let conn = ComposioConnection { - id: "c1".into(), - toolkit: "slack".into(), - status: status.into(), - created_at: None, - account_email: None, - workspace: None, - username: None, - }; - assert!(!conn.is_active(), "status {status:?} should not be active"); - } -} - -#[test] -fn connection_normalizes_toolkit_for_runtime_matching() { - let conn = ComposioConnection { - id: "c1".into(), - toolkit: " Slack ".into(), - status: "ACTIVE".into(), - created_at: None, - account_email: None, - workspace: None, - username: None, - }; - assert_eq!(conn.normalized_toolkit(), "slack"); -} - -#[test] -fn toolkits_response_defaults_to_empty() { - let resp: ComposioToolkitsResponse = serde_json::from_str("{}").unwrap(); - assert!(resp.toolkits.is_empty()); -} - -#[test] -fn toolkits_response_roundtrips() { - let resp = ComposioToolkitsResponse { - toolkits: vec!["gmail".into(), "notion".into()], - ..Default::default() - }; - let value = serde_json::to_value(&resp).unwrap(); - // Empty catalog is skipped on the wire — back-compat with old cores. - assert_eq!(value, json!({ "toolkits": ["gmail", "notion"] })); - let back: ComposioToolkitsResponse = serde_json::from_value(value).unwrap(); - assert_eq!(back.toolkits, vec!["gmail", "notion"]); - assert!(back.catalog.is_empty()); -} - -#[test] -fn toolkits_response_forwards_catalog() { - // A backend that sends the dynamic catalog must deserialize and - // re-serialize verbatim so the field reaches the desktop UI. - let raw = json!({ - "toolkits": ["gmail"], - "catalog": [ - { - "slug": "gmail", - "name": "Gmail", - "logo": "https://logos.composio.dev/api/gmail", - "description": "Send and read email", - "categories": ["productivity"], - "enabled": true - } - ] - }); - let resp: ComposioToolkitsResponse = serde_json::from_value(raw).unwrap(); - assert_eq!(resp.catalog.len(), 1); - let entry = &resp.catalog[0]; - assert_eq!(entry.slug, "gmail"); - assert_eq!(entry.name, "Gmail"); - assert_eq!(entry.enabled, Some(true)); - assert_eq!(entry.categories, vec!["productivity".to_string()]); - - // Round-trips back out with the catalog intact. - let value = serde_json::to_value(&resp).unwrap(); - assert_eq!(value["catalog"][0]["slug"], "gmail"); - assert_eq!(value["catalog"][0]["enabled"], true); -} - -#[test] -fn connection_parses_and_serializes_camelcase_created_at() { - let raw = json!({ - "id": "conn_1", - "toolkit": "gmail", - "status": "ACTIVE", - "createdAt": "2026-02-01T00:00:00Z" - }); - let conn: ComposioConnection = serde_json::from_value(raw.clone()).unwrap(); - assert_eq!(conn.id, "conn_1"); - assert_eq!(conn.toolkit, "gmail"); - assert_eq!(conn.status, "ACTIVE"); - assert_eq!(conn.created_at.as_deref(), Some("2026-02-01T00:00:00Z")); - - // Round-trip must use camelCase too. - let serialized = serde_json::to_value(&conn).unwrap(); - assert!(serialized.get("createdAt").is_some()); -} - -#[test] -fn connection_without_created_at_omits_field_when_serialized() { - let conn = ComposioConnection { - id: "x".into(), - toolkit: "notion".into(), - status: "PENDING".into(), - created_at: None, - account_email: None, - workspace: None, - username: None, - }; - let s = serde_json::to_value(&conn).unwrap(); - assert!( - s.get("createdAt").is_none(), - "createdAt must be skipped when None" - ); -} - -#[test] -fn authorize_response_uses_camelcase_keys() { - let raw = json!({ - "connectUrl": "https://composio.dev/oauth/abc", - "connectionId": "conn_2" - }); - let resp: ComposioAuthorizeResponse = serde_json::from_value(raw).unwrap(); - assert_eq!(resp.connect_url, "https://composio.dev/oauth/abc"); - assert_eq!(resp.connection_id, "conn_2"); - - let s = serde_json::to_value(&resp).unwrap(); - assert!(s.get("connectUrl").is_some()); - assert!(s.get("connectionId").is_some()); -} - -#[test] -fn tool_schema_defaults_type_field_to_function() { - let raw = json!({ - "function": { - "name": "GMAIL_SEND_EMAIL", - "description": "Send an email", - "parameters": { "type": "object" } - } - }); - let tool: ComposioToolSchema = serde_json::from_value(raw).unwrap(); - assert_eq!(tool.kind, "function"); - assert_eq!(tool.function.name, "GMAIL_SEND_EMAIL"); - assert_eq!(tool.function.description.as_deref(), Some("Send an email")); - assert!(tool.function.parameters.is_some()); -} - -#[test] -fn tool_function_tolerates_missing_description_and_parameters() { - let raw = json!({ "function": { "name": "SLUG_ONLY" } }); - let tool: ComposioToolSchema = serde_json::from_value(raw).unwrap(); - assert_eq!(tool.function.name, "SLUG_ONLY"); - assert!(tool.function.description.is_none()); - assert!(tool.function.parameters.is_none()); -} - -#[test] -fn execute_response_parses_cost_and_error() { - let raw = json!({ - "data": { "messageId": "m-1" }, - "successful": true, - "error": null, - "costUsd": 0.0025 - }); - let resp: ComposioExecuteResponse = serde_json::from_value(raw).unwrap(); - assert!(resp.successful); - assert!(resp.error.is_none()); - assert!((resp.cost_usd - 0.0025).abs() < f64::EPSILON); -} - -#[test] -fn execute_response_defaults_when_fields_missing() { - let resp: ComposioExecuteResponse = serde_json::from_str("{}").unwrap(); - assert!(!resp.successful); - assert!(resp.error.is_none()); - assert_eq!(resp.cost_usd, 0.0); - assert!(resp.data.is_null()); -} - -#[test] -fn available_trigger_deserializes_and_serializes_camelcase_fields() { - let raw = json!({ - "slug": "GMAIL_NEW_GMAIL_MESSAGE", - "scope": "static", - "defaultConfig": { "labelIds": ["INBOX"] }, - "requiredConfigKeys": ["labelIds"], - "repo": { "owner": "acme", "repo": "inbox" } - }); - let trigger: ComposioAvailableTrigger = serde_json::from_value(raw).unwrap(); - assert_eq!(trigger.slug, "GMAIL_NEW_GMAIL_MESSAGE"); - assert_eq!(trigger.scope, "static"); - assert_eq!( - trigger.default_config, - Some(json!({ "labelIds": ["INBOX"] })) - ); - assert_eq!( - trigger.required_config_keys, - Some(vec!["labelIds".to_string()]) - ); - let repo = trigger.repo.as_ref().expect("repo"); - assert_eq!(repo.owner, "acme"); - assert_eq!(repo.repo, "inbox"); - - let value = serde_json::to_value(&trigger).unwrap(); - assert!(value.get("defaultConfig").is_some()); - assert!(value.get("requiredConfigKeys").is_some()); -} - -#[test] -fn active_trigger_parses_connection_id_and_optional_fields() { - let raw = json!({ - "id": "ti_1", - "slug": "GMAIL_NEW_GMAIL_MESSAGE", - "toolkit": "gmail", - "connectionId": "c-1", - "triggerConfig": { "labelIds": "INBOX" }, - "state": "active" - }); - let trigger: ComposioActiveTrigger = serde_json::from_value(raw).unwrap(); - assert_eq!(trigger.id, "ti_1"); - assert_eq!(trigger.slug, "GMAIL_NEW_GMAIL_MESSAGE"); - assert_eq!(trigger.connection_id, "c-1"); - assert_eq!(trigger.trigger_config, Some(json!({"labelIds":"INBOX"}))); - assert_eq!(trigger.state.as_deref(), Some("active")); - - let value = serde_json::to_value(&trigger).unwrap(); - assert!(value.get("connectionId").is_some()); - assert!(value.get("triggerConfig").is_some()); - assert!(value.get("state").is_some()); -} - -#[test] -fn trigger_enable_response_uses_camelcase_and_optional_defaults() { - let raw = json!({ - "triggerId": "ti_9", - "slug": "GMAIL_NEW_GMAIL_MESSAGE", - "connectionId": "c-9" - }); - let resp: ComposioEnableTriggerResponse = serde_json::from_value(raw).unwrap(); - assert_eq!(resp.trigger_id, "ti_9"); - assert_eq!(resp.slug, "GMAIL_NEW_GMAIL_MESSAGE"); - assert_eq!(resp.connection_id, "c-9"); - - let serialized = serde_json::to_value(&resp).unwrap(); - assert_eq!(serialized.get("triggerId").unwrap(), "ti_9"); - assert_eq!(serialized.get("connectionId").unwrap(), "c-9"); -} - -#[test] -fn delete_trigger_response_defaults_deleted_to_false() { - let raw = json!({}); - let resp: ComposioDisableTriggerResponse = serde_json::from_value(raw).unwrap(); - assert!(!resp.deleted); -} - -#[test] -fn trigger_event_defaults_empty_fields_to_empty_strings() { - let ev: ComposioTriggerEvent = serde_json::from_str("{}").unwrap(); - assert_eq!(ev.toolkit, ""); - assert_eq!(ev.trigger, ""); - assert_eq!(ev.metadata.id, ""); - assert_eq!(ev.metadata.uuid, ""); - assert!(ev.payload.is_null()); -} - -#[test] -fn trigger_event_parses_full_payload() { - let raw = json!({ - "toolkit": "gmail", - "trigger": "GMAIL_NEW_GMAIL_MESSAGE", - "payload": { "subject": "hi" }, - "metadata": { "id": "evt-1", "uuid": "uuid-1" } - }); - let ev: ComposioTriggerEvent = serde_json::from_value(raw).unwrap(); - assert_eq!(ev.toolkit, "gmail"); - assert_eq!(ev.trigger, "GMAIL_NEW_GMAIL_MESSAGE"); - assert_eq!(ev.metadata.id, "evt-1"); - assert_eq!(ev.metadata.uuid, "uuid-1"); - assert_eq!(ev.payload["subject"], "hi"); -} - -#[test] -fn active_trigger_accepts_string_fields() { - let v = json!({ - "id": "t1", - "slug": "GMAIL_NEW_MAIL", - "toolkit": "gmail", - "connectionId": "c1", - "state": "ACTIVE", - }); - let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); - assert_eq!(trig.id, "t1"); - assert_eq!(trig.slug, "GMAIL_NEW_MAIL"); - assert_eq!(trig.toolkit, "gmail"); - assert_eq!(trig.connection_id, "c1"); - assert_eq!(trig.state.as_deref(), Some("ACTIVE")); -} - -#[test] -fn active_trigger_accepts_object_fields() { - // Mirrors upstream API drift where these fields arrive as objects - // rather than plain strings. - let v = json!({ - "id": {"id": "t1"}, - "slug": {"slug": "GMAIL_NEW_MAIL"}, - "toolkit": {"slug": "gmail", "logo": "https://…"}, - "connectionId": {"id": "c1"}, - "state": {"state": "ACTIVE", "slug": "should-be-ignored"}, - }); - let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); - assert_eq!(trig.id, "t1"); - assert_eq!(trig.slug, "GMAIL_NEW_MAIL"); - assert_eq!(trig.toolkit, "gmail"); - assert_eq!(trig.connection_id, "c1"); - // `state` priority must prefer the literal `state` key over metadata. - assert_eq!(trig.state.as_deref(), Some("ACTIVE")); -} - -#[test] -fn active_trigger_state_falls_back_to_value() { - let v = json!({ - "id": "t1", - "slug": "X", - "toolkit": "gmail", - "connectionId": "c1", - "state": {"value": "PENDING"}, - }); - let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); - assert_eq!(trig.state.as_deref(), Some("PENDING")); -} - -#[test] -fn active_trigger_state_missing_or_unknown_returns_none() { - let v = json!({ - "id": "t1", - "slug": "X", - "toolkit": "gmail", - "connectionId": "c1", - }); - let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); - assert!(trig.state.is_none()); - - let v = json!({ - "id": "t1", - "slug": "X", - "toolkit": "gmail", - "connectionId": "c1", - "state": {"unrelated": 42}, - }); - let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); - assert!(trig.state.is_none()); -} - -#[test] -fn active_trigger_required_field_rejects_unsupported_object() { - // Object without any of slug/id/name/key must fail loudly so we - // notice further upstream shape drift instead of silently dropping - // the trigger. - let v = json!({ - "id": {"unrelated": 42}, - "slug": "X", - "toolkit": "gmail", - "connectionId": "c1", - }); - let err = serde_json::from_value::(v).unwrap_err(); - assert!(err.to_string().contains("expected string or object")); -} diff --git a/crates/tinymemory-api/src/host/mod.rs b/crates/tinymemory-api/src/host/mod.rs index 1f0e1588..b747aa72 100644 --- a/crates/tinymemory-api/src/host/mod.rs +++ b/crates/tinymemory-api/src/host/mod.rs @@ -41,7 +41,6 @@ //! them and they can go home. pub mod cloud_providers; -pub mod composio; pub mod local_ai; pub mod scheduler_gate; pub mod storage_memory; diff --git a/crates/tinymemory-core/src/composio_host.rs b/crates/tinymemory-core/src/composio_host.rs deleted file mode 100644 index 71337830..00000000 --- a/crates/tinymemory-core/src/composio_host.rs +++ /dev/null @@ -1,185 +0,0 @@ -//! [`ComposioHost`] — the Composio integration, as the memory sync layer sees it. -//! -//! The sync pipelines need three things from Composio: which connections are -//! active, the ability to execute a tool against one, and the direct-mode API -//! key. Everything else about the integration — OAuth, the backend session -//! token, per-toolkit allowlists, HMAC-verified trigger fan-out, and the choice -//! between backend-proxied and direct mode — is host concern. -//! -//! # Why the client itself stayed in the host -//! -//! `ComposioClientKind::Direct` wraps an `Arc`, -//! a host *agent tool*. There is no way to name that from here, and no reason -//! to: mode dispatch reads `config.composio.mode` and fails loud on a typo, -//! which is exactly the kind of policy the README's split assigns to the host. -//! -//! So this trait is deliberately **behavioural, not structural**. It hides -//! `ComposioClientKind` entirely — the core never learns that two modes exist, -//! and the three call sites that used to do -//! `create_composio_client(config)?` → `match kind` → call collapse to one -//! method each. -//! -//! The value types those methods return did move, to -//! [`tinymemory_api::host::composio`], because the pipelines read their fields -//! directly. -//! -//! # Unwired is an error -//! -//! Same reasoning as [`crate::embedding_host`]: a sync run that quietly saw -//! zero connections would look like "nothing to sync" rather than "not wired -//! up", and the difference would only surface as missing memory days later. - -use std::sync::Arc; - -use async_trait::async_trait; -use parking_lot::RwLock; - -use crate::Config; - -pub use tinymemory_api::host::composio::{ - ComposioCapability, ComposioConnection, ComposioExecuteResponse, -}; - -/// The Composio operations the memory sync layer performs. -#[async_trait] -pub trait ComposioHost: Send + Sync + std::fmt::Debug { - /// Every connection the signed-in user has, active or not. - /// - /// Filtering to active ones is the caller's job — `ComposioConnection` - /// carries the status and treats an empty one as inactive, so a malformed - /// upstream row is never presented as connected. - /// - /// # Errors - /// - /// Returns `Err` when no client can be built (no backend session, bad mode - /// string) or the upstream call fails. - async fn list_connections(&self, config: &Config) -> Result, String>; - - /// Execute `tool` against a connection. - /// - /// # Errors - /// - /// Returns `Err` when no client can be built or the call fails. A provider - /// that answers with `successful: false` is **not** an error — that is - /// reported in the returned [`ComposioExecuteResponse`]. - async fn execute( - &self, - config: &Config, - tool: &str, - arguments: Option, - entity_id: &str, - connection_id: Option<&str>, - ) -> Result; - - /// The direct-mode Composio API key from the host's credential store, or - /// `None` when direct mode is not configured. - fn api_key(&self, config: &Config) -> Option; - - /// The OpenHuman backend bearer for proxied ("backend") mode. - /// - /// A seam rather than a config field, and that is the whole point of it. - /// The bearer is an app-session JWT the host refreshes; a value captured - /// once — at module load, say — works until it expires and then makes every - /// sync fail with an auth error that reads as the user being signed out. - /// Asking per call means the answer is always the one that is valid now. - /// - /// `None` means the host has no session to lend, which is a signed-out user - /// rather than a broken one. The caller must not read that as "nothing to - /// sync": [`composio_config`](crate::sync::pipelines::host::composio_config) - /// turns it into a named refusal instead. - /// - /// Defaulted to `None` so a host that predates this member still compiles - /// and simply falls back to whatever `Config::session_token` answers, which - /// is exactly the behaviour it had before the member existed. - fn session_bearer(&self, config: &Config) -> Option { - let _ = config; - None - } - - /// Whether *some* viable client resolves for the current config. - /// - /// The sync layer uses this as its "is the user signed in?" probe. It must - /// answer for **either** mode: direct-mode users typically have no backend - /// session token, and probing for one alone would falsely skip them. - fn is_available(&self, config: &Config) -> bool; -} - -static HOST: RwLock>> = RwLock::new(None); - -const NOT_INSTALLED: &str = - "no ComposioHost installed — the host must call memory::composio_host::set_composio_host \ - during startup wiring, before any sync runs"; - -/// Install the host's Composio integration. Called once during startup wiring. -pub fn set_composio_host(host: Arc) { - *HOST.write() = Some(host); -} - -/// Remove any installed host. For tests. -pub fn clear_composio_host() { - *HOST.write() = None; -} - -/// The installed host, or `None` when nothing has been wired up. -#[must_use] -pub fn composio_host() -> Option> { - HOST.read().clone() -} - -/// The installed host. -/// -/// # Errors -/// -/// Returns `Err` when no host has been installed. -pub fn require_composio_host() -> Result, String> { - composio_host().ok_or_else(|| NOT_INSTALLED.to_string()) -} - -/// Active-or-not connections for the signed-in user. -/// -/// # Errors -/// -/// Returns `Err` when no host is installed, or the upstream call fails. -pub async fn list_connections(config: &Config) -> Result, String> { - require_composio_host()?.list_connections(config).await -} - -/// Execute a Composio tool. -/// -/// # Errors -/// -/// Returns `Err` when no host is installed, or the call fails. -pub async fn execute( - config: &Config, - tool: &str, - arguments: Option, - entity_id: &str, - connection_id: Option<&str>, -) -> Result { - require_composio_host()? - .execute(config, tool, arguments, entity_id, connection_id) - .await -} - -/// The direct-mode API key, or `None` when unset or unwired. -#[must_use] -pub fn api_key(config: &Config) -> Option { - composio_host()?.api_key(config) -} - -/// The backend bearer from the installed host, or `None` when no host is -/// installed or the host has no session. -/// -/// The two are deliberately not distinguished here: both mean "this process -/// cannot authenticate a proxied Composio call right now", and the caller's -/// fallback and error message are the same either way. -#[must_use] -pub fn session_bearer(config: &Config) -> Option { - composio_host()?.session_bearer(config) -} - -/// Whether a viable Composio client resolves. `false` when unwired. -#[must_use] -pub fn is_available(config: &Config) -> bool { - composio_host().is_some_and(|host| host.is_available(config)) -} diff --git a/crates/tinymemory-core/src/engine/mod.rs b/crates/tinymemory-core/src/engine/mod.rs index 63e11556..6a936456 100644 --- a/crates/tinymemory-core/src/engine/mod.rs +++ b/crates/tinymemory-core/src/engine/mod.rs @@ -65,11 +65,9 @@ pub use seal::{ }; pub use summariser::HostSummariser; pub use sync::{ - estimate_cost_usd, load_composio_sync_state, needs_rebuild, raw_coverage, read_audit_log, - rebuild_tree_from_raw, run_composio_connection, run_composio_connection_with_budgets, - run_github_sync, run_gmail_backfill, run_slack_search_backfill, run_source_pipeline, - sync_context, HostSyncAdapter, RawCoverage, RawFileRef, RealCostAccumulator, RebuildOutcome, - SourcePipelineFailure, HOST_SYNC_STATE_NAMESPACE, + estimate_cost_usd, needs_rebuild, raw_coverage, read_audit_log, rebuild_tree_from_raw, + run_github_sync, run_source_pipeline, sync_context, HostSyncAdapter, RawCoverage, RawFileRef, + RealCostAccumulator, RebuildOutcome, SourcePipelineFailure, HOST_SYNC_STATE_NAMESPACE, }; // Crate-private seam for `crate::sources::sync` (openhuman#5820); not host surface. pub(crate) use sync::run_source_pipeline_core; diff --git a/crates/tinymemory-core/src/engine/sync.rs b/crates/tinymemory-core/src/engine/sync.rs index c8372d85..9f4497ea 100644 --- a/crates/tinymemory-core/src/engine/sync.rs +++ b/crates/tinymemory-core/src/engine/sync.rs @@ -57,6 +57,21 @@ impl SourcePipelineFailure { } } +/// [`SyncOutcome`] plus `tree_ingest_failures` — the count of items whose +/// fetch-and-store committed but whose memory-tree ingest did not +/// (openhuman#5820). The vendored engine's own `SyncOutcome` has no field for +/// this, so [`run_source_pipeline_core`] returns this richer type instead and +/// [`run_source_pipeline`] converts down to the engine type for callers that +/// do not need the tree half's verdict. +pub(crate) struct SourcePipelineOutcome { + pub records_ingested: u32, + pub more_pending: bool, + pub actions_called: u32, + pub provider_cost_usd: f64, + pub note: Option, + pub tree_ingest_failures: u32, +} + impl HostSyncAdapter { pub fn new(memory: MemoryClientRef) -> Self { Self { @@ -264,7 +279,15 @@ impl ExternalSourceReader for HostSyncAdapter { .as_ref() .ok_or_else(|| anyhow::anyhow!("external source reader requires host config"))?; let host_source: MemorySourceEntry = serde_json::from_value(serde_json::to_value(source)?)?; - let reader = crate::sources::readers::reader_for(&host_source.kind); + // A kind with no reader is an error, not an empty listing: answering + // "0 items" for a source nobody read would let the caller record the + // sync as complete and move its cursor past everything it skipped. + let reader = crate::sources::readers::reader_for(&host_source.kind).ok_or_else(|| { + anyhow::anyhow!( + "no reader for source kind {:?}: it is fetched outside this crate", + host_source.kind + ) + })?; let items = reader .list_items(&host_source, &**config) .await @@ -282,7 +305,12 @@ impl ExternalSourceReader for HostSyncAdapter { .as_ref() .ok_or_else(|| anyhow::anyhow!("external source reader requires host config"))?; let host_source: MemorySourceEntry = serde_json::from_value(serde_json::to_value(source)?)?; - let reader = crate::sources::readers::reader_for(&host_source.kind); + let reader = crate::sources::readers::reader_for(&host_source.kind).ok_or_else(|| { + anyhow::anyhow!( + "no reader for source kind {:?}: it is fetched outside this crate", + host_source.kind + ) + })?; let content = reader .read_item(&host_source, item_id, &**config) .await @@ -341,49 +369,29 @@ pub async fn run_source_pipeline( }) } -/// [`run_source_pipeline`] returning the core pipelines' own -/// [`crate::sync::pipelines::traits::SyncOutcome`], which additionally carries -/// `tree_ingest_failures` — the "fetch committed, tree ingest did not" count a -/// sync verdict must not launder into success (openhuman#5820). The engine's -/// outcome type stays untouched; this is the boundary where the richer count -/// would otherwise be dropped. Crate-private: it is the seam -/// `crate::sources::sync` reads through, not host surface. +/// [`run_source_pipeline`] returning [`SourcePipelineOutcome`], which +/// additionally carries `tree_ingest_failures` — the "fetch committed, tree +/// ingest did not" count a sync verdict must not launder into success +/// (openhuman#5820). The engine's outcome type stays untouched; this is the +/// boundary where the richer count would otherwise be dropped. Crate-private: +/// it is the seam `crate::sources::sync` reads through, not host surface. pub(crate) async fn run_source_pipeline_core( source: &MemorySourceEntry, config: &Config, -) -> Result { - // Composio sources run on the engine-free pipelines (#18 §B1); this seam - // keeps only the tree-coupled kinds (folder/repo/rss/web — they summarise - // into the engine tree by design) and converts at the boundary for its - // OpenHuman-facing callers. +) -> Result { + // Composio sources are read by the connector module, not here: reaching a + // connected account needs a credential this crate does not hold and must + // not. The host fetches through `tinyconnectors` and hands the records + // back through `MemorySourceSink::accept_source_items`. + // + // Refused rather than skipped. A pipeline that answered "0 records, no + // error" for a source it never read would advance the caller's cursor past + // items nobody looked at, and report a healthy sync while the user's mail + // stopped arriving. if source.kind == SourceKind::Composio { - let toolkit = source - .toolkit - .as_deref() - .map(str::trim) - .filter(|toolkit| !toolkit.is_empty()) - .ok_or_else(|| SourcePipelineFailure::without_usage("composio source missing toolkit"))? - .to_ascii_lowercase(); - let connection_id = source - .connection_id - .as_deref() - .map(str::trim) - .filter(|connection_id| !connection_id.is_empty()) - .ok_or_else(|| { - SourcePipelineFailure::without_usage("composio source missing connection_id") - })?; - return crate::sync::pipelines::host::run_composio_connection_with_caps( - &toolkit, - connection_id, - config, - crate::sync::pipelines::host::SourceCaps::from_source(source), - ) - .await - .map_err(|failure| SourcePipelineFailure { - message: failure.message, - actions_called: failure.actions_called, - provider_cost_usd: failure.provider_cost_usd, - }); + return Err(SourcePipelineFailure::without_usage( + "composio sources are synced through the connector module, not this pipeline", + )); } let memory = crate::global::client_if_ready() @@ -419,7 +427,7 @@ pub(crate) async fn run_source_pipeline_core( provider_cost_usd: usage.map_or(0.0, |error| error.provider_cost_usd), } })?; - Ok(crate::sync::pipelines::traits::SyncOutcome { + Ok(SourcePipelineOutcome { records_ingested: outcome.records_ingested, more_pending: outcome.more_pending, actions_called: outcome.actions_called, @@ -429,157 +437,6 @@ pub(crate) async fn run_source_pipeline_core( }) } -/// Run a Composio connection through tinycortex, preserving any source-level -/// budgets already configured in OpenHuman's registry. -pub async fn run_composio_connection( - toolkit: &str, - connection_id: &str, - config: &Config, -) -> Result { - run_composio_connection_with_budgets(toolkit, connection_id, config, None, None).await -} - -/// Run a Composio connection with request-scoped budget overrides. -/// -/// Provider RPCs carry these values in `ProviderContext`, before a source has -/// necessarily been persisted in the registry. Explicit values therefore take -/// precedence, while `None` preserves the registered/default source budget. -pub async fn run_composio_connection_with_budgets( - toolkit: &str, - connection_id: &str, - config: &Config, - max_items: Option, - sync_depth_days: Option, -) -> Result { - let mut source = crate::sources::decode_memory_sources(config) - .iter() - .find(|source| { - source.kind == SourceKind::Composio - && source.connection_id.as_deref() == Some(connection_id) - }) - .cloned() - .unwrap_or_else(|| { - let (max_items, sync_depth_days) = - crate::sources::memory_sync_defaults_for_toolkit(toolkit); - MemorySourceEntry { - id: format!("composio:{toolkit}:{connection_id}"), - kind: SourceKind::Composio, - label: format!("{toolkit} connection"), - enabled: true, - toolkit: Some(toolkit.to_ascii_lowercase()), - connection_id: Some(connection_id.to_string()), - path: None, - glob: None, - url: None, - branch: None, - paths: Vec::new(), - max_commits: None, - max_issues: None, - max_prs: None, - query: None, - since_days: None, - max_items, - selector: None, - max_tokens_per_sync: None, - max_cost_per_sync_usd: None, - sync_depth_days, - } - }); - - source.max_items = max_items; - source.sync_depth_days = sync_depth_days; - - tracing::debug!( - toolkit, - connection_id, - source_id = %source.id, - max_items = ?source.max_items, - sync_depth_days = ?source.sync_depth_days, - "[tinycortex:sync] dispatching Composio connection" - ); - run_source_pipeline(&source, config).await -} - -/// Load the persisted Composio sync state, in core's own vocabulary. -/// -/// Was typed with the engine's `SyncState`; the copies share one serde shape -/// and one KV namespace (pinned by tests in -/// `sync::composio::providers::sync_state`), so the retype changes no bytes. -/// Kept in the engine module only because OpenHuman reaches it through the -/// engine shim path. -pub async fn load_composio_sync_state( - toolkit: &str, - connection_id: &str, -) -> anyhow::Result { - // `load` is an extension-trait method since the state shape moved to the - // contract crate (#5560); the trait has to be in scope to call it. - use crate::sync::composio::providers::sync_state::PersistedSyncState; - - let memory = crate::global::client_if_ready() - .ok_or_else(|| anyhow::anyhow!("memory client is not ready"))?; - let host = crate::sync::pipelines::host::PipelineHost::without_tree_ingest(memory); - crate::sync::composio::providers::sync_state::SyncState::load(&host, toolkit, connection_id) - .await -} - -pub async fn run_slack_search_backfill( - connection_id: &str, - backfill_days: i64, - config: &Config, -) -> Result { - // Delegates to the engine-free pipelines (#18 §B1); kept here because - // OpenHuman reaches this function through the engine shim path. - let outcome = crate::sync::pipelines::host::run_slack_search_backfill( - connection_id, - backfill_days, - config, - ) - .await - .map_err(|failure| SourcePipelineFailure { - message: failure.message, - actions_called: failure.actions_called, - provider_cost_usd: failure.provider_cost_usd, - })?; - Ok(SyncOutcome { - records_ingested: outcome.records_ingested, - more_pending: outcome.more_pending, - actions_called: outcome.actions_called, - provider_cost_usd: outcome.provider_cost_usd, - note: outcome.note, - }) -} - -/// Delegates to the engine-free pipelines (#18 §B1); kept because OpenHuman's -/// backfill binary reaches it through the engine shim path. -pub async fn run_gmail_backfill( - connection_id: &str, - query: &str, - max_pages: usize, - page_size: usize, - config: &Config, -) -> Result { - let outcome = crate::sync::pipelines::host::run_gmail_backfill( - connection_id, - query, - max_pages, - page_size, - config, - ) - .await - .map_err(|failure| SourcePipelineFailure { - message: failure.message, - actions_called: failure.actions_called, - provider_cost_usd: failure.provider_cost_usd, - })?; - Ok(SyncOutcome { - records_ingested: outcome.records_ingested, - more_pending: outcome.more_pending, - actions_called: outcome.actions_called, - provider_cost_usd: outcome.provider_cost_usd, - note: outcome.note, - }) -} - fn build_pipeline( source: &MemorySourceEntry, _config: &Config, @@ -756,34 +613,6 @@ impl SyncStateStore for HostSyncAdapter { } } -/// Core's state seam, on the engine adapter. -/// -/// `SyncState` is core-owned now (#18 §B1a) and its `load`/`save` take -/// core's `SyncStateStore`; OpenHuman pairs that type with this adapter in -/// its integration tests. Same KV calls as the engine-trait impl below — -/// one storage, two trait names during the transition. -#[async_trait] -impl crate::sync::composio::providers::sync_state::SyncStateStore for HostSyncAdapter { - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - self.memory - .kv_get(Some(namespace), key) - .await - .map_err(anyhow::Error::msg) - } - - async fn set( - &self, - namespace: &str, - key: &str, - value: &serde_json::Value, - ) -> anyhow::Result<()> { - self.memory - .kv_set(Some(namespace), key, value) - .await - .map_err(anyhow::Error::msg) - } -} - #[async_trait] impl SyncEventSink for HostSyncAdapter { async fn emit(&self, event: SyncEvent) -> anyhow::Result<()> { diff --git a/crates/tinymemory-core/src/engine/sync_tests.rs b/crates/tinymemory-core/src/engine/sync_tests.rs index 940a0a5c..c9ff08fd 100644 --- a/crates/tinymemory-core/src/engine/sync_tests.rs +++ b/crates/tinymemory-core/src/engine/sync_tests.rs @@ -1,12 +1,7 @@ //! Tests for the surrounding module. -use super::{ - build_pipeline, run_composio_connection, run_composio_connection_with_budgets, - run_gmail_backfill, run_slack_search_backfill, run_source_pipeline, -}; +use super::{build_pipeline, run_source_pipeline}; use crate::sources::MemorySourceEntry; -use crate::sync::composio::{get_composio_sync_provider, init_default_composio_sync_providers}; -use crate::sync::pipelines::host::{is_composio_toolkit_syncable, syncable_composio_toolkits}; /// The context the production path used to build inline; kept here since /// `run_source_pipeline_core` took over that call site with a caller-held @@ -90,25 +85,6 @@ async fn adapter_state_and_document_seams_round_trip_locally() { .expect("get engine state"), Some(value.clone()) ); - crate::sync::composio::providers::sync_state::SyncStateStore::set( - &adapter, - "host-state", - "slack", - &value, - ) - .await - .expect("set host state"); - assert_eq!( - crate::sync::composio::providers::sync_state::SyncStateStore::get( - &adapter, - "host-state", - "slack", - ) - .await - .expect("get host state"), - Some(value) - ); - SkillDocSink::store( &adapter, SkillDocument { @@ -305,25 +281,21 @@ fn pipeline_builder_covers_every_tree_coupled_source_kind() { } #[tokio::test] -async fn composio_validation_reports_missing_fields_without_usage() { +async fn composio_sources_are_refused_by_this_pipeline() { + // Composio sources are read by the connector module, not the engine: this + // seam must refuse rather than half-dispatch, regardless of which fields + // are present (openhuman#18 connector extraction). let config = tinymemory_api::host::test_support::TestHostConfig::default(); - let missing_toolkit = source( + let source = source( "composio", - serde_json::json!({"connection_id": "connection-1"}), + serde_json::json!({"toolkit": "gmail", "connection_id": "connection-1"}), ); - let failure = super::run_source_pipeline(&missing_toolkit, &config) + let failure = super::run_source_pipeline(&source, &config) .await - .expect_err("toolkit is required"); - assert!(failure.message.contains("missing toolkit")); + .expect_err("composio sources are refused, not dispatched"); + assert!(failure.message.contains("connector module")); assert_eq!(failure.actions_called, 0); assert_eq!(failure.provider_cost_usd, 0.0); - - let missing_connection = source("composio", serde_json::json!({"toolkit": "gmail"})); - let failure = super::run_source_pipeline(&missing_connection, &config) - .await - .expect_err("connection is required"); - assert!(failure.message.contains("missing connection_id")); - assert_eq!(failure.actions_called, 0); } #[tokio::test] @@ -357,36 +329,6 @@ async fn raw_archive_and_stage_helpers_cover_empty_local_state() { ); } -#[tokio::test] -async fn public_composio_wrappers_fail_before_transport_and_preserve_zero_usage() { - let (_tmp, config, _memory) = memory_fixture(); - for result in [ - run_composio_connection("gmail", "connection-missing-auth", &config).await, - run_composio_connection_with_budgets( - "slack", - "connection-missing-auth", - &config, - Some(7), - Some(2), - ) - .await, - run_slack_search_backfill("connection-missing-auth", 14, &config).await, - run_gmail_backfill( - "connection-missing-auth", - "after:2024/01/01", - 2, - 25, - &config, - ) - .await, - ] { - let failure = result.unwrap_err(); - assert_eq!(failure.actions_called, 0); - assert_eq!(failure.provider_cost_usd, 0.0); - assert!(!failure.message.is_empty()); - } -} - #[tokio::test] async fn source_pipeline_invalid_local_input_fails_without_provider_usage() { let (_tmp, config, _memory) = memory_fixture(); @@ -397,67 +339,6 @@ async fn source_pipeline_invalid_local_input_fails_without_provider_usage() { assert!(!failure.message.is_empty()); } -/// The advertised set (`memory_sources.supported_toolkits`, sourced from the -/// provider registry) and the syncable set (`build_pipeline`) must not -/// diverge: a toolkit that is advertised but has no pipeline reports ACTIVE -/// and then silently never ingests — the exact defect of #4957. -/// -/// Both directions are asserted against an explicit built-in slug set. The -/// provider registry is process-global and sibling tests register throwaway -/// providers into it without unregistering, so walking it directly would be -/// order-flaky; pinning the built-in set keeps this deterministic. -#[test] -fn advertised_and_syncable_toolkit_sets_cannot_diverge() { - init_default_composio_sync_providers(); - - // Every syncable toolkit must have a registered provider — otherwise it - // could never be advertised or auto-registered in the first place. - for &slug in syncable_composio_toolkits() { - assert!( - get_composio_sync_provider(slug).is_some(), - "syncable toolkit `{slug}` has no registered memory-sync provider" - ); - } - - // Every built-in provider shipped by `init_default_composio_sync_providers` - // must be syncable. This is the #4957 direction: advertising a provider - // that `build_pipeline` rejects is the silent failure we guard against. - // - // We pin the built-in slug set explicitly rather than walking - // `all_composio_sync_providers()`: that registry is process-global and - // sibling tests register throwaway providers into it that they never - // unregister (e.g. `provideronly` in composio/tools_tests.rs, `stub-no-active` - // in composio/identity.rs), so a raw registry walk fails nondeterministically - // depending on test execution order. A new built-in toolkit must be added to - // this list, to `syncable_composio_toolkits`, and to `build_pipeline` together - // — the assert_eq below fails loudly if the first two ever drift apart. - const BUILTIN_SYNC_PROVIDERS: &[&str] = - &["clickup", "github", "gmail", "linear", "notion", "slack"]; - - let mut builtin = BUILTIN_SYNC_PROVIDERS.to_vec(); - builtin.sort_unstable(); - let mut syncable = syncable_composio_toolkits().to_vec(); - syncable.sort_unstable(); - assert_eq!( - builtin, syncable, - "the built-in provider set and syncable set diverged — a provider is \ - advertised without a matching `build_pipeline` arm, or vice versa (#4957)" - ); - - for &slug in BUILTIN_SYNC_PROVIDERS { - assert!( - get_composio_sync_provider(slug).is_some(), - "built-in provider `{slug}` is not registered by \ - init_default_composio_sync_providers" - ); - assert!( - is_composio_toolkit_syncable(slug), - "built-in provider `{slug}` is advertised but has no build_pipeline arm — \ - it would report ACTIVE and silently fail to sync (#4957)" - ); - } -} - /// Behavioural regression for #4957: an unsupported Composio toolkit is /// rejected by `build_pipeline` *before* any credential/client resolution. /// @@ -495,19 +376,6 @@ fn build_pipeline_refuses_composio_sources() { ); } -/// Locks the reported prod failures (googlecalendar / googlesheets) as -/// non-syncable, and pins case-insensitive/trimming behaviour. -#[test] -fn is_composio_toolkit_syncable_classifies_known_slugs() { - assert!(!is_composio_toolkit_syncable("googlecalendar")); - assert!(!is_composio_toolkit_syncable("googlesheets")); - assert!(!is_composio_toolkit_syncable("discord")); - assert!(!is_composio_toolkit_syncable("")); - assert!(is_composio_toolkit_syncable("gmail")); - assert!(is_composio_toolkit_syncable("Gmail")); - assert!(is_composio_toolkit_syncable(" slack ")); -} - /// Regression for #5473: a Composio connector sync must feed the memory tree, /// not just the `skill-` document store. The TinyCortex migration /// (#4794) dropped the tree-ingest half, so synced items stopped producing diff --git a/crates/tinymemory-core/src/lib.rs b/crates/tinymemory-core/src/lib.rs index 8d9b5fbb..079670fe 100644 --- a/crates/tinymemory-core/src/lib.rs +++ b/crates/tinymemory-core/src/lib.rs @@ -33,7 +33,6 @@ pub type Config = dyn tinymemory_api::host::MemoryHostConfig; pub mod chat; pub mod chat_host; -pub mod composio_host; pub mod config_loader; pub mod conversations; pub(crate) mod corruption; diff --git a/crates/tinymemory-core/src/sources/readers/composio.rs b/crates/tinymemory-core/src/sources/readers/composio.rs deleted file mode 100644 index d7c69d86..00000000 --- a/crates/tinymemory-core/src/sources/readers/composio.rs +++ /dev/null @@ -1,73 +0,0 @@ -//! Composio source reader — delegates to the existing composio sync layer. -//! -//! For Composio sources, `list_items` returns the sync targets and -//! `read_item` is not meaningful (sync is provider-driven, not -//! item-by-item). The reader exists so the registry can uniformly -//! query all source kinds. - -use async_trait::async_trait; - -#[cfg(test)] -use tinymemory_api::host::test_support::TestHostConfig; - -use crate::sources::types::{ - ContentType, MemorySourceEntry, SourceContent, SourceItem, SourceKind, -}; -use crate::Config; - -use super::SourceReader; - -pub struct ComposioReader; - -#[async_trait] -impl SourceReader for ComposioReader { - fn kind(&self) -> SourceKind { - SourceKind::Composio - } - - async fn list_items( - &self, - source: &MemorySourceEntry, - _config: &Config, - ) -> Result, String> { - let toolkit = source.toolkit.as_deref().unwrap_or("unknown"); - let connection_id = source.connection_id.as_deref().unwrap_or("unknown"); - - tracing::debug!( - toolkit = %toolkit, - connection_id = %connection_id, - "[memory_sources:composio] list_items" - ); - - Ok(vec![SourceItem { - id: connection_id.to_string(), - title: format!("{toolkit} connection"), - updated_at_ms: None, - }]) - } - - async fn read_item( - &self, - source: &MemorySourceEntry, - item_id: &str, - _config: &Config, - ) -> Result { - let toolkit = source.toolkit.as_deref().unwrap_or("unknown"); - Ok(SourceContent { - id: item_id.to_string(), - title: format!("{toolkit} sync data"), - body: format!( - "Composio {toolkit} data is synced via the provider sync pipeline, not read item-by-item." - ), - content_type: ContentType::Plaintext, - metadata: serde_json::json!({ - "toolkit": toolkit, - "connection_id": source.connection_id, - }), - }) - } -} - -#[cfg(test)] -#[path = "composio_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sources/readers/composio_tests.rs b/crates/tinymemory-core/src/sources/readers/composio_tests.rs deleted file mode 100644 index bb25ffe4..00000000 --- a/crates/tinymemory-core/src/sources/readers/composio_tests.rs +++ /dev/null @@ -1,39 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; -use crate::sources::types::MemorySourceEntry; - -fn test_source() -> MemorySourceEntry { - MemorySourceEntry { - id: "src_1".into(), - kind: SourceKind::Composio, - label: "Gmail".into(), - enabled: true, - toolkit: Some("gmail".into()), - connection_id: Some("cmp_123".into()), - path: None, - glob: None, - url: None, - branch: None, - paths: Vec::new(), - query: None, - since_days: None, - max_items: None, - max_commits: None, - max_issues: None, - max_prs: None, - selector: None, - max_tokens_per_sync: None, - max_cost_per_sync_usd: None, - sync_depth_days: None, - } -} - -#[tokio::test] -async fn list_items_returns_connection_as_item() { - let reader = ComposioReader; - let config = TestHostConfig::default(); - let items = reader.list_items(&test_source(), &config).await.unwrap(); - assert_eq!(items.len(), 1); - assert_eq!(items[0].id, "cmp_123"); -} diff --git a/crates/tinymemory-core/src/sources/readers/mod.rs b/crates/tinymemory-core/src/sources/readers/mod.rs index 0836d768..c3ebe1ab 100644 --- a/crates/tinymemory-core/src/sources/readers/mod.rs +++ b/crates/tinymemory-core/src/sources/readers/mod.rs @@ -1,6 +1,5 @@ //! Source reader trait and per-kind implementations. -pub mod composio; pub mod conversation; pub mod folder; pub mod github; @@ -30,15 +29,28 @@ pub trait SourceReader: Send + Sync { ) -> Result; } -/// Get the reader for a given source kind. -pub fn reader_for(kind: &SourceKind) -> Box { +/// Get the reader for a given source kind, if this crate has one. +/// +/// `None` for [`SourceKind::Composio`]. The kind itself stays — records +/// synced from a connected account are still stored, still queried, and still +/// forgotten under it, and removing it would orphan every row already written. +/// What left is the *reading*: an OAuth connector is reached with a credential +/// this crate does not hold and must not, so the host fetches through +/// `tinyconnectors` and hands the records here through +/// [`crate::provider::MemorySourceSink`]. +/// +/// Returning `Option` rather than a stub reader that always errors is +/// deliberate: a caller has to decide what to do about a kind it cannot read, +/// and a stub would let it call and discover the same thing at runtime, once +/// per item. +pub fn reader_for(kind: &SourceKind) -> Option> { match kind { - SourceKind::Composio => Box::new(composio::ComposioReader), - SourceKind::Conversation => Box::new(conversation::ConversationReader), - SourceKind::Folder => Box::new(folder::FolderReader), - SourceKind::GithubRepo => Box::new(github::GithubReader), - SourceKind::TwitterQuery => Box::new(twitter::TwitterReader), - SourceKind::RssFeed => Box::new(rss::RssReader::new()), - SourceKind::WebPage => Box::new(web_page::WebPageReader), + SourceKind::Composio => None, + SourceKind::Conversation => Some(Box::new(conversation::ConversationReader)), + SourceKind::Folder => Some(Box::new(folder::FolderReader)), + SourceKind::GithubRepo => Some(Box::new(github::GithubReader)), + SourceKind::TwitterQuery => Some(Box::new(twitter::TwitterReader)), + SourceKind::RssFeed => Some(Box::new(rss::RssReader::new())), + SourceKind::WebPage => Some(Box::new(web_page::WebPageReader)), } } diff --git a/crates/tinymemory-core/src/sources/reconcile.rs b/crates/tinymemory-core/src/sources/reconcile.rs index 0e1e1abe..8ba1e0e5 100644 --- a/crates/tinymemory-core/src/sources/reconcile.rs +++ b/crates/tinymemory-core/src/sources/reconcile.rs @@ -11,111 +11,11 @@ use crate::config_loader as config_rpc; use crate::sources::registry; use crate::sources::types::{MemorySourceEntry, SourceKind}; -use crate::sync::composio; -use std::collections::HashSet; /// Current version of the caps migration. Bump when the migration logic changes /// so installs that ran an earlier revision re-run it exactly once. const CURRENT_CAPS_MIGRATION_VERSION: u32 = 1; -/// Reconcile active Composio connections into the memory sources registry and -/// return the live active-connection set scanned this call. -/// -/// Returns `Some(connection_ids)` — the `connection_id`s of every active sync -/// target — when the live Composio scan **succeeded**, so callers (notably -/// `rpc::list_rpc`) can filter the listing down to connections that are still -/// active and dedupe identical rows. Returns `None` when the scan could not run -/// (config load / network / auth failure); callers must treat `None` as "active -/// set unavailable" and **not** hide any sources — an empty scan from a transient -/// blip must never be read as "everything is inactive". -pub async fn ensure_composio_sources() -> Option> { - tracing::debug!("[memory_sources:reconcile] starting composio reconciliation"); - - let config = match config_rpc::load_config_with_timeout().await { - Ok(c) => c, - Err(e) => { - tracing::warn!( - error = %e, - "[memory_sources:reconcile] failed to load config; skipping" - ); - return None; - } - }; - - // Always hit Composio directly here — using list_sync_targets would - // short-circuit through the registry and miss new connections. - let targets = match composio::scan_active_sync_targets(&*config).await { - Ok(t) => t, - Err(e) => { - tracing::debug!( - error = %e, - "[memory_sources:reconcile] no composio sync targets available; skipping" - ); - return None; - } - }; - - // Build the upsert targets up front, then apply them with a single config - // load + save via the batch path. The per-call upsert does its own - // load-modify-save, so the old loop cost 2N config round-trips for N - // connections; batching collapses that to 2. - let upsert_targets = build_upsert_targets(&targets); - let upserted = match registry::upsert_composio_sources_batch(&upsert_targets).await { - Ok(n) => n, - Err(e) => { - tracing::warn!( - targets = targets.len(), - error = %e, - "[memory_sources:reconcile] batch upsert failed" - ); - 0 - } - }; - - if !targets.is_empty() { - tracing::info!( - targets = targets.len(), - upserted = upserted, - "[memory_sources:reconcile] composio reconciliation complete" - ); - } - - // Run the one-time caps migration after the reconcile loop so any - // sources upserted just above are also considered. - if let Err(e) = apply_composio_source_caps_migration().await { - tracing::warn!( - error = %e, - "[memory_sources:reconcile] caps migration failed (non-fatal, will retry next time)" - ); - } - - // The scan succeeded — surface the live active-connection set so the list - // path can hide rows for connections that are no longer active (re-auth / - // token expiry mints a fresh connection_id, stranding the old row) and - // collapse identical same-id duplicates. - Some(targets.iter().map(|t| t.connection_id.clone()).collect()) -} - -/// Build the `(toolkit, connection_id, label)` upsert targets for a batch -/// reconcile from the scanned Composio sync targets. -/// -/// The label is a title-cased toolkit name plus the truncated connection id so -/// distinct accounts of the same toolkit (e.g. two Gmail logins) don't all show -/// as "Gmail connection". Pure (no I/O) so it can be unit-tested directly. -fn build_upsert_targets(targets: &[composio::SyncTarget]) -> Vec { - targets - .iter() - .map(|target| { - let label = format!( - "{} · {}", - title_case(&target.toolkit), - short_id(&target.connection_id) - ); - (target.toolkit.clone(), target.connection_id.clone(), label) - }) - .collect() -} - /// Apply conservative default caps in-place to every cap-less source. /// /// For a Composio source with no `max_items`/`sync_depth_days`, writes the diff --git a/crates/tinymemory-core/src/sources/reconcile_tests.rs b/crates/tinymemory-core/src/sources/reconcile_tests.rs index 32e86df1..e36e3d94 100644 --- a/crates/tinymemory-core/src/sources/reconcile_tests.rs +++ b/crates/tinymemory-core/src/sources/reconcile_tests.rs @@ -105,36 +105,6 @@ fn migration_applies_correct_defaults_per_toolkit() { } } -fn sync_target(toolkit: &str, connection_id: &str) -> composio::SyncTarget { - composio::SyncTarget { - toolkit: toolkit.to_string(), - connection_id: connection_id.to_string(), - } -} - -#[test] -fn build_upsert_targets_formats_label_and_preserves_order() { - let targets = vec![ - sync_target("gmail", "ca_WaktIDFlZwXO"), - sync_target("slack", "short"), - ]; - let out = build_upsert_targets(&targets); - assert_eq!(out.len(), 2); - // (toolkit, connection_id, label) — toolkit/connection_id carried through verbatim. - assert_eq!(out[0].0, "gmail"); - assert_eq!(out[0].1, "ca_WaktIDFlZwXO"); - assert_eq!(out[0].2, "Gmail · IDFlZwXO"); - assert_eq!(out[1].0, "slack"); - assert_eq!(out[1].1, "short"); - assert_eq!(out[1].2, "Slack · short"); -} - -#[test] -fn build_upsert_targets_empty_is_empty() { - let out = build_upsert_targets(&[]); - assert!(out.is_empty()); -} - #[test] fn short_id_truncates_ascii() { assert_eq!(short_id("ca_WaktIDFlZwXO"), "IDFlZwXO"); diff --git a/crates/tinymemory-core/src/sources/sync.rs b/crates/tinymemory-core/src/sources/sync.rs index a51aa2b3..56aa626d 100644 --- a/crates/tinymemory-core/src/sources/sync.rs +++ b/crates/tinymemory-core/src/sources/sync.rs @@ -18,7 +18,7 @@ use std::sync::Mutex; use tinymemory_api::host::test_support::TestHostConfig; use crate::sources::types::{MemorySourceEntry, SourceKind}; -use crate::sync::composio::ComposioUsage; +use crate::sync::usage::ProviderUsage; use crate::sync_events::{emit_sync_stage, MemorySyncStage, MemorySyncTrigger}; use crate::Config; @@ -86,7 +86,7 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu let sync_start = std::time::Instant::now(); // Composio billable-action usage for this run, populated by // `sync_composio` (#3111). Stays zero for non-Composio kinds. - let mut composio_usage = ComposioUsage::default(); + let mut composio_usage = ProviderUsage::default(); // Every kind runs through `run_source_pipeline_core`, whose // outcome carries `tree_ingest_failures` — the count of items that // were fetched-and-stored but never reached the memory tree. The diff --git a/crates/tinymemory-core/src/store/entities.rs b/crates/tinymemory-core/src/store/entities.rs index 7836d96b..f96c69a3 100644 --- a/crates/tinymemory-core/src/store/entities.rs +++ b/crates/tinymemory-core/src/store/entities.rs @@ -8,7 +8,7 @@ use crate::engine::backend::store::entity_index::{ use anyhow::Result; use crate::engine::memory_config_from; -use crate::sync::composio::providers::profile::{is_self_identity_any_toolkit, IdentityKind}; +use crate::store::identity::{is_self_identity_any_toolkit, IdentityKind}; use crate::Config; /// Aggregate entity-index row for capability providers. diff --git a/crates/tinymemory-core/src/store/identity.rs b/crates/tinymemory-core/src/store/identity.rs new file mode 100644 index 00000000..1a5e4c7a --- /dev/null +++ b/crates/tinymemory-core/src/store/identity.rs @@ -0,0 +1,146 @@ +//! Which identities on a stored row are the user's own. +//! +//! # Why this is here and not with the connector +//! +//! It reads *this crate's* profile store — the `skill::: +//! ` rows written when a connected account's profile is ingested — and +//! answers a question the memory tree asks while building entity rows: is this +//! email address the user themselves? +//! +//! It lived under the Composio sync tree only because that is what wrote the +//! rows. Writing them is the connector module's job now; reading them was +//! never anything but memory's, so it stays. + +use serde::{Deserialize, Serialize}; + +/// The facet kinds a connected account's profile is stored as. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IdentityKind { + /// Platform-canonical immutable id — Slack `U123ABC`, Notion UUID. + UserId, + Email, + /// `@`-style screen name, canonicalised without the leading `@`. + Handle, + /// E.164 phone number. + Phone, + /// Human display label. Weak signal — never auto-promotes to is_self. + DisplayName, + /// Not for matching; kept for UI / prompt rendering. + AvatarUrl, + /// Not for matching; kept for UI / prompt rendering. + ProfileUrl, +} + +impl IdentityKind { + pub fn as_str(self) -> &'static str { + match self { + Self::UserId => "user_id", + Self::Email => "email", + Self::Handle => "handle", + Self::Phone => "phone", + Self::DisplayName => "display_name", + Self::AvatarUrl => "avatar_url", + Self::ProfileUrl => "profile_url", + } + } + + pub fn parse(s: &str) -> Option { + Some(match s { + "user_id" => Self::UserId, + "email" => Self::Email, + "handle" => Self::Handle, + "phone" => Self::Phone, + "display_name" => Self::DisplayName, + "avatar_url" => Self::AvatarUrl, + "profile_url" => Self::ProfileUrl, + _ => return None, + }) + } + + /// Confidence the matcher records on the row. Hard kinds auto-promote + /// a chunk to `is_self`; weak kinds require corroboration. + pub fn confidence(self) -> f64 { + match self { + Self::UserId | Self::Phone => 1.00, + Self::Email => 0.95, + Self::Handle => 0.70, + Self::DisplayName => 0.40, + Self::AvatarUrl | Self::ProfileUrl => 0.50, + } + } + + /// True if this kind is a real identity signal worth running through + /// the matcher (vs. UI-only fields). + pub fn is_matchable(self) -> bool { + matches!( + self, + Self::UserId | Self::Email | Self::Handle | Self::Phone | Self::DisplayName + ) + } +} + +/// Canonicalize a raw value for storage and lookup. The same routine runs +/// on the entity side at match time, so equality of canonical forms is the +/// matcher's only test — no `COLLATE NOCASE`, no per-call lowercasing. +pub fn canonicalize(kind: IdentityKind, raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + Some(match kind { + IdentityKind::Email => trimmed.to_lowercase(), + IdentityKind::Handle => trimmed.trim_start_matches('@').to_lowercase(), + IdentityKind::Phone => trimmed + .chars() + .filter(|c| c.is_ascii_digit() || *c == '+') + .collect(), + IdentityKind::DisplayName => trimmed.split_whitespace().collect::>().join(" "), + IdentityKind::UserId | IdentityKind::AvatarUrl | IdentityKind::ProfileUrl => { + trimmed.to_string() + } + }) +} + +/// Cross-toolkit variant — matches against every connected provider's +/// rows of this kind. Used for marking memory-tree entity rows: an email +/// in a Slack message that matches the user's Gmail address is still +/// "me," regardless of which source produced the chunk. +pub fn is_self_identity_any_toolkit(kind: IdentityKind, raw_value: &str) -> bool { + if !kind.is_matchable() { + return false; + } + let Some(canonical) = canonicalize(kind, raw_value) else { + return false; + }; + let Some(client) = crate::global::client_if_ready() else { + return false; + }; + let key_pattern = format!("skill:%:%:{}", kind.as_str()); + client + .profile_store() + .skill_identity_matches(&key_pattern, &canonical) +} + +/// Render a compact section for prompt injection. Skips `user_id` (not +/// human-readable), prefixes `handle` with `@`. + +/// Fold a token to the shape used in a profile-store key. +pub fn normalize_token(raw: &str) -> String { + let mut out = String::with_capacity(raw.len()); + for ch in raw.chars() { + let lower = ch.to_ascii_lowercase(); + if lower.is_ascii_alphanumeric() || lower == '-' || lower == '_' { + out.push(lower); + } else { + out.push('_'); + } + } + out.trim_matches('_').to_string() +} + +/// Fold a connection identifier the same way. +#[must_use] +pub fn normalize_connection_identifier(raw: &str) -> String { + normalize_token(raw) +} diff --git a/crates/tinymemory-core/src/store/mod.rs b/crates/tinymemory-core/src/store/mod.rs index ba809f36..d4aa5b9a 100644 --- a/crates/tinymemory-core/src/store/mod.rs +++ b/crates/tinymemory-core/src/store/mod.rs @@ -27,6 +27,7 @@ pub mod chunks; pub mod content; pub mod entities; +pub mod identity; pub mod kinds; pub mod kv; pub mod namespace_store; diff --git a/crates/tinymemory-core/src/sync/composio/mod.rs b/crates/tinymemory-core/src/sync/composio/mod.rs deleted file mode 100644 index 581225e4..00000000 --- a/crates/tinymemory-core/src/sync/composio/mod.rs +++ /dev/null @@ -1,218 +0,0 @@ -//! Composio-backed sync pipelines. -//! -//! This module owns the "pull upstream provider data into memory" side of -//! Composio integrations: -//! -//! - provider sync implementations (`providers/*/provider.rs`, `sync.rs`) -//! - periodic scheduler (`periodic.rs`) -//! - trigger / connection-created event subscribers (`bus.rs`) -//! - sync-state persistence and profile-to-memory shaping -//! -//! The host's sibling `integrations::composio` domain still owns auth, -//! connection management, action execution, and general Composio RPC/tool -//! surfaces. This submodule is specifically the memory-sync half of that -//! integration boundary. - -use std::sync::Arc; -pub mod periodic; -pub mod providers; - -use crate::composio_host::{self, ComposioConnection}; -use crate::Config; - -pub use periodic::{record_sync_success, start_periodic_sync}; -pub use providers::{ - all_providers as all_composio_sync_providers, get_provider as get_composio_sync_provider, - init_default_providers as init_default_composio_sync_providers, ComposioProvider, - ComposioUsage, ProviderContext, ProviderUserProfile, SyncOutcome, SyncReason, -}; - -/// One provider-backed connection that the memory sync layer can execute. -#[derive(Debug, Clone)] -pub struct SyncTarget { - pub toolkit: String, - pub connection_id: String, -} - -/// List active Composio connections that have a native memory-sync provider. -/// -/// When memory_sources entries exist with `kind=composio` and `enabled=true`, -/// those are used as the authoritative source list (user curated). When no -/// memory_sources composio entries exist, falls back to scanning all active -/// Composio connections (legacy behavior). -pub async fn list_sync_targets(config: &Config) -> Result, String> { - init_default_composio_sync_providers(); - - // Try memory_sources registry first (user-curated list). - let registry_sources = - crate::sources::list_enabled_by_kind(crate::sources::SourceKind::Composio) - .await - .unwrap_or_default(); - - if !registry_sources.is_empty() { - let from_registry: Vec = registry_sources - .into_iter() - .filter_map(|s| { - let toolkit = s.toolkit?; - let connection_id = s.connection_id?; - get_composio_sync_provider(&toolkit).map(|_| SyncTarget { - toolkit, - connection_id, - }) - }) - .collect(); - if !from_registry.is_empty() { - tracing::debug!( - count = from_registry.len(), - "[composio:sync] using memory_sources registry for sync targets" - ); - return Ok(from_registry); - } - // Registry has entries but none yielded a valid target (missing - // fields or unregistered toolkit). Fall through to a fresh scan - // rather than reporting an empty target list — otherwise newly - // connected integrations stay invisible until reconcile runs. - tracing::debug!( - "[composio:sync] registry yielded zero valid targets; falling back to connection scan" - ); - } else { - tracing::debug!( - "[composio:sync] no memory_sources entries; falling back to connection scan" - ); - } - - scan_active_sync_targets(config).await -} - -/// Scan all active Composio connections that have a native memory-sync -/// provider. Always hits Composio directly — does not consult the -/// memory_sources registry. Used by reconciliation to seed the registry. -pub async fn scan_active_sync_targets(config: &Config) -> Result, String> { - init_default_composio_sync_providers(); - - // Mode dispatch lives in the host's `ComposioHost` impl: backend mode - // walks the tinyhumans tenant, direct mode the user's own Composio v3 - // tenant. Either way this side gets one flat list. - let connections = composio_host::list_connections(config).await?; - - Ok(connections - .into_iter() - .filter_map(connection_to_sync_target) - .collect()) -} - -/// Run one provider-backed sync end-to-end in-process. -/// -/// Returns the provider's [`SyncOutcome`] together with the -/// [`ComposioUsage`] tally (billable action count + actual USD cost) -/// accumulated at the `execute` chokepoint during this run, so the -/// sync-audit caller can record Composio API-call cost alongside the LLM -/// summarisation cost (#3111). -pub async fn run_connection_sync( - config: Arc, - connection_id: &str, - reason: SyncReason, -) -> Result<(SyncOutcome, ComposioUsage), (String, ComposioUsage)> { - init_default_composio_sync_providers(); - - let no_usage = |e: String| (e, ComposioUsage::default()); - - let target = list_sync_targets(&*config) - .await - .map_err(no_usage)? - .into_iter() - .find(|target| target.connection_id == connection_id) - .ok_or_else(|| { - no_usage(format!( - "no provider-backed active sync target for connection_id={connection_id}", - )) - })?; - - let provider = get_composio_sync_provider(&target.toolkit).ok_or_else(|| { - no_usage(format!( - "no native memory sync provider registered for toolkit '{}'", - target.toolkit, - )) - })?; - - // Look up the source entry to obtain any user-configured caps. - // Non-fatal: if the registry read fails we proceed uncapped. - let (src_max_items, src_sync_depth_days) = { - let registry_sources = - crate::sources::list_enabled_by_kind(crate::sources::SourceKind::Composio) - .await - .unwrap_or_default(); - registry_sources - .iter() - .find(|s| s.connection_id.as_deref() == Some(&target.connection_id)) - .map(|s| (s.max_items, s.sync_depth_days)) - .unwrap_or((None, None)) - }; - - tracing::debug!( - connection_id = %target.connection_id, - max_items = ?src_max_items, - sync_depth_days = ?src_sync_depth_days, - "[composio:sync] run_connection_sync: caps from registry" - ); - - let _ = provider; - let started_at_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - match crate::sync::pipelines::host::run_composio_connection( - &target.toolkit, - &target.connection_id, - &*config, - src_max_items, - src_sync_depth_days, - ) - .await - { - Ok(outcome) => { - let usage = ComposioUsage { - actions_called: outcome.actions_called, - cost_usd: outcome.provider_cost_usd, - }; - Ok(( - SyncOutcome { - toolkit: target.toolkit, - connection_id: Some(target.connection_id), - reason: reason.as_str().to_string(), - items_ingested: outcome.records_ingested as usize, - started_at_ms, - finished_at_ms: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64, - summary: outcome.note.unwrap_or_else(|| "sync completed".to_string()), - details: serde_json::json!({ "more_pending": outcome.more_pending }), - }, - usage, - )) - } - Err(error) => Err(( - error.to_string(), - ComposioUsage { - actions_called: error.actions_called, - cost_usd: error.provider_cost_usd, - }, - )), - } -} - -fn connection_to_sync_target(connection: ComposioConnection) -> Option { - if !connection.is_active() { - return None; - } - let toolkit = connection.normalized_toolkit(); - get_composio_sync_provider(&toolkit).map(|_| SyncTarget { - toolkit, - connection_id: connection.id, - }) -} - -#[cfg(test)] -#[path = "mod_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/mod_tests.rs b/crates/tinymemory-core/src/sync/composio/mod_tests.rs deleted file mode 100644 index 8494e4fc..00000000 --- a/crates/tinymemory-core/src/sync/composio/mod_tests.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Tests for active-connection filtering at the Composio sync boundary. - -use super::*; - -fn connection(toolkit: &str, status: &str) -> ComposioConnection { - serde_json::from_value(serde_json::json!({ - "id": format!("connection-{toolkit}"), - "toolkit": toolkit, - "status": status - })) - .unwrap() -} - -#[test] -fn connection_target_requires_active_registered_provider() { - init_default_composio_sync_providers(); - assert!(connection_to_sync_target(connection("gmail", "inactive")).is_none()); - assert!(connection_to_sync_target(connection("unknown", "active")).is_none()); - let target = connection_to_sync_target(connection("GMAIL", "active")).unwrap(); - assert_eq!(target.toolkit, "gmail"); - assert_eq!(target.connection_id, "connection-GMAIL"); -} diff --git a/crates/tinymemory-core/src/sync/composio/periodic.rs b/crates/tinymemory-core/src/sync/composio/periodic.rs deleted file mode 100644 index 5d927174..00000000 --- a/crates/tinymemory-core/src/sync/composio/periodic.rs +++ /dev/null @@ -1,740 +0,0 @@ -//! Periodic sync scheduler for the Composio domain. -//! -//! Spawned once at startup. The scheduler walks every active Composio -//! connection on a fixed tick, looks up the matching native provider, -//! and dispatches the matching tinycortex pipeline if enough time -//! has elapsed since that connection's last sync (per the provider's -//! `sync_interval_secs`). -//! -//! ## Direct mode (`[composio-direct]`) -//! -//! As of #1710 Wave 1, the scheduler is **mode-aware**: it resolves the -//! client via `create_composio_client` each tick so a direct-mode -//! user's personal Composio v3 tenant gets walked (via -//! `direct_list_connections`) instead of returning an empty list from -//! the tinyhumans tenant. The per-connection sync calls go through -//! `ProviderContext::execute` which is itself mode-aware. -//! -//! Real-time trigger webhooks (`composio:trigger` socket.io events -//! fanned out from `wss://api.tinyhumans.ai`) still do not reach the -//! core when `config.composio().mode == "direct"`, because the backend -//! HMAC-verifies the Composio webhook and pushes it down a per-user -//! socket — direct-mode users see synchronous tool execution and -//! periodic poll-based sync, but not async trigger pushes in this -//! release. See the `composio.direct_mode_triggers_gap` capability -//! entry in `about_app/catalog.rs` for the user-visible status. -//! -//! Design notes: -//! -//! * One global tick (5min) drives every provider — we don't spawn a -//! task per connection, because the number of connections per user -//! is small and a single tick keeps the bookkeeping trivial. -//! * Per-connection state (last sync timestamp) lives in a -//! process-global `Arc>` keyed by `(toolkit, -//! connection_id)`. The map is shared with event-driven sync paths -//! (bus subscribers, `on_connection_created`) via -//! [`record_sync_success`] so a recent non-periodic sync prevents -//! the scheduler from redundantly re-firing. The map is rebuilt on -//! restart; to keep a user-configured cadence (e.g. "Sync every 24h", -//! #3302) from re-firing on every cold start, the due-check falls back -//! to the **persisted** sync-audit timestamp (`read_audit_log`) when -//! the in-memory record is absent — see `persisted_since_last_sync`. -//! * Errors are logged and swallowed; the scheduler must never panic -//! out of its loop or periodic sync stops silently for the rest of -//! the process lifetime. - -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, OnceLock}; -use std::time::{Duration, Instant}; - -use tokio::time::interval; - -use crate::config_loader as config_rpc; -use crate::scheduler_gate::PauseReason; -use crate::scheduler_gate::{current_policy, resume_notify}; -use crate::sources::{memory_sync_defaults_for_toolkit, MemorySourceEntry, SourceKind}; -use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; - -use super::providers::{get_provider, ComposioUsage}; -use crate::composio_host; -use crate::sync::audit::{append_audit_entry, read_audit_log, SyncAuditEntry}; -use chrono::{DateTime, Utc}; - -/// How often the scheduler wakes up to look for due syncs. Independent -/// from per-provider `sync_interval_secs` — this just bounds how long -/// past a provider's interval we might fire. -/// -/// 20 min trades a little staleness for noticeably less foreground load: -/// each tick triggers an HTTP fetch + DB write per due connection, and -/// for users with several connected providers the old 60s cadence kept -/// the laptop visibly busy. Per-provider `sync_interval_secs` still -/// caps the *minimum* delay between actual syncs — this only loosens -/// the upper bound. -const TICK_SECONDS: u64 = 1200; - -/// Process-wide guard so the scheduler is only started once even -/// when both `start_channels` and `bootstrap_core_runtime` call into -/// us during startup. Without this we'd end up with two parallel tick -/// loops competing for the same connections. -static SCHEDULER_STARTED: OnceLock<()> = OnceLock::new(); - -/// Process-wide map of `(toolkit, connection_id) → last successful sync -/// instant`. Shared between the periodic scheduler loop and event-driven -/// sync paths (e.g. `ComposioConnectionCreatedSubscriber`, -/// `on_connection_created`) so that a recent non-periodic sync prevents -/// the scheduler from firing immediately on the next tick. -type SyncTimestampMap = Arc>>; - -static LAST_SYNC_AT: OnceLock = OnceLock::new(); - -/// Get (or lazily initialise) the shared last-sync-at map. -fn last_sync_map() -> SyncTimestampMap { - LAST_SYNC_AT - .get_or_init(|| Arc::new(Mutex::new(HashMap::new()))) - .clone() -} - -/// Record a successful sync for the given `(toolkit, connection_id)` key. -/// Called by the periodic scheduler after a successful sync and by -/// event-driven paths (bus subscribers, `on_connection_created`) so the -/// periodic ticker respects recent non-periodic syncs. -pub fn record_sync_success(toolkit: &str, connection_id: &str) { - if let Ok(mut map) = last_sync_map().lock() { - map.insert( - (toolkit.to_string(), connection_id.to_string()), - Instant::now(), - ); - } -} - -/// Resolve the effective periodic sync interval (seconds) for one connection, -/// combining the provider's own default with the user's global -/// memory-sync cadence ([`Config::memory_sync_interval_secs`], #3302). -/// -/// - `global == Some(0)` → `None`: "Manual only" — the scheduler skips this -/// source entirely (manual sync still works). -/// - `global == Some(n)` → `Some(max(n, provider_default))`: the user's -/// cadence overrides the provider default but is floored at it, so we never -/// sync *more* often than the provider intended. -/// - `global == None` → `Some(max(DEFAULT, provider_default))`: no explicit -/// user choice, so fall back to the 24h default cadence (also floored at the -/// provider default). -pub(crate) fn effective_interval_secs(provider_default: u64, global: Option) -> Option { - match global { - Some(0) => None, - Some(n) => Some(n.max(provider_default)), - None => Some(DEFAULT_MEMORY_SYNC_INTERVAL_SECS.max(provider_default)), - } -} - -/// Decide whether a connection is due for a periodic sync right now, given the -/// effective interval and how long ago it last synced this run. -/// -/// `since_last_sync == None` means we have no record of a sync this process -/// lifetime, so we fire immediately (the restart-recovery path). Kept pure so -/// the due-check can be simulated without driving the real `Instant` clock. -pub(crate) fn connection_is_due(interval_secs: u64, since_last_sync: Option) -> bool { - match since_last_sync { - Some(elapsed) => elapsed >= Duration::from_secs(interval_secs), - None => true, - } -} - -/// Build an index of `connection_id → most recent successful Composio sync -/// timestamp` from the persisted sync audit log (#3302). -/// -/// The periodic loop writes audit entries with `source_id = connection_id` and -/// `scope = "{toolkit}:{connection_id}"`; we accept either shape and key by the -/// connection id. Only successful syncs count — matching the in-memory -/// [`record_sync_success`] semantics, which never records a failed tick so the -/// next tick retries. This is the wall-clock record that lets the cadence -/// survive restarts (the in-memory monotonic map cannot). -fn index_last_success_by_connection(entries: &[SyncAuditEntry]) -> HashMap> { - let mut idx: HashMap> = HashMap::new(); - for e in entries { - if e.source_kind != "composio" || !e.success { - continue; - } - let connection_id = e - .scope - .rsplit_once(':') - .map(|(_, c)| c.to_string()) - .filter(|c| !c.is_empty()) - .unwrap_or_else(|| e.source_id.clone()); - idx.entry(connection_id) - .and_modify(|t| { - if e.timestamp > *t { - *t = e.timestamp; - } - }) - .or_insert(e.timestamp); - } - idx -} - -/// Wall-clock elapsed since a connection's last persisted successful sync, if -/// any. Saturates at zero for a future timestamp (clock skew), so a skewed -/// record never reads as "wildly overdue". Returns `None` when the connection -/// has no persisted sync — letting the caller treat it as never-synced. -fn persisted_since_last_sync( - idx: &HashMap>, - connection_id: &str, - now: DateTime, -) -> Option { - idx.get(connection_id).map(|ts| { - let secs = (now - *ts).num_seconds().max(0) as u64; - Duration::from_secs(secs) - }) -} - -/// Outcome of consulting the per-source registry for one Composio connection -/// during a periodic tick (#2831). -#[derive(Debug, PartialEq, Eq)] -enum PeriodicSourceDecision { - /// The user toggled this source **off** — skip background sync entirely. - /// Manual `memory_sources_sync` still works (it has its own `enabled` - /// guard); only the automatic loop honours this here. - Skip, - /// Sync this connection with the given caps (`None` = uncapped for that - /// dimension). - Sync { - max_items: Option, - sync_depth_days: Option, - }, -} - -/// Decide whether — and with what caps — to periodically sync one connection, -/// honouring the per-source `enabled` toggle (#2831). Pure so the three -/// branches can be unit-tested without async/registry I/O. -/// -/// - **disabled row** → [`PeriodicSourceDecision::Skip`]: the background loop -/// must not sync a source the user switched off (this was the row-2 leak — -/// previously the loop only read the registry for caps and synced disabled -/// sources anyway, uncapped). -/// - **enabled row** → `Sync` with the row's caps. -/// - **no row yet** → `Sync` with conservative per-toolkit defaults -/// ([`memory_sync_defaults_for_toolkit`]). `reconcile` normally backfills an -/// enabled, capped row for every connection; this covers the brief -/// pre-reconcile window. The no-match path defaults to *sync-bounded*, never -/// *skip* and never *uncapped*, so a missing/mismatched row degrades safely -/// (data keeps flowing, just capped) instead of silently going dark. -fn decide_periodic_source( - source: Option<&MemorySourceEntry>, - toolkit: &str, -) -> PeriodicSourceDecision { - match source { - Some(s) if !s.enabled => PeriodicSourceDecision::Skip, - Some(s) => PeriodicSourceDecision::Sync { - max_items: s.max_items, - sync_depth_days: s.sync_depth_days, - }, - None => { - let (max_items, sync_depth_days) = memory_sync_defaults_for_toolkit(toolkit); - PeriodicSourceDecision::Sync { - max_items, - sync_depth_days, - } - } - } -} - -/// Spawn the periodic sync background task. Idempotent: only the -/// first call actually spawns the loop, every subsequent call is a -/// cheap no-op (logged at `debug` so it's visible during startup -/// tracing without spamming `info`). -pub fn start_periodic_sync() { - if SCHEDULER_STARTED.get().is_some() { - tracing::debug!("[composio:periodic] scheduler already running, skipping start"); - return; - } - // Race-safe: only the thread that wins `set` runs the spawn body. - if SCHEDULER_STARTED.set(()).is_err() { - tracing::debug!("[composio:periodic] scheduler already running (race), skipping start"); - return; - } - - tokio::spawn(async move { - tracing::info!( - tick_seconds = TICK_SECONDS, - "[composio:periodic] scheduler starting" - ); - run_loop().await; - // run_loop only returns on a fatal error in the bus — log it - // so the silent stop is at least visible in the trace. - tracing::error!("[composio:periodic] scheduler loop exited"); - }); -} - -/// Inner loop, broken out so it's easy to mock-replace in tests if we -/// ever want to drive ticks deterministically. -/// -/// Each iteration waits on whichever comes first (#2831): -/// * the 20-min `ticker` — the steady-state cadence, or -/// * the scheduler-gate **resume** notify — fired when the user toggles -/// Memory Tree back on or signs back in. -/// -/// On a resume wake we run a tick **immediately** (so sync restarts within -/// seconds, not at the next ≤20-min boundary) and `reset()` the ticker so the -/// *next* scheduled tick is a full `TICK_SECONDS` out. The reset is what stops -/// rapid off-on toggling from bunch-firing: many wakes collapse into at most -/// one extra tick (the `Notify` stores a single permit), and the cadence -/// re-bases from the last actual tick. -async fn run_loop() { - let mut ticker = interval(Duration::from_secs(TICK_SECONDS)); - let resume = resume_notify(); - // Skip the immediate-fire tick so startup isn't slammed before the - // user even has time to sign in. - ticker.tick().await; - - loop { - tokio::select! { - _ = ticker.tick() => {} - _ = resume.notified() => { - // Woke early on a resume transition. Re-base the cadence so the - // next scheduled tick is TICK_SECONDS from now, then fall - // through and run the tick immediately. - ticker.reset(); - } - } - if let Err(e) = run_one_tick().await { - tracing::warn!( - error = %e, - "[composio:periodic] tick failed (continuing)" - ); - } - } -} - -/// Inspect the scheduler-gate policy and decide whether this tick should -/// fire at all. Returns `Some(reason)` for paused states so the caller can -/// log a single, attributable line instead of doing the work and discovering -/// per-LLM-call later that everything's gated. -/// -/// Covers two reasons the memory subsystem treats as "do no background -/// work": -/// - [`PauseReason::UserDisabled`] — user flipped the Memory Tree toggle off -/// in Settings (#1856 Part 1). The 20-min Composio fetch loop honouring -/// this flag is the explicit follow-up listed in the #2719 PR body. -/// - [`PauseReason::SignedOut`] — no live session; periodic work would just -/// 401-loop against the backend. -/// -/// Other [`PauseReason`] variants: -/// - `OnBattery` / `CpuPressure` (future, per #1073) — intentionally **not** -/// gated here; periodic Composio fetch is network-light, so battery / CPU -/// pressure shouldn't stop the user's data flowing in. Those signals -/// already throttle LLM-bound work through the regular gate. -/// - `Unknown` — documented in `scheduler_gate::policy` as a safe fallback; -/// `Policy::pause_reason()` returns it only when the gate state is in a -/// transitional / not-yet-resolved condition. Letting the tick proceed -/// here keeps periodic sync running through brief transitions instead of -/// pausing on stale unresolved state. -pub(crate) fn periodic_pause_reason() -> Option { - // Delegate the `Policy::Paused { .. }` → `PauseReason` extraction to - // the existing `Policy::pause_reason()` helper (avoids re-implementing - // the same destructure twice). The allow-list below is the only thing - // this site has to own — future `PauseReason` variants stay opt-in. - let reason = current_policy().pause_reason()?; - matches!(reason, PauseReason::UserDisabled | PauseReason::SignedOut).then_some(reason) -} - -/// Process-level "was the last tick paused?" tracker for transition logging. -/// -/// We want `info!` *once* when the periodic loop crosses the pause boundary -/// (so fleet operators investigating "why is Composio not syncing?" see a -/// breadcrumb at default log level), without spamming `info` every 20 min -/// while the user has the toggle off. `Relaxed` ordering is fine because -/// the only consumer is the inside of `run_one_tick`, which is serialised -/// by the singleton scheduler loop. -static LAST_TICK_WAS_PAUSED: AtomicBool = AtomicBool::new(false); - -/// Run a single scheduler tick. Public-ish (`pub(crate)`) so the test -/// module can drive ticks without spinning up the real `interval`. -pub(crate) async fn run_one_tick() -> Result<(), String> { - // Step 0: scheduler-gate check. When the user has paused Memory Tree - // via the Settings toggle, every subsequent tick should be a cheap - // no-op — no `list_connections` call, no provider walk, no API budget - // burn. The check runs **before** config load + auth-client build so - // a paused session never even resolves the API token. - // - // Transition logging: emit `info!` once when the loop crosses the - // pause boundary in either direction; stay at `debug!` for the - // already-paused / already-running steady state. Without this, fleet - // operators investigating "why is Composio not syncing?" see nothing - // at default log level. - if let Some(reason) = periodic_pause_reason() { - let was_paused = LAST_TICK_WAS_PAUSED.swap(true, Ordering::Relaxed); - if was_paused { - tracing::debug!( - reason = reason.as_str(), - "[composio:periodic] scheduler-gate paused — skipping tick" - ); - } else { - tracing::info!( - reason = reason.as_str(), - "[composio:periodic] scheduler-gate paused — pausing periodic Composio sync" - ); - } - return Ok(()); - } else { - let was_paused = LAST_TICK_WAS_PAUSED.swap(false, Ordering::Relaxed); - if was_paused { - tracing::info!( - "[composio:periodic] scheduler-gate resumed — periodic Composio sync re-enabled" - ); - } - } - - // Step 1: load config (also gives us the auth token via the - // shared integrations client builder). - let config = config_rpc::load_config_with_timeout() - .await - .map_err(|e| format!("load_config: {e}"))?; - - // Step 2: list active connections — mode-aware. Backend mode walks - // the tinyhumans tenant; direct mode walks the user's personal - // Composio v3 tenant. Mirrors `ops::composio_list_connections` so - // direct-mode users get periodic sync against their own connections - // instead of seeing an empty list (#1710). - // Mode dispatch lives in the host's `ComposioHost` impl, and so does the - // 401 classification that used to happen here: the direct-mode v3 - // `/connected_accounts` 401 shape is a property of the client, not of this - // loop, and the host reports it through the same observability classifier - // the UI poll uses. A failure here means "not signed in / no direct key" as - // often as it means a real fault, so the tick skips rather than erroring. - let connections = match composio_host::list_connections(&*config).await { - Ok(connections) => connections, - Err(e) => { - tracing::debug!( - error = %e, - "[composio:periodic] no connections (not signed in? no direct key?), skipping tick" - ); - return Ok(()); - } - }; - - let sync_map = last_sync_map(); - - // Global, user-configurable memory-sync cadence (#3302). Applied to every - // opted-in source as a floor/override over the provider's own default; a - // value of `Some(0)` disables periodic auto-sync ("Manual only"). - let global_interval = config.memory_sync_interval_secs(); - - // Persisted last-sync fallback (#3302). The in-memory `LAST_SYNC_AT` map is - // rebuilt empty on every launch, so without this a cold start would re-fire - // every connection on the first tick — silently breaking the configured - // "Sync every 24h" gap across app restarts. We index the persisted sync - // audit log (wall-clock timestamps that survive restarts) and use it as the - // due-check fallback whenever the in-memory monotonic record is absent. - let (audit_index, audit_available) = - composio_audit_state(read_audit_log(config.workspace_dir())); - if !audit_available { - tracing::warn!( - "[memory_sync:periodic] audit unavailable; sources without in-memory cadence will be skipped" - ); - } - let now = Utc::now(); - - // Per-source registry snapshot (#2831). The periodic loop gates on the - // per-source `enabled` toggle so a source the user switched off stops - // syncing in the background — matching the manual paths - // (`memory_sources::sync_source`, `memory_sources_sync_all`), which already - // early-return on `!enabled`. Index every Composio source (enabled and - // disabled) by connection id; the per-connection branch below resolves - // skip/caps via `decide_periodic_source`. - // - // Built from the **already-loaded** `config` snapshot (Step 1), not a second - // `list_sources()` read. A separate read whose error we swallowed to an - // empty map would make every disabled source fall through to the - // `decide_periodic_source(None, ..)` default-caps path — silently - // re-enabling background sync for sources the user switched off on a - // transient config-read failure. Reusing the tick's snapshot is fail-closed - // (a disabled row stays disabled) and avoids the extra read entirely. - let composio_sources: HashMap = - crate::sources::decode_memory_sources(&*config) - .iter() - .filter(|s| s.kind == SourceKind::Composio) - .filter_map(|s| s.connection_id.clone().map(|id| (id, s.clone()))) - .collect(); - - let mut considered = 0usize; - let mut fired = 0usize; - for conn in connections { - considered += 1; - - // Skip connections that aren't actually live yet. - if !conn.is_active() { - continue; - } - - let toolkit = conn.normalized_toolkit(); - let Some(provider) = get_provider(&toolkit) else { - // No provider registered for this toolkit — that's fine, - // we just don't have native code for it. Tools still work - // through `composio_execute`. - continue; - }; - - let Some(provider_default) = provider.sync_interval_secs() else { - // Provider opted out of periodic sync entirely. - continue; - }; - - let Some(interval_secs) = effective_interval_secs(provider_default, global_interval) else { - // User selected "Manual only" — skip auto-sync for this source. - // Manual `memory_sources_sync` still works. - tracing::debug!( - toolkit = %toolkit, - connection_id = %conn.id, - "[composio:periodic] manual-only mode — skipping periodic sync" - ); - continue; - }; - - let key = (toolkit.clone(), conn.id.clone()); - // Prefer the in-memory monotonic record (most accurate within this run); - // fall back to the persisted audit timestamp so the configured cadence - // is honoured across restarts instead of re-firing on every cold start. - let in_memory_since = { - let map = sync_map.lock().unwrap_or_else(|e| e.into_inner()); - map.get(&key).map(|when| when.elapsed()) - }; - let Some(since_last_sync) = cadence_from_audit( - in_memory_since, - audit_available, - persisted_since_last_sync(&audit_index, &conn.id, now), - ) else { - tracing::debug!( - toolkit = %toolkit, - "[composio:periodic] source has unknown cadence while audit is unavailable; skipping" - ); - continue; - }; - if !connection_is_due(interval_secs, since_last_sync) { - continue; - } - - // Per-source gate + caps from the memory_sources registry (#2831). - // A disabled source is skipped here (the background-sync half of the - // toggle); enabled sources sync with their caps; a connection with no - // registry row yet syncs with conservative per-toolkit defaults. - let (src_max_items, src_sync_depth_days) = - match decide_periodic_source(composio_sources.get(&conn.id), &toolkit) { - PeriodicSourceDecision::Skip => { - tracing::debug!( - toolkit = %toolkit, - connection_id = %conn.id, - "[composio:periodic] source disabled — skipping periodic sync" - ); - continue; - } - PeriodicSourceDecision::Sync { - max_items, - sync_depth_days, - } => (max_items, sync_depth_days), - }; - - tracing::debug!( - toolkit = %toolkit, - connection_id = %conn.id, - max_items = ?src_max_items, - sync_depth_days = ?src_sync_depth_days, - "[composio:periodic] caps from registry" - ); - - let mut source = composio_sources - .get(&conn.id) - .cloned() - .unwrap_or_else(|| periodic_source(&toolkit, &conn.id)); - source.max_items = src_max_items; - source.sync_depth_days = src_sync_depth_days; - - tracing::debug!( - toolkit = %conn.toolkit, - connection_id = %conn.id, - interval_secs, - "[composio:periodic] firing sync" - ); - let sync_started = Instant::now(); - let result = crate::sync::pipelines::host::run_composio_connection_with_caps( - &toolkit, - &conn.id, - &*config, - crate::sync::pipelines::host::SourceCaps::from_source(&source), - ) - .await; - let duration_ms = sync_started.elapsed().as_millis() as u64; - - match result { - Ok(outcome) => { - let usage = ComposioUsage { - actions_called: outcome.actions_called, - cost_usd: outcome.provider_cost_usd, - }; - if outcome.tree_ingest_failures > 0 { - tracing::warn!( - toolkit = %conn.toolkit, - connection_id = %conn.id, - items = outcome.records_ingested, - tree_failures = outcome.tree_ingest_failures, - "[composio:periodic] fetch ok but the memory-tree half dropped \ - items; auditing the run as failed" - ); - } else { - tracing::debug!( - toolkit = %conn.toolkit, - connection_id = %conn.id, - items = outcome.records_ingested, - composio_actions = usage.actions_called, - "[composio:periodic] sync ok" - ); - } - let entry = build_periodic_audit_entry( - &toolkit, - &conn.id, - &usage, - outcome.records_ingested as usize, - duration_ms, - None, - outcome.tree_ingest_failures, - ); - if let Err(error) = append_audit_entry(config.workspace_dir(), &entry) { - tracing::warn!(%error, "[memory_sync:audit] append failed"); - } - record_sync_success(&conn.toolkit, &conn.id); - fired += 1; - } - Err(e) => { - let usage = ComposioUsage { - actions_called: e.actions_called, - cost_usd: e.provider_cost_usd, - }; - tracing::warn!( - toolkit = %conn.toolkit, - connection_id = %conn.id, - error = %e, - "[composio:periodic] sync failed (will retry next tick)" - ); - // A failed tick may still have fired billable fetch actions - // before erroring — audit the partial cost so it isn't lost. - let entry = build_periodic_audit_entry( - &toolkit, - &conn.id, - &usage, - 0, - duration_ms, - Some(e.to_string()), - 0, - ); - if let Err(error) = append_audit_entry(config.workspace_dir(), &entry) { - tracing::warn!(%error, "[memory_sync:audit] append failed"); - } - // Intentionally do NOT update last_sync_at on failure - // so the next tick retries immediately. - } - } - } - - tracing::debug!(considered, fired, "[composio:periodic] tick complete"); - Ok(()) -} - -fn composio_audit_state( - read: anyhow::Result>, -) -> (HashMap>, bool) { - match read { - Ok(entries) => (index_last_success_by_connection(&entries), true), - Err(error) => { - tracing::warn!(%error, "[memory_sync:periodic] audit read failed"); - (HashMap::new(), false) - } - } -} - -fn cadence_from_audit( - in_memory_since: Option, - audit_available: bool, - persisted_since: Option, -) -> Option> { - match in_memory_since { - Some(since) => Some(Some(since)), - None if audit_available => Some(persisted_since), - None => None, - } -} - -fn periodic_source(toolkit: &str, connection_id: &str) -> MemorySourceEntry { - MemorySourceEntry { - id: format!("composio:{connection_id}"), - kind: SourceKind::Composio, - label: toolkit.to_string(), - enabled: true, - toolkit: Some(toolkit.to_string()), - connection_id: Some(connection_id.to_string()), - path: None, - glob: None, - url: None, - branch: None, - paths: Vec::new(), - max_commits: None, - max_issues: None, - max_prs: None, - query: None, - since_days: None, - max_items: None, - selector: None, - max_tokens_per_sync: None, - max_cost_per_sync_usd: None, - sync_depth_days: None, - } -} - -/// Build a [`SyncAuditEntry`] for one periodic Composio sync tick (#3111 -/// follow-up). -/// -/// Periodic syncs only fetch + ingest; summarisation runs later in the async -/// job worker, so the LLM-cost columns (tokens, estimated / actual charge) -/// are zero here. The meaningful spend is the Composio billable actions the -/// fetch fired, carried in `usage`. `scope` is `{toolkit}:{connection_id}` to -/// match the owner shape the per-source memory-tree ingest uses, and -/// `source_kind` is `"composio"` so the Sync History panel groups periodic -/// rows alongside the manual-sync rows the dispatcher already writes. -fn build_periodic_audit_entry( - toolkit: &str, - connection_id: &str, - usage: &ComposioUsage, - items_ingested: usize, - duration_ms: u64, - error: Option, - tree_ingest_failures: u32, -) -> SyncAuditEntry { - // A run whose fetch committed but whose tree half dropped items must not - // read as success in Sync History (openhuman#5820). - let success = error.is_none() && tree_ingest_failures == 0; - let tree_error = (tree_ingest_failures > 0).then(|| { - format!("{tree_ingest_failures} item(s) fetched but not ingested into the memory tree") - }); - SyncAuditEntry { - timestamp: chrono::Utc::now(), - source_id: connection_id.to_string(), - source_kind: "composio".to_string(), - scope: format!("{toolkit}:{connection_id}"), - items_fetched: items_ingested as u32, - batches: 0, - input_tokens: 0, - output_tokens: 0, - estimated_cost_usd: 0.0, - composio_actions_called: usage.actions_called, - composio_cost_usd: usage.cost_usd, - actual_charged_usd: None, - duration_ms, - success, - error, - tree_ingest_failures, - tree_error, - } -} - -#[cfg(test)] -#[path = "periodic_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/periodic_tests.rs b/crates/tinymemory-core/src/sync/composio/periodic_tests.rs deleted file mode 100644 index d9116762..00000000 --- a/crates/tinymemory-core/src/sync/composio/periodic_tests.rs +++ /dev/null @@ -1,587 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; -use crate::test_env_lock::TEST_ENV_LOCK as ENV_LOCK; -use tempfile::tempdir; - -#[test] -fn tick_seconds_is_sane_default() { - // Sanity check: don't accidentally ship a 1-second tick. - const _: () = assert!(TICK_SECONDS >= 30); - const _: () = assert!(TICK_SECONDS <= 3600); -} - -#[test] -fn effective_interval_none_falls_back_to_default() { - // No user choice → 24h default, floored at the provider default. - assert_eq!( - effective_interval_secs(15 * 60, None), - Some(DEFAULT_MEMORY_SYNC_INTERVAL_SECS) - ); -} - -#[test] -fn effective_interval_manual_disables_sync() { - // Some(0) is the "Manual only" sentinel — periodic sync is skipped. - assert_eq!(effective_interval_secs(15 * 60, Some(0)), None); -} - -#[test] -fn effective_interval_override_is_floored_at_provider_default() { - // A user cadence longer than the provider default is honoured as-is. - assert_eq!( - effective_interval_secs(15 * 60, Some(4 * 3600)), - Some(4 * 3600) - ); - // A user cadence shorter than the provider default is clamped up to it - // so we never sync more often than the provider intends. - assert_eq!(effective_interval_secs(30 * 60, Some(60)), Some(30 * 60)); - // Exactly equal stays equal. - assert_eq!(effective_interval_secs(1800, Some(1800)), Some(1800)); -} - -#[test] -fn effective_interval_default_is_floored_at_a_longer_provider_default() { - // If a provider ever defaults to longer than 24h, that wins under None. - let long = DEFAULT_MEMORY_SYNC_INTERVAL_SECS + 3600; - assert_eq!(effective_interval_secs(long, None), Some(long)); -} - -#[test] -fn connection_is_due_compares_elapsed_against_interval() { - let interval = 4 * 3600; - // Never synced this run → always due. - assert!(connection_is_due(interval, None)); - // Synced more recently than the interval → not due. - assert!(!connection_is_due( - interval, - Some(Duration::from_secs(3600)) - )); - // Synced exactly at the interval boundary → due. - assert!(connection_is_due( - interval, - Some(Duration::from_secs(interval)) - )); - // Synced longer ago than the interval → due. - assert!(connection_is_due( - interval, - Some(Duration::from_secs(interval + 1)) - )); -} - -/// Build a minimal Composio `MemorySourceEntry` for the per-source gate -/// tests — only the fields `decide_periodic_source` reads are meaningful. -fn composio_source( - enabled: bool, - max_items: Option, - sync_depth_days: Option, -) -> MemorySourceEntry { - MemorySourceEntry { - id: "src_test".to_string(), - kind: SourceKind::Composio, - label: "test".to_string(), - enabled, - toolkit: Some("gmail".to_string()), - connection_id: Some("cmp-1".to_string()), - path: None, - glob: None, - url: None, - branch: None, - paths: Vec::new(), - max_commits: None, - max_issues: None, - max_prs: None, - query: None, - since_days: None, - max_items, - selector: None, - max_tokens_per_sync: None, - max_cost_per_sync_usd: None, - sync_depth_days, - } -} - -/// #2831 row 2: a source explicitly toggled **off** must be skipped by the -/// background loop — this is the leak the gate closes. -#[test] -fn decide_periodic_source_skips_disabled_source() { - let src = composio_source(false, Some(100), Some(30)); - assert_eq!( - decide_periodic_source(Some(&src), "gmail"), - PeriodicSourceDecision::Skip - ); -} - -/// An enabled source syncs with exactly its configured caps (no defaulting). -#[test] -fn decide_periodic_source_uses_enabled_source_caps() { - let src = composio_source(true, Some(42), Some(7)); - assert_eq!( - decide_periodic_source(Some(&src), "gmail"), - PeriodicSourceDecision::Sync { - max_items: Some(42), - sync_depth_days: Some(7), - } - ); -} - -/// A connection with no registry row yet (pre-reconcile window) syncs with -/// the conservative per-toolkit defaults — **bounded**, never uncapped, and -/// never skipped. This is the safe-direction fallback for a missing match. -#[test] -fn decide_periodic_source_defaults_caps_when_no_row() { - let (want_items, want_depth) = memory_sync_defaults_for_toolkit("gmail"); - assert_eq!( - decide_periodic_source(None, "gmail"), - PeriodicSourceDecision::Sync { - max_items: want_items, - sync_depth_days: want_depth, - } - ); - // The defaults are bounded for a known toolkit (regression guard against - // an accidental return to uncapped background fetches). - assert!(want_items.is_some()); -} - -/// Multi-account regression (#3443 added multiple account connections per -/// toolkit): two live connections of the *same* toolkit must be gated -/// **independently** by their own per-`connection_id` source rows. This -/// pins the loop's connection-id keying — re-keying the lookup by toolkit -/// would collapse the two accounts and is the regression this guards. -#[test] -fn per_connection_gate_is_independent_across_accounts_of_same_toolkit() { - // gmail account A: enabled with caps; gmail account B: disabled. - let mut a = composio_source(true, Some(10), Some(5)); - a.connection_id = Some("conn-A".to_string()); - a.toolkit = Some("gmail".to_string()); - let mut b = composio_source(false, Some(99), Some(99)); - b.connection_id = Some("conn-B".to_string()); - b.toolkit = Some("gmail".to_string()); - - // Build the same connection_id → entry index the live tick builds. - let index: HashMap = [a, b] - .into_iter() - .filter_map(|s| s.connection_id.clone().map(|id| (id, s))) - .collect(); - - // Account A (enabled) syncs with its own caps... - assert_eq!( - decide_periodic_source(index.get("conn-A"), "gmail"), - PeriodicSourceDecision::Sync { - max_items: Some(10), - sync_depth_days: Some(5), - } - ); - // ...account B (disabled) is skipped, even though it shares the toolkit. - assert_eq!( - decide_periodic_source(index.get("conn-B"), "gmail"), - PeriodicSourceDecision::Skip - ); - // A third, not-yet-registered account of the same toolkit falls back to - // bounded defaults (never skipped, never uncapped). - let (def_items, def_depth) = memory_sync_defaults_for_toolkit("gmail"); - assert_eq!( - decide_periodic_source(index.get("conn-C"), "gmail"), - PeriodicSourceDecision::Sync { - max_items: def_items, - sync_depth_days: def_depth, - } - ); -} - -/// End-to-end simulation of the scheduler's per-connection decision: prove -/// that **changing the global setting changes when the next sync fires** -/// (issue #3302 acceptance criterion). We drive the same two pure helpers -/// the live tick uses (`effective_interval_secs` → `connection_is_due`) -/// across realistic last-sync ages, so no clock or network is needed. -#[test] -fn scheduler_decision_honors_the_global_setting() { - // A chatty provider that natively wants to sync every 15 minutes. - let provider_default = 15 * 60; - - // Helper mirroring the live loop: returns whether the connection would - // fire right now, or `None` for "Manual only" (skipped entirely). - let decide = |global: Option, since: Option| -> Option { - effective_interval_secs(provider_default, global) - .map(|interval| connection_is_due(interval, since)) - }; - - let one_hour_ago = Some(Duration::from_secs(3600)); - let five_hours_ago = Some(Duration::from_secs(5 * 3600)); - - // Baseline (no global override): with only the 15m provider default, a - // connection synced an hour ago is already overdue and WOULD fire. - // (This is the behavior the feature is reining in.) - assert!(connection_is_due(provider_default, one_hour_ago)); - - // User picks "every 4h": now that same hour-old connection must NOT - // fire — the global cadence (not the 15m default) governs the gap… - assert_eq!(decide(Some(4 * 3600), one_hour_ago), Some(false)); - // …but once 5h have passed it fires again. - assert_eq!(decide(Some(4 * 3600), five_hours_ago), Some(true)); - - // User picks "Manual only" (0): never auto-fires, no matter how stale. - assert_eq!(decide(Some(0), five_hours_ago), None); - assert_eq!(decide(Some(0), None), None); - - // Unset (None) → 24h default: the hour-old connection is not yet due, - // confirming the default is far more conservative than the 15m native - // cadence. - assert_eq!(decide(None, one_hour_ago), Some(false)); - assert_eq!( - decide(None, Some(Duration::from_secs(25 * 3600))), - Some(true) - ); - - // A never-synced connection fires on any non-manual setting (the - // restart-recovery path). - assert_eq!(decide(Some(4 * 3600), None), Some(true)); -} - -fn audit_entry( - connection_id: &str, - scope: &str, - success: bool, - ts: DateTime, -) -> SyncAuditEntry { - SyncAuditEntry { - timestamp: ts, - source_id: connection_id.to_string(), - source_kind: "composio".to_string(), - scope: scope.to_string(), - items_fetched: 1, - batches: 0, - input_tokens: 0, - output_tokens: 0, - estimated_cost_usd: 0.0, - composio_actions_called: 1, - composio_cost_usd: 0.0, - actual_charged_usd: None, - duration_ms: 10, - success, - error: None, - tree_ingest_failures: 0, - tree_error: None, - } -} - -#[test] -fn index_last_success_keeps_latest_success_and_ignores_failures() { - let now = Utc::now(); - let older = now - chrono::Duration::hours(6); - let newer = now - chrono::Duration::hours(1); - let entries = vec![ - audit_entry("cmp-1", "gmail:cmp-1", true, older), - audit_entry("cmp-1", "gmail:cmp-1", true, newer), // newer success wins - audit_entry("cmp-1", "gmail:cmp-1", false, now), // failure ignored - audit_entry("cmp-2", "slack:cmp-2", false, now), // only-failure → absent - ]; - let idx = index_last_success_by_connection(&entries); - assert_eq!(idx.get("cmp-1"), Some(&newer)); - assert!( - !idx.contains_key("cmp-2"), - "a connection with only failed syncs is not indexed" - ); -} - -#[test] -fn index_last_success_falls_back_to_source_id_without_scope_suffix() { - let now = Utc::now(); - // A non-composio kind is skipped entirely. - let entries = vec![ - SyncAuditEntry { - source_kind: "github_repo".to_string(), - ..audit_entry("ignored", "github:org/repo", true, now) - }, - // Composio entry whose scope has no ':' → key by source_id. - audit_entry("cmp-3", "noscope", true, now), - ]; - let idx = index_last_success_by_connection(&entries); - assert!(idx.contains_key("cmp-3")); - assert!(!idx.contains_key("ignored")); -} - -#[test] -fn persisted_since_last_sync_computes_and_saturates() { - let now = Utc::now(); - let mut idx = HashMap::new(); - idx.insert("cmp-1".to_string(), now - chrono::Duration::hours(3)); - idx.insert("future".to_string(), now + chrono::Duration::hours(2)); - - let elapsed = persisted_since_last_sync(&idx, "cmp-1", now).unwrap(); - // ~3h, allow a small window for test execution time. - assert!(elapsed >= Duration::from_secs(3 * 3600 - 5)); - assert!(elapsed <= Duration::from_secs(3 * 3600 + 5)); - // Clock skew (future timestamp) saturates to zero, not a huge value. - assert_eq!( - persisted_since_last_sync(&idx, "future", now), - Some(Duration::ZERO) - ); - // Unknown connection → None (treated as never synced). - assert_eq!(persisted_since_last_sync(&idx, "unknown", now), None); -} - -/// The cadence must survive a restart: with the in-memory map cold, the -/// persisted audit timestamp drives the due-check so a connection synced -/// 1h ago does NOT re-fire under a 4h setting, but one synced 5h ago does. -#[test] -fn cadence_survives_restart_via_persisted_audit() { - let now = Utc::now(); - let mut idx = HashMap::new(); - idx.insert("cmp-1".to_string(), now - chrono::Duration::hours(1)); - idx.insert("cmp-2".to_string(), now - chrono::Duration::hours(5)); - - let interval = effective_interval_secs(15 * 60, Some(4 * 3600)).unwrap(); - - // cmp-1 (synced 1h ago) — in-memory cold, persisted fallback says NOT due. - let cmp1 = None.or_else(|| persisted_since_last_sync(&idx, "cmp-1", now)); - assert!(!connection_is_due(interval, cmp1)); - - // cmp-2 (synced 5h ago) — persisted fallback says due. - let cmp2 = None.or_else(|| persisted_since_last_sync(&idx, "cmp-2", now)); - assert!(connection_is_due(interval, cmp2)); - - // A connection with no persisted record still fires (truly fresh). - let fresh = None.or_else(|| persisted_since_last_sync(&idx, "cmp-new", now)); - assert!(connection_is_due(interval, fresh)); -} - -#[test] -fn audit_failure_is_unavailable_and_unknown_cadence_is_skipped() { - let (index, available) = - composio_audit_state(Err(anyhow::anyhow!("simulated audit I/O failure"))); - assert!(index.is_empty()); - assert!(!available); - assert_eq!(cadence_from_audit(None, available, None), None); - - let known = Duration::from_secs(60); - assert_eq!( - cadence_from_audit(Some(known), available, None), - Some(Some(known)) - ); -} - -#[test] -fn readable_empty_audit_preserves_first_sync_behavior() { - let (index, available) = composio_audit_state(Ok(Vec::new())); - assert!(index.is_empty()); - assert!(available); - - let cadence = cadence_from_audit(None, available, None) - .expect("readable empty audit keeps the source eligible"); - assert!(connection_is_due(3600, cadence)); -} - -/// A successful periodic tick produces a Composio-kind audit entry that -/// carries the billable-action tally + cost and zeroes the LLM-cost -/// columns (summarisation happens later in the job worker). Pins the -/// shape the Sync History panel reads (#3111 follow-up). -#[test] -fn periodic_audit_entry_records_composio_cost_on_success() { - let usage = ComposioUsage { - actions_called: 3, - cost_usd: 0.042, - }; - let entry = build_periodic_audit_entry("gmail", "cmp-123", &usage, 17, 1234, None, 0); - - assert_eq!(entry.source_kind, "composio"); - assert_eq!(entry.source_id, "cmp-123"); - assert_eq!(entry.scope, "gmail:cmp-123"); - assert_eq!(entry.items_fetched, 17); - assert_eq!(entry.composio_actions_called, 3); - assert!((entry.composio_cost_usd - 0.042).abs() < f64::EPSILON); - assert!(entry.success); - assert!(entry.error.is_none()); - // Periodic fetch does no summarisation — LLM cost columns stay zero, - // and the Composio spend is the whole combined cost. - assert_eq!(entry.input_tokens, 0); - assert_eq!(entry.estimated_cost_usd, 0.0); - assert!((entry.combined_cost_usd() - 0.042).abs() < f64::EPSILON); -} - -/// A failed periodic tick still records the partial billable cost it -/// incurred before erroring (the fetch may have fired actions), with -/// `success = false` and the error message preserved. -#[test] -fn periodic_audit_entry_preserves_partial_cost_on_failure() { - let usage = ComposioUsage { - actions_called: 1, - cost_usd: 0.01, - }; - let entry = build_periodic_audit_entry( - "notion", - "cmp-9", - &usage, - 0, - 500, - Some("fetch timed out".to_string()), - 0, - ); - - assert!(!entry.success); - assert_eq!(entry.error.as_deref(), Some("fetch timed out")); - assert_eq!(entry.items_fetched, 0); - // The billable action it managed to fire before failing is still - // recorded so cost isn't under-reported on failures. - assert_eq!(entry.composio_actions_called, 1); - assert!((entry.composio_cost_usd - 0.01).abs() < f64::EPSILON); -} - -#[test] -fn record_sync_success_stores_timestamp_keyed_by_toolkit_and_connection() { - // Use unique keys so this test doesn't collide with other tests - // writing into the process-wide map. - let toolkit = "test_periodic_toolkit_a"; - let conn = "test-conn-a"; - record_sync_success(toolkit, conn); - let map = last_sync_map(); - let guard = map.lock().expect("lock"); - let ts = guard - .get(&(toolkit.to_string(), conn.to_string())) - .expect("entry recorded"); - // Just-recorded timestamps should be very recent. - assert!(ts.elapsed() < Duration::from_secs(5)); -} - -#[test] -fn record_sync_success_overwrites_previous_timestamp() { - let toolkit = "test_periodic_toolkit_b"; - let conn = "test-conn-b"; - record_sync_success(toolkit, conn); - let first = last_sync_map() - .lock() - .expect("lock") - .get(&(toolkit.to_string(), conn.to_string())) - .copied() - .expect("first entry"); - // Second call must replace (not keep the older) timestamp. - std::thread::sleep(Duration::from_millis(5)); - record_sync_success(toolkit, conn); - let second = last_sync_map() - .lock() - .expect("lock") - .get(&(toolkit.to_string(), conn.to_string())) - .copied() - .expect("second entry"); - assert!( - second >= first, - "record_sync_success should advance the stored Instant" - ); -} - -// The `_guard` below is held deliberately across the `run_one_tick().await` -// in this test: it's a std::sync::Mutex used purely as a test-isolation -// gate around the process-global `OPENHUMAN_WORKSPACE` env var, not an -// async resource lock guarding shared runtime state. Dropping it before -// the await would let a sibling test mutate the env var mid-tick, -// defeating the isolation this guard exists to provide. -#[allow(clippy::await_holding_lock)] -#[tokio::test] -async fn run_one_tick_returns_ok_when_no_client() { - // Isolate the workspace/env so config loading doesn't contend with - // sibling tests mutating OPENHUMAN_WORKSPACE in parallel. - let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let tmp = tempdir().expect("tempdir"); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp.path()); - } - - // With no session stored in the isolated workspace, - // `build_composio_client` returns None and the tick should - // silently skip (returning Ok). This covers the early-return - // path that's otherwise only hit in production. - let inner = tokio::time::timeout(Duration::from_secs(5), run_one_tick()) - .await - .expect("run_one_tick should not hang indefinitely during tests"); - assert!( - inner.is_ok(), - "run_one_tick should return Ok when no client is available: {inner:?}" - ); - - unsafe { - std::env::remove_var("OPENHUMAN_WORKSPACE"); - } -} - -#[tokio::test] -async fn start_periodic_sync_is_idempotent() { - // First call installs the scheduler via the OnceLock; subsequent - // calls must be cheap no-ops without panicking. `tokio::spawn` - // needs an ambient runtime, so this test runs under `tokio::test`. - start_periodic_sync(); - start_periodic_sync(); - assert!(SCHEDULER_STARTED.get().is_some()); -} - -#[test] -fn record_sync_success_distinguishes_connections() { - let toolkit = "test_periodic_toolkit_c"; - record_sync_success(toolkit, "conn-1"); - record_sync_success(toolkit, "conn-2"); - let map = last_sync_map(); - let guard = map.lock().expect("lock"); - assert!(guard - .get(&(toolkit.to_string(), "conn-1".to_string())) - .is_some()); - assert!(guard - .get(&(toolkit.to_string(), "conn-2".to_string())) - .is_some()); - // Unrelated key should be absent. - assert!(guard - .get(&(toolkit.to_string(), "conn-3".to_string())) - .is_none()); -} - -/// In unit tests `scheduler_gate::STATE` is never initialised, so -/// `current_policy()` returns `Policy::Normal` and the helper must -/// return `None` — i.e. the tick is allowed to proceed. This pins the -/// happy-path wiring; an accidental "always pause" regression in the -/// helper would break every `run_one_tick`-driven test that follows it. -/// -/// (The redundant "does-not-short-circuit" tick-level test that was -/// here in the first review pass was dropped per @oxoxDev's -/// [#2825 review](https://github.com/tinyhumansai/openhuman/pull/2825): -/// it duplicated `run_one_tick_returns_ok_when_no_client` because -/// both exited at the same `create_composio_client` no-client branch, -/// so neither actually proved the new gate-check arm fired in the -/// right direction. Asserting log-line absence via `tracing-test` -/// would prove it but adds a new dev-dependency for one assertion — -/// the helper-level test below already pins the wiring.) -#[test] -fn periodic_pause_reason_returns_none_when_gate_not_initialised() { - // Calling without `scheduler_gate::init_global(...)` exercises the - // OnceLock-uninitialised branch in `current_policy`, which is the - // realistic test-environment state. - assert!( - periodic_pause_reason().is_none(), - "expected None (i.e. tick proceeds) when scheduler_gate is in default Normal state, \ - got {:?}", - periodic_pause_reason() - ); -} - -#[test] -fn synthesized_periodic_source_is_enabled_scoped_and_uncapped() { - let source = periodic_source("gmail", "connection-42"); - assert_eq!(source.id, "composio:connection-42"); - assert_eq!(source.kind, SourceKind::Composio); - assert_eq!(source.label, "gmail"); - assert!(source.enabled); - assert_eq!(source.toolkit.as_deref(), Some("gmail")); - assert_eq!(source.connection_id.as_deref(), Some("connection-42")); - assert!(source.path.is_none()); - assert!(source.glob.is_none()); - assert!(source.url.is_none()); - assert!(source.branch.is_none()); - assert!(source.paths.is_empty()); - assert!(source.max_commits.is_none()); - assert!(source.max_issues.is_none()); - assert!(source.max_prs.is_none()); - assert!(source.query.is_none()); - assert!(source.since_days.is_none()); - assert!(source.max_items.is_none()); - assert!(source.selector.is_none()); - assert!(source.max_tokens_per_sync.is_none()); - assert!(source.max_cost_per_sync_usd.is_none()); - assert!(source.sync_depth_days.is_none()); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/catalogs.rs b/crates/tinymemory-core/src/sync/composio/providers/catalogs.rs deleted file mode 100644 index a8461ad8..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/catalogs.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! The curated catalogs, re-exported at their historical path. -//! -//! The tables themselves moved to [`tinymemory_api::composio::catalogs`] -//! (OpenHuman#5560): they are `&'static str` slugs with no dependency, and the -//! *host* is their heaviest reader — it filters the agent's visible tool list -//! and renders the unlock hints. While they lived here, every one of those -//! reads was a compile-time link to this crate. -//! -//! Nothing about the data changed. This module keeps -//! `providers::catalogs::SLACK_CURATED` and its siblings resolving for the -//! provider impls beside it. - -pub use tinymemory_api::composio::catalogs::business::{ - AIRTABLE_CURATED, FIGMA_CURATED, HUBSPOT_CURATED, SALESFORCE_CURATED, SHOPIFY_CURATED, - STRIPE_CURATED, -}; -pub use tinymemory_api::composio::catalogs::google::{ - GOOGLECALENDAR_CURATED, GOOGLEDOCS_CURATED, GOOGLEDRIVE_CURATED, GOOGLESHEETS_CURATED, -}; -pub use tinymemory_api::composio::catalogs::messaging::{ - DISCORD_CURATED, MICROSOFT_TEAMS_CURATED, SLACK_CURATED, TELEGRAM_CURATED, WHATSAPP_CURATED, -}; -pub use tinymemory_api::composio::catalogs::microsoft::{EXCEL_CURATED, ONE_DRIVE_CURATED}; -pub use tinymemory_api::composio::catalogs::productivity::{ - ASANA_CURATED, DROPBOX_CURATED, JIRA_CURATED, OUTLOOK_CURATED, TODOIST_CURATED, TRELLO_CURATED, -}; -pub use tinymemory_api::composio::catalogs::social_media::{ - SPOTIFY_CURATED, TWITTER_CURATED, YOUTUBE_CURATED, -}; diff --git a/crates/tinymemory-core/src/sync/composio/providers/catalogs_compat.rs b/crates/tinymemory-core/src/sync/composio/providers/catalogs_compat.rs deleted file mode 100644 index ca3db4ec..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/catalogs_compat.rs +++ /dev/null @@ -1,59 +0,0 @@ -//! Historical per-category catalog module paths. -//! -//! Before OpenHuman#5560 moved the curated catalogs into the contract crate, -//! each category lived in its own `pub mod catalogs_` here (e.g. -//! `providers::catalogs_business::SHOPIFY_CURATED`). The move consolidated -//! them into [`super::catalogs`], which flattens every constant to one level -//! (`providers::catalogs::SHOPIFY_CURATED`) rather than nesting them by -//! category — so the six original module names stopped resolving even though -//! [`super::catalogs`] kept every constant reachable under a different path. -//! -//! `AGENTS.md`'s SemVer policy treats a removed public path as a breaking -//! change unless the crate takes a major (pre-1.0: minor) bump for it. Rather -//! than force that bump for a rename, these six modules re-export the same -//! constants under their historical names — pure re-exports, no behavior, no -//! new dependency. -//! -//! # Deletion -//! -//! This module is a deprecation shim, not a permanent home. It may be deleted -//! in the next minor version bump that is *already* taking other breaking -//! changes (so the cost is paid once), or once nothing in this workspace or a -//! known downstream consumer (the OpenHuman host) still names a -//! `catalogs_` path — check with -//! `grep -rn 'catalogs_business\|catalogs_google\|catalogs_messaging\|catalogs_microsoft\|catalogs_productivity\|catalogs_social_media'` -//! across both repositories before removing it. - -pub mod catalogs_business { - //! Historical compat shim — see the module docs above. - pub use tinymemory_api::composio::catalogs::business::*; -} - -pub mod catalogs_google { - //! Historical compat shim — see the module docs above. - pub use tinymemory_api::composio::catalogs::google::*; -} - -pub mod catalogs_messaging { - //! Historical compat shim — see the module docs above. - pub use tinymemory_api::composio::catalogs::messaging::*; -} - -pub mod catalogs_microsoft { - //! Historical compat shim — see the module docs above. - pub use tinymemory_api::composio::catalogs::microsoft::*; -} - -pub mod catalogs_productivity { - //! Historical compat shim — see the module docs above. - pub use tinymemory_api::composio::catalogs::productivity::*; -} - -pub mod catalogs_social_media { - //! Historical compat shim — see the module docs above. - pub use tinymemory_api::composio::catalogs::social_media::*; -} - -#[cfg(test)] -#[path = "catalogs_compat_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/catalogs_compat_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/catalogs_compat_tests.rs deleted file mode 100644 index 9091002d..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/catalogs_compat_tests.rs +++ /dev/null @@ -1,31 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; - -#[test] -fn historical_module_paths_resolve_to_the_same_constants_as_the_new_path() { - assert_eq!( - catalogs_business::SHOPIFY_CURATED.len(), - crate::sync::composio::providers::catalogs::SHOPIFY_CURATED.len() - ); - assert_eq!( - catalogs_google::GOOGLEDRIVE_CURATED.len(), - crate::sync::composio::providers::catalogs::GOOGLEDRIVE_CURATED.len() - ); - assert_eq!( - catalogs_messaging::SLACK_CURATED.len(), - crate::sync::composio::providers::catalogs::SLACK_CURATED.len() - ); - assert_eq!( - catalogs_microsoft::EXCEL_CURATED.len(), - crate::sync::composio::providers::catalogs::EXCEL_CURATED.len() - ); - assert_eq!( - catalogs_productivity::JIRA_CURATED.len(), - crate::sync::composio::providers::catalogs::JIRA_CURATED.len() - ); - assert_eq!( - catalogs_social_media::TWITTER_CURATED.len(), - crate::sync::composio::providers::catalogs::TWITTER_CURATED.len() - ); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/clickup/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/clickup/mod.rs deleted file mode 100644 index 7ca8115c..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/clickup/mod.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! ClickUp Composio provider — incremental Memory Tree ingest for -//! tasks owned by (or assigned to) the connected user. -//! -//! Mirrors the [`crate::sync::composio::providers::notion`] layout -//! so anyone familiar with Notion/Slack ingestion can read this without -//! re-learning a new shape: -//! -//! - `provider.rs` — `impl ComposioProvider for ClickUpProvider` -//! - `normalization` — payload-shape helpers, owned by `tinymemory-sync` -//! (issue #18 §B3) -//! - `ingest.rs` — memory_tree document ingest (issue #2885) -//! - `tools.rs` — `CLICKUP_CURATED` whitelist of Composio actions -//! - `tests.rs` — unit tests for the helpers + trait metadata -//! -//! Issue: #2288 (introduction); #2885 (memory_tree migration). - -// The payload normalisers moved to tinycortex (they are pure Value -// transforms, i.e. driver-side). Aliased under the old module name so -// every `normalization::extract_*` call site below stays unchanged. -use tinymemory_sync::clickup as normalization; -mod provider; -#[cfg(test)] -mod tests; -pub mod tools; - -pub use provider::ClickUpProvider; -pub use tools::CLICKUP_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/clickup/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/clickup/provider.rs deleted file mode 100644 index 7b37dd49..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/clickup/provider.rs +++ /dev/null @@ -1,271 +0,0 @@ -//! ClickUp provider — incremental sync of tasks assigned to the -//! authenticated user, with per-item persistence into the Memory Tree. -//! -//! On each sync pass: -//! -//! 1. Load persistent [`SyncState`] from the KV store. -//! 2. Check the daily request budget — bail early if exhausted. -//! 3. If we don't yet know the user's numeric ID, call -//! `CLICKUP_GET_AUTHORIZED_USER` and cache the result in memory -//! (it doesn't change for the lifetime of the connection). -//! 4. If we don't yet know which workspaces (teams) the connection -//! can see, call `CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES` and -//! cache the list. -//! 5. For each workspace, page through -//! `CLICKUP_GET_FILTERED_TEAM_TASKS` filtered to the user as -//! assignee, sorted by `date_updated` descending. Stop a workspace -//! early once we hit tasks older than the cursor. -//! 6. For each task, persist as a single memory document if it's new -//! *or* edited since the last sync. -//! 7. Advance the cursor to the newest `date_updated` seen and save. -//! -//! Privacy posture: we only pull tasks the user is assigned to, never -//! the whole workspace's task graph. This mirrors the -//! "fetch-what-the-user-sees" model `gmail` / `notion` already follow -//! and avoids accidentally ingesting other teammates' private tasks. - -use async_trait::async_trait; -use serde_json::json; - -use super::normalization; -use crate::sync::composio::providers::{ - first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, - CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind, -}; - -pub(crate) const ACTION_GET_AUTHORIZED_USER: &str = "CLICKUP_GET_AUTHORIZED_USER"; -pub(crate) const ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES: &str = - "CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES"; -pub(crate) const ACTION_GET_FILTERED_TEAM_TASKS: &str = "CLICKUP_GET_FILTERED_TEAM_TASKS"; - -/// Paths for extracting a task's unique ID. Composio sometimes wraps -/// the upstream payload under `data`, so we check both shapes. -pub(super) const TASK_ID_PATHS: &[&str] = &["id", "data.id", "task_id", "data.task_id"]; - -pub struct ClickUpProvider; - -impl ClickUpProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for ClickUpProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for ClickUpProvider { - fn toolkit_slug(&self) -> &'static str { - "clickup" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(super::tools::CLICKUP_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - // 30 minutes — same cadence as Notion. ClickUp tasks change - // more slowly than chat but faster than email, so this is in - // the middle. - Some(resolve_sync_interval_secs("clickup", 30 * 60)) - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:clickup] fetch_user_profile via {ACTION_GET_AUTHORIZED_USER}" - ); - - let resp = ctx - .execute(ACTION_GET_AUTHORIZED_USER, Some(json!({}))) - .await - .map_err(|e| { - format!("[composio:clickup] {ACTION_GET_AUTHORIZED_USER} failed: {e:#}") - })?; - - if !resp.successful { - let err = resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!( - "[composio:clickup] {ACTION_GET_AUTHORIZED_USER}: {err}" - )); - } - - // Composio's wrapping puts ClickUp's `{user: {…}}` payload at - // `data` or `data.user`. We probe both — `pick_str` walks dotted - // paths so `user.username` and `data.user.username` both work. - let data = &resp.data; - let display_name = pick_str(data, &["user.username", "data.user.username", "username"]); - let email = pick_str(data, &["user.email", "data.user.email", "email"]); - let username = normalization::extract_user_id(data); - let avatar_url = pick_str( - data, - &[ - "user.profilePicture", - "data.user.profilePicture", - "profilePicture", - ], - ); - let profile_url = None; - - Ok(ProviderUserProfile { - toolkit: "clickup".to_string(), - connection_id: ctx.connection_id.clone(), - display_name, - email, - username, - avatar_url, - profile_url, - extras: data.clone(), - }) - } - - /// Incremental sync via the generic - /// `orchestrator`: - /// user/workspace resolution, the per-workspace page loop, dedup, the - /// `max_items` cap, the epoch-ms `sync_depth_days` window, and cursor - /// handling live in `run_sync`; the ClickUp-specific primitives live in - /// `super::source`. - async fn fetch_tasks( - &self, - ctx: &ProviderContext, - filter: &TaskFetchFilter, - ) -> Result, String> { - let max = filter.effective_max(); - tracing::debug!( - connection_id = ?ctx.connection_id, - max, - team_id = ?filter.team_id, - assignee_is_me = filter.assignee_is_me, - "[composio:clickup] fetch_tasks" - ); - - // Resolve which workspaces (teams) to query. An explicit - // `team_id` from the filter wins; otherwise enumerate every - // workspace the connection can see. - let workspaces = match &filter.team_id { - Some(team) if !team.trim().is_empty() => vec![team.trim().to_string()], - _ => { - let resp = ctx - .execute(ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES, Some(json!({}))) - .await - .map_err(|e| { - format!( - "[composio:clickup] {ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES}: {e:#}" - ) - })?; - if !resp.successful { - return Err(format!( - "[composio:clickup] {ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - normalization::extract_workspace_ids(&resp.data) - } - }; - - // Resolve the current user id only when the filter scopes to - // "assigned to me". - let assignees: Vec = if filter.assignee_is_me { - let resp = ctx - .execute(ACTION_GET_AUTHORIZED_USER, Some(json!({}))) - .await - .map_err(|e| format!("[composio:clickup] {ACTION_GET_AUTHORIZED_USER}: {e:#}"))?; - // Fail closed: if we can't resolve the user, error rather than - // silently dropping the assignee filter and fetching the whole - // workspace's tasks. - if !resp.successful { - return Err(format!( - "[composio:clickup] {ACTION_GET_AUTHORIZED_USER}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - let id = normalization::extract_user_id(&resp.data).ok_or_else(|| { - "[composio:clickup] CLICKUP_GET_AUTHORIZED_USER returned no user.id".to_string() - })?; - vec![id] - } else { - Vec::new() - }; - - let mut out: Vec = Vec::new(); - 'workspaces: for workspace_id in &workspaces { - let mut args = json!({ - "team_id": workspace_id, - "order_by": "updated", - "reverse": true, - "page": 0, - "page_size": max.min(100) as u32, - "subtasks": true, - }); - if !assignees.is_empty() { - args["assignees"] = json!(assignees); - } - if let Some(list_id) = filter.list_id.as_deref().filter(|s| !s.trim().is_empty()) { - args["list_ids"] = json!([list_id]); - } - merge_extra(&mut args, &filter.extra); - - let resp = ctx - .execute(ACTION_GET_FILTERED_TEAM_TASKS, Some(args)) - .await - .map_err(|e| { - format!("[composio:clickup] {ACTION_GET_FILTERED_TEAM_TASKS} ws={workspace_id}: {e:#}") - })?; - if !resp.successful { - return Err(format!( - "[composio:clickup] {ACTION_GET_FILTERED_TEAM_TASKS} ws={workspace_id}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - - for task in normalization::extract_tasks(&resp.data) { - if out.len() >= max { - break 'workspaces; - } - if let Some(nt) = normalize_clickup_task(&task) { - out.push(nt); - } - } - } - - tracing::debug!(count = out.len(), "[composio:clickup] fetch_tasks complete"); - Ok(out) - } -} - -/// Map a raw ClickUp task payload into a [`NormalizedTask`]. Returns -/// `None` only when the task has no extractable id (unroutable). -pub(super) fn normalize_clickup_task(task: &serde_json::Value) -> Option { - let external_id = pick_str(task, TASK_ID_PATHS)?; - let title = normalization::extract_task_name(task) - .unwrap_or_else(|| format!("ClickUp task {external_id}")); - Some(NormalizedTask { - external_id, - source_id: String::new(), - provider: "clickup".to_string(), - kind: TaskKind::Generic, - title, - body: pick_str(task, &["description", "data.description", "text_content"]), - url: pick_str(task, &["url", "data.url"]), - status: pick_str(task, &["status.status", "data.status.status", "status"]), - assignee: first_array_str( - task, - &["assignees", "data.assignees"], - &["username", "email"], - ), - due: pick_str(task, &["due_date", "data.due_date"]), - labels: Vec::new(), - priority: pick_str(task, &["priority.priority", "data.priority.priority"]), - updated_at: normalization::extract_task_updated(task), - raw: task.clone(), - }) -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/clickup/tests.rs b/crates/tinymemory-core/src/sync/composio/providers/clickup/tests.rs deleted file mode 100644 index 7c2aa813..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/clickup/tests.rs +++ /dev/null @@ -1,233 +0,0 @@ -//! Unit tests for the ClickUp provider. - -use super::normalization::{ - extract_task_name, extract_task_updated, extract_tasks, extract_user_id, extract_workspace_ids, -}; -use super::provider::{ - normalize_clickup_task, ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES, ACTION_GET_AUTHORIZED_USER, - ACTION_GET_FILTERED_TEAM_TASKS, -}; -use super::ClickUpProvider; -use crate::sync::composio::providers::{ - ComposioProvider, ComposioUsageHandle, ProviderContext, TaskFetchFilter, TaskKind, -}; -use serde_json::json; -use std::sync::Arc; - -fn context() -> ProviderContext { - ProviderContext { - config: Arc::new(tinymemory_api::host::test_support::TestHostConfig::default()) - as Arc, - toolkit: "clickup".into(), - connection_id: Some("connection-1".into()), - usage: ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - } -} - -#[test] -fn extract_tasks_walks_common_shapes() { - let v1 = json!({ "data": { "tasks": [{"id": "t1"}] } }); - let v2 = json!({ "tasks": [{"id": "t2"}, {"id": "t3"}] }); - let v3 = json!({ "data": {} }); - assert_eq!(extract_tasks(&v1).len(), 1); - assert_eq!(extract_tasks(&v2).len(), 2); - assert_eq!(extract_tasks(&v3).len(), 0); -} - -#[test] -fn extract_task_name_finds_name_field() { - let task = json!({ "id": "abc", "name": "Build feature X" }); - assert_eq!(extract_task_name(&task), Some("Build feature X".into())); -} - -#[test] -fn extract_task_name_falls_back_to_wrapped_data() { - let task = json!({ "data": { "name": "Wrapped" } }); - assert_eq!(extract_task_name(&task), Some("Wrapped".into())); -} - -#[test] -fn extract_task_name_returns_none_when_missing() { - let task = json!({ "id": "abc" }); - assert!(extract_task_name(&task).is_none()); -} - -#[test] -fn extract_task_updated_handles_string_form() { - let task = json!({ "date_updated": "1733412345678" }); - assert_eq!( - extract_task_updated(&task), - Some("1733412345678".to_string()) - ); -} - -#[test] -fn extract_task_updated_handles_nested_data() { - let task = json!({ "data": { "dateUpdated": "1700000000000" } }); - assert_eq!( - extract_task_updated(&task), - Some("1700000000000".to_string()) - ); -} - -#[test] -fn extract_task_updated_returns_none_when_missing() { - let task = json!({ "id": "abc" }); - assert!(extract_task_updated(&task).is_none()); -} - -#[test] -fn extract_user_id_handles_numeric_id() { - let data = json!({ "user": { "id": 12345 } }); - assert_eq!(extract_user_id(&data), Some("12345".to_string())); -} - -#[test] -fn extract_user_id_handles_wrapped_payload() { - let data = json!({ "data": { "user": { "id": "777" } } }); - assert_eq!(extract_user_id(&data), Some("777".to_string())); -} - -#[test] -fn extract_user_id_none_when_missing() { - let data = json!({ "foo": "bar" }); - assert!(extract_user_id(&data).is_none()); -} - -#[test] -fn extract_workspace_ids_from_teams_array() { - let data = json!({ - "teams": [ - { "id": "ws1", "name": "Personal" }, - { "id": "ws2", "name": "Acme" }, - ] - }); - assert_eq!(extract_workspace_ids(&data), vec!["ws1", "ws2"]); -} - -#[test] -fn extract_workspace_ids_handles_wrapped_payload() { - let data = json!({ - "data": { - "teams": [ - { "id": "ws1" }, - { "id": "ws2" }, - { "id": "ws3" }, - ] - } - }); - assert_eq!(extract_workspace_ids(&data), vec!["ws1", "ws2", "ws3"]); -} - -#[test] -fn extract_workspace_ids_empty_when_no_teams() { - let data = json!({ "foo": "bar" }); - assert!(extract_workspace_ids(&data).is_empty()); -} - -#[test] -fn extract_workspace_ids_skips_entries_without_id() { - let data = json!({ - "teams": [ - { "name": "Anonymous" }, - { "id": "ws1", "name": "Real" }, - ] - }); - assert_eq!(extract_workspace_ids(&data), vec!["ws1"]); -} - -#[test] -fn provider_metadata_is_stable() { - let p = ClickUpProvider::new(); - assert_eq!(p.toolkit_slug(), "clickup"); - assert_eq!(p.sync_interval_secs(), Some(30 * 60)); - assert!(p.curated_tools().is_some()); -} - -#[test] -fn curated_tools_contains_core_read_surface() { - let p = ClickUpProvider::new(); - let curated = p.curated_tools().expect("CLICKUP_CURATED is registered"); - let slugs: Vec<&str> = curated.iter().map(|t| t.slug).collect(); - // The three actions the sync path depends on must be advertised. - assert!(slugs.contains(&"CLICKUP_GET_AUTHORIZED_USER")); - assert!(slugs.contains(&"CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES")); - assert!(slugs.contains(&"CLICKUP_GET_FILTERED_TEAM_TASKS")); -} - -#[test] -fn default_impl_matches_new() { - // `ClickUpProvider` is a unit struct, so we compare observable - // trait surface instead of deriving `PartialEq`. This catches a - // future regression where `new()` and `default()` drift apart - // (e.g. one is given an extra field but the other is forgotten). - let a = ClickUpProvider::new(); - let b = ::default(); - assert_eq!(a.toolkit_slug(), b.toolkit_slug()); - assert_eq!(a.sync_interval_secs(), b.sync_interval_secs()); - assert_eq!( - a.curated_tools().map(<[_]>::len), - b.curated_tools().map(<[_]>::len), - ); -} - -#[tokio::test] -async fn provider_calls_fail_with_the_action_that_needs_a_host() { - let provider = ClickUpProvider::new(); - let ctx = context(); - let profile = provider - .fetch_user_profile(&ctx) - .await - .expect_err("profile requires a configured host"); - assert!(profile.contains(ACTION_GET_AUTHORIZED_USER), "{profile}"); - - let workspaces = provider - .fetch_tasks(&ctx, &TaskFetchFilter::default()) - .await - .expect_err("workspace discovery requires a configured host"); - assert!( - workspaces.contains(ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES), - "{workspaces}" - ); - - let tasks = provider - .fetch_tasks( - &ctx, - &TaskFetchFilter { - team_id: Some("team-1".into()), - assignee_is_me: false, - ..Default::default() - }, - ) - .await - .expect_err("task fetch requires a configured host"); - assert!(tasks.contains(ACTION_GET_FILTERED_TEAM_TASKS), "{tasks}"); -} - -#[test] -fn task_normalization_maps_wrapped_fields_and_rejects_missing_ids() { - let task = normalize_clickup_task(&json!({ - "data": { - "task_id": "task-7", - "description": "Ship deterministic tests", - "url": "https://app.clickup.com/t/task-7", - "status": {"status": "in progress"}, - "assignees": [{"username": "alice"}], - "due_date": "1700000000000", - "priority": {"priority": "high"}, - "dateUpdated": "1690000000000" - } - })) - .expect("wrapped task normalizes"); - assert_eq!(task.external_id, "task-7"); - assert_eq!(task.title, "ClickUp task task-7"); - assert_eq!(task.kind, TaskKind::Generic); - assert_eq!(task.body.as_deref(), Some("Ship deterministic tests")); - assert_eq!(task.assignee.as_deref(), Some("alice")); - assert_eq!(task.status.as_deref(), Some("in progress")); - assert_eq!(task.priority.as_deref(), Some("high")); - assert_eq!(task.labels, Vec::::new()); - assert!(normalize_clickup_task(&json!({"name": "unroutable"})).is_none()); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/clickup/tools.rs b/crates/tinymemory-core/src/sync/composio/providers/clickup/tools.rs deleted file mode 100644 index b52c6dd7..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/clickup/tools.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! The curated `clickup` catalog, re-exported at its historical path. -//! -//! The table moved to [`tinymemory_api::composio::catalogs::clickup`] with every -//! other catalog — see [`super::super::catalogs`] for why. - -pub use tinymemory_api::composio::catalogs::clickup::CLICKUP_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/descriptions.rs b/crates/tinymemory-core/src/sync/composio/providers/descriptions.rs deleted file mode 100644 index b617a833..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/descriptions.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! Human-readable capability summaries, re-exported at their historical path. -//! -//! Moved to [`tinymemory_api::composio::catalogs::descriptions`] with the -//! catalogs — see [`super::catalogs`] for why. - -pub use tinymemory_api::composio::catalogs::descriptions::toolkit_description; diff --git a/crates/tinymemory-core/src/sync/composio/providers/github/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/github/mod.rs deleted file mode 100644 index 89dbd5b4..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/github/mod.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! GitHub Composio provider — incremental Memory Tree ingest for issues and -//! pull requests involving the connected user. -//! -//! Mirrors the [`crate::sync::composio::providers::clickup`] layout so -//! anyone familiar with ClickUp/Notion ingestion can read this without -//! re-learning a new shape: -//! -//! - `provider.rs` — `impl ComposioProvider for GitHubProvider` -//! - `normalization` — payload-shape helpers, now reached through -//! `tinymemory-sync` (issue #18 §B3) -//! - `tools.rs` — `GITHUB_CURATED` whitelist of Composio actions -//! - `tests.rs` — unit tests for the helpers + trait metadata -//! -//! Issue: #2408. - -// The payload normalisers moved to tinycortex (they are pure Value -// transforms, i.e. driver-side). Aliased under the old module name so -// every `normalization::extract_*` call site below stays unchanged. -use tinymemory_sync::github as normalization; -mod provider; -#[cfg(test)] -mod tests; -pub mod tools; - -pub use provider::GitHubProvider; -pub use tools::GITHUB_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/github/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/github/provider.rs deleted file mode 100644 index e01992a6..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/github/provider.rs +++ /dev/null @@ -1,547 +0,0 @@ -//! GitHub provider — incremental sync of issues and pull requests involving -//! the authenticated user, with per-item persistence into the Memory Tree. -//! -//! On each sync pass: -//! -//! 1. Load persistent [`SyncState`] from the KV store. -//! 2. Check the daily request budget — bail early if exhausted. -//! 3. Resolve the authenticated user's GitHub login (used in the search -//! query); cached cheaply across re-fetches. -//! 4. Search for issues and PRs involving the user via -//! `GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS` with `involves:{login}`, filtered to items -//! updated since the cursor (when available). -//! 5. For each result, persist as a single memory document if it's new -//! *or* edited since the last sync. -//! 6. Advance the cursor to the newest `updated_at` seen and save. -//! -//! Privacy posture: the `involves:` search qualifier returns only items the -//! user created, was assigned to, mentioned in, or commented on — it never -//! surfaces private repos the user can't access. This mirrors the -//! "fetch-what-the-user-sees" model gmail / notion already follow. - -use async_trait::async_trait; -use serde_json::{json, Value}; -use std::time::Duration; - -use super::normalization; -use crate::sync::composio::providers::{ - merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, - GithubFetchMode, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, - TaskKind, -}; - -pub(crate) const ACTION_GET_AUTHENTICATED_USER: &str = "GITHUB_GET_THE_AUTHENTICATED_USER"; -pub(crate) const ACTION_SEARCH_ISSUES: &str = "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS"; - -const GH_CLI_TIMEOUT: Duration = Duration::from_secs(30); -const GITHUB_TASK_SEARCH_TIMEOUT: Duration = Duration::from_secs(20); - -pub struct GitHubProvider; - -impl GitHubProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for GitHubProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for GitHubProvider { - fn toolkit_slug(&self) -> &'static str { - "github" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(super::tools::GITHUB_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - // 30 minutes — GitHub issues change less frequently than Slack - // messages, so a half-hour cadence keeps the memory fresh without - // hammering the search API. - Some(resolve_sync_interval_secs("github", 30 * 60)) - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:github] fetch_user_profile via {ACTION_GET_AUTHENTICATED_USER}" - ); - - let resp = ctx - .execute(ACTION_GET_AUTHENTICATED_USER, Some(json!({}))) - .await - .map_err(|e| { - format!("[composio:github] {ACTION_GET_AUTHENTICATED_USER} failed: {e:#}") - })?; - - if !resp.successful { - let err = resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!( - "[composio:github] {ACTION_GET_AUTHENTICATED_USER}: {err}" - )); - } - - let data = &resp.data; - let login = normalization::extract_user_login(data); - let display_name = pick_str(data, &["name", "data.name"]).or_else(|| login.clone()); - let email = pick_str(data, &["email", "data.email"]); - let avatar_url = pick_str(data, &["avatar_url", "data.avatar_url"]); - let profile_url = pick_str(data, &["html_url", "data.html_url"]); - - Ok(ProviderUserProfile { - toolkit: "github".to_string(), - connection_id: ctx.connection_id.clone(), - display_name, - email, - username: login, - avatar_url, - profile_url, - extras: data.clone(), - }) - } - - /// Incremental sync via the generic - /// `orchestrator`: - /// login resolution, pagination, dedup, the `max_items` cap, and cursor - /// handling live in `run_sync`; the GitHub-specific primitives — including - /// the **server-side** `sync_depth_days` window — live in `super::source`. - async fn fetch_tasks( - &self, - ctx: &ProviderContext, - filter: &TaskFetchFilter, - ) -> Result, String> { - let max = filter.effective_max(); - let query = build_fetch_query(filter); - tracing::debug!( - connection_id = ?ctx.connection_id, - max, - mode = ?filter.github_fetch_mode, - query = %query, - "[composio:github] fetch_tasks" - ); - - // Select the data source by the user-configured fetch mode. `Auto` - // (the default) keeps the shipped Composio path as primary and treats - // local `gh`/REST as a true fallback — only used when the Composio - // round-trip errors or is unavailable. `Composio` / `Local` force one - // path. Normalization happens ONCE below regardless of source. - let data = match filter.github_fetch_mode { - GithubFetchMode::Composio => { - fetch_github_tasks_composio(ctx, &query, max, &filter.extra).await? - } - GithubFetchMode::Local => fetch_github_tasks_local(&query, max, &filter.extra).await?, - GithubFetchMode::Auto => { - match fetch_github_tasks_composio(ctx, &query, max, &filter.extra).await { - Ok(d) => d, - Err(e) => { - tracing::info!( - error = %e, - "[composio:github] Composio fetch unavailable; falling back to local gh/REST" - ); - fetch_github_tasks_local(&query, max, &filter.extra).await? - } - } - } - }; - - let mut out: Vec = Vec::new(); - for issue in normalization::extract_issues(&data) { - if out.len() >= max { - break; - } - if let Some(nt) = normalize_github_issue(&issue) { - out.push(nt); - } - } - tracing::debug!(count = out.len(), "[composio:github] fetch_tasks complete"); - Ok(out) - } -} - -/// Fetch GitHub issues/PRs through the connected Composio account. -/// -/// This is the original shipped `fetch_tasks` data path: it builds the -/// `GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS` search args, merges any advanced -/// `extra` query fragment, fires the action through the mode-aware -/// `ctx.execute` chokepoint, and returns the raw response `data` for the -/// shared normalization loop. Kept as a sibling of -/// [`fetch_github_tasks_local`] so `fetch_tasks` can select between them by -/// [`GithubFetchMode`]. -async fn fetch_github_tasks_composio( - ctx: &ProviderContext, - query: &str, - max: usize, - extra: &Value, -) -> Result { - let mut args = json!({ - "q": query, - "sort": "updated", - "order": "desc", - "per_page": max.min(100) as u32, - "page": 1, - }); - merge_extra(&mut args, extra); - - let resp = ctx - .execute(ACTION_SEARCH_ISSUES, Some(args)) - .await - .map_err(|e| format!("[composio:github] {ACTION_SEARCH_ISSUES}: {e:#}"))?; - if !resp.successful { - return Err(format!( - "[composio:github] {ACTION_SEARCH_ISSUES}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - Ok(resp.data) -} - -async fn fetch_github_tasks_local(query: &str, max: usize, extra: &Value) -> Result { - let mut args = json!({ - "q": query, - "sort": "updated", - "order": "desc", - "per_page": max.min(100) as u32, - "page": 1, - }); - merge_extra(&mut args, extra); - expand_me_in_github_search_args(&mut args).await; - - match gh_search_issues(&args).await { - Ok(data) => Ok(data), - Err(gh_err) => { - tracing::debug!( - error = %gh_err, - "[task_sources:github] gh api search failed, falling back to REST" - ); - rest_search_issues(&args).await.map_err(|rest_err| { - format!("[task_sources:github] local GitHub search failed: gh: {gh_err}; REST: {rest_err}") - }) - } - } -} - -async fn gh_search_issues(args: &Value) -> Result { - let mut cmd = tokio::process::Command::new("gh"); - cmd.arg("api") - .arg("--method") - .arg("GET") - .arg("search/issues"); - for (key, value) in github_search_arg_pairs(args)? { - cmd.arg("-f").arg(format!("{key}={value}")); - } - - let output = tokio::time::timeout(GH_CLI_TIMEOUT, cmd.output()) - .await - .map_err(|_| format!("gh command timed out after {}s", GH_CLI_TIMEOUT.as_secs()))? - .map_err(|e| format!("gh command failed: {e}"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("gh exited {}: {stderr}", output.status)); - } - - let stdout = - String::from_utf8(output.stdout).map_err(|e| format!("gh output not utf8: {e}"))?; - serde_json::from_str(&stdout).map_err(|e| format!("parse gh search response: {e}")) -} - -async fn rest_search_issues(args: &Value) -> Result { - let client = reqwest::Client::builder() - .timeout(GITHUB_TASK_SEARCH_TIMEOUT) - .build() - .map_err(|e| format!("failed to build GitHub client: {e}"))?; - - let mut request = client - .get("https://api.github.com/search/issues") - .header("User-Agent", "openhuman") - .header("Accept", "application/vnd.github+json"); - - if let Some(token) = github_env_token() { - request = request.header("Authorization", format!("Bearer {token}")); - } - - let pairs = github_search_arg_pairs(args)?; - let resp = request - .query(&pairs) - .send() - .await - .map_err(|e| format!("GitHub API request failed: {e}"))?; - - if !resp.status().is_success() { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - return Err(format!("GitHub API returned {status}: {body}")); - } - - resp.json::() - .await - .map_err(|e| format!("parse GitHub API response: {e}")) -} - -pub(super) fn github_env_token() -> Option { - std::env::var("GH_TOKEN") - .or_else(|_| std::env::var("GITHUB_TOKEN")) - .ok() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) -} - -async fn expand_me_in_github_search_args(args: &mut Value) { - let Some(query) = args.get("q").and_then(Value::as_str).map(str::to_string) else { - return; - }; - if !query.contains("@me") { - return; - } - let Some(login) = resolve_github_login().await else { - return; - }; - if let Some(obj) = args.as_object_mut() { - obj.insert("q".to_string(), Value::String(query.replace("@me", &login))); - } -} - -async fn resolve_github_login() -> Option { - if let Some(login) = resolve_github_login_with_gh().await { - return Some(login); - } - resolve_github_login_with_rest().await -} - -async fn resolve_github_login_with_gh() -> Option { - let output = tokio::time::timeout( - GH_CLI_TIMEOUT, - tokio::process::Command::new("gh") - .arg("api") - .arg("user") - .arg("--jq") - .arg(".login") - .output(), - ) - .await - .ok()? - .ok()?; - if !output.status.success() { - return None; - } - String::from_utf8(output.stdout) - .ok() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) -} - -async fn resolve_github_login_with_rest() -> Option { - let token = github_env_token()?; - let client = reqwest::Client::builder() - .timeout(GITHUB_TASK_SEARCH_TIMEOUT) - .build() - .ok()?; - let resp = client - .get("https://api.github.com/user") - .header("User-Agent", "openhuman") - .header("Accept", "application/vnd.github+json") - .header("Authorization", format!("Bearer {token}")) - .send() - .await - .ok()?; - if !resp.status().is_success() { - return None; - } - resp.json::() - .await - .ok() - .and_then(|value| { - value - .get("login") - .and_then(Value::as_str) - .map(str::to_string) - }) - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) -} - -pub(super) fn github_search_arg_pairs(args: &Value) -> Result, String> { - let obj = args - .as_object() - .ok_or_else(|| "GitHub search args must be a JSON object".to_string())?; - let mut out = Vec::with_capacity(obj.len()); - for (key, value) in obj { - let rendered = match value { - Value::String(s) => s.trim().to_string(), - Value::Number(n) => n.to_string(), - Value::Bool(b) => b.to_string(), - Value::Null => continue, - other => other.to_string(), - }; - if !rendered.is_empty() { - out.push((key.clone(), rendered)); - } - } - Ok(out) -} - -/// Build a GitHub Search-Issues query from a [`TaskFetchFilter`]. -/// -/// Combines repo / label / state / assignee qualifiers. When the filter -/// carries no scoping constraints at all we fall back to `involves:@me` so a -/// task source never accidentally pulls the entire public issue universe. -/// -/// State bias: when the filter sets no explicit `state`, we append `is:open` -/// so closed issues and merged/closed PRs aren't fetched in the first place -/// (the unconditional skip in `normalize_github_issue` is the hard guarantee; -/// this is the fetch-side optimization). An explicit `state` is respected and -/// `is:open` is not double-added. -pub(super) fn build_fetch_query(filter: &TaskFetchFilter) -> String { - let mut parts: Vec = Vec::new(); - if let Some(repo) = filter - .repo - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(normalize_github_repo_filter) - { - parts.push(format!("repo:{repo}")); - } - for label in filter - .labels - .iter() - .map(|l| l.trim()) - .filter(|l| !l.is_empty()) - { - parts.push(format!("label:\"{label}\"")); - } - if filter.assignee_is_me { - parts.push("assignee:@me".to_string()); - } - // If no repo/label/assignee scoping was supplied, fall back to - // `involves:@me` (plus the open bias) rather than the whole issue universe. - if parts.is_empty() { - parts.push("involves:@me".to_string()); - } - let explicit_state = filter - .state - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - match explicit_state { - // Caller pinned a state — respect it verbatim, don't add `is:open`. - Some(state) => parts.push(format!("state:{state}")), - // No explicit state — bias the fetch toward open items. - None => parts.push("is:open".to_string()), - } - parts.join(" ") -} - -pub(super) fn normalize_github_repo_filter(raw: &str) -> String { - let trimmed = raw.trim(); - let without_scheme = trimmed - .strip_prefix("https://github.com/") - .or_else(|| trimmed.strip_prefix("http://github.com/")) - .or_else(|| trimmed.strip_prefix("git@github.com:")) - .unwrap_or(trimmed); - let cleaned = without_scheme - .trim_start_matches('/') - .trim_end_matches('/') - .trim_end_matches(".git"); - let mut parts = cleaned.split('/').filter(|part| !part.is_empty()); - match (parts.next(), parts.next()) { - (Some(owner), Some(repo)) => { - let repo = repo.trim_end_matches(".git"); - if owner.is_empty() || repo.is_empty() { - trimmed.to_string() - } else { - format!("{owner}/{repo}") - } - } - _ => trimmed.to_string(), - } -} - -/// Map a raw GitHub issue/PR payload into a [`NormalizedTask`]. -/// -/// GitHub's search-issues-and-PRs endpoint returns both shapes; a hit is a -/// pull request iff it carries a `pull_request` object. We tag the kind here -/// so enrichment can phrase the objective as "review" vs "resolve". -/// -/// Returns `None` when the item's state is `"closed"` — a merged/closed PR -/// and a closed issue both report `state == "closed"`, and there is no point -/// ingesting work that is already done. This skip is unconditional (it does -/// not depend on the fetch query), so even if a `closed` item slips through -/// the query bias it is dropped here. -pub(super) fn normalize_github_issue(issue: &serde_json::Value) -> Option { - let external_id = normalization::extract_issue_id(issue)?; - let status = pick_str(issue, &["state", "data.state"]); - if status - .as_deref() - .map(|s| s.eq_ignore_ascii_case("closed")) - .unwrap_or(false) - { - tracing::debug!( - external_id = %external_id, - "[composio:github] normalize_github_issue: skipping closed item (merged PR / closed issue)" - ); - return None; - } - let title = normalization::extract_issue_title(issue) - .unwrap_or_else(|| format!("GitHub issue {external_id}")); - let kind = if is_pull_request(issue) { - TaskKind::PullRequest - } else { - TaskKind::Issue - }; - Some(NormalizedTask { - external_id, - source_id: String::new(), - provider: "github".to_string(), - kind, - title, - body: pick_str(issue, &["body", "data.body"]), - url: pick_str(issue, &["html_url", "data.html_url"]), - status, - assignee: pick_str(issue, &["assignee.login", "data.assignee.login"]), - due: None, - labels: extract_github_labels(issue), - priority: None, - updated_at: normalization::extract_issue_updated_at(issue), - raw: issue.clone(), - }) -} - -/// A GitHub search hit is a pull request iff it carries a non-null -/// `pull_request` object (issues never do). Tolerant of the Composio `data` -/// wrapper. -fn is_pull_request(issue: &serde_json::Value) -> bool { - let pr = issue - .get("pull_request") - .or_else(|| issue.get("data").and_then(|d| d.get("pull_request"))); - matches!(pr, Some(v) if !v.is_null()) -} - -/// Extract label names from a GitHub issue payload (`labels` is an array -/// of `{ name }` objects). Tolerant of the Composio `data` wrapper. -fn extract_github_labels(issue: &serde_json::Value) -> Vec { - let arr = issue - .get("labels") - .or_else(|| issue.get("data").and_then(|d| d.get("labels"))) - .and_then(|v| v.as_array()); - match arr { - Some(items) => items - .iter() - .filter_map(|l| l.get("name").and_then(|n| n.as_str())) - .map(|s| s.to_string()) - .collect(), - None => Vec::new(), - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/github/tests.rs b/crates/tinymemory-core/src/sync/composio/providers/github/tests.rs deleted file mode 100644 index d63506fd..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/github/tests.rs +++ /dev/null @@ -1,760 +0,0 @@ -//! Unit tests for the GitHub Composio provider. - -use super::normalization::{ - extract_issue_id, extract_issue_title, extract_issue_updated_at, extract_issues, - extract_user_login, -}; -use super::provider::github_env_token; -use super::provider::{ - build_fetch_query, github_search_arg_pairs, normalize_github_issue, - normalize_github_repo_filter, ACTION_GET_AUTHENTICATED_USER, ACTION_SEARCH_ISSUES, -}; -use super::tools::GITHUB_CURATED; -use super::GitHubProvider; -use crate::sync::composio::providers::ComposioProvider; -use crate::sync::composio::providers::{ - ComposioUsageHandle, GithubFetchMode, ProviderContext, TaskFetchFilter, TaskKind, -}; -use serde_json::json; -use std::sync::Arc; - -// ── extract_issues ─────────────────────────────────────────────────────────── - -#[test] -fn extract_issues_walks_data_items_shape() { - let data = json!({ "data": { "items": [{"id": 1u64}] } }); - assert_eq!(extract_issues(&data).len(), 1); -} - -#[test] -fn extract_issues_walks_top_level_items_shape() { - let data = json!({ "items": [{"id": 1u64}, {"id": 2u64}] }); - assert_eq!(extract_issues(&data).len(), 2); -} - -#[test] -fn extract_issues_returns_empty_when_no_items_key() { - let data = json!({ "foo": "bar" }); - assert!(extract_issues(&data).is_empty()); -} - -#[test] -fn extract_issues_handles_data_data_nesting() { - let data = json!({ "data": { "data": { "items": [{"id": 9u64}] } } }); - assert_eq!(extract_issues(&data).len(), 1); -} - -// ── extract_issue_id ───────────────────────────────────────────────────────── - -#[test] -fn extract_issue_id_from_numeric_id() { - let issue = json!({ "id": 123456789u64, "title": "Fix race" }); - assert_eq!(extract_issue_id(&issue), Some("123456789".to_string())); -} - -#[test] -fn extract_issue_id_from_wrapped_data() { - let issue = json!({ "data": { "id": 42u64 } }); - assert_eq!(extract_issue_id(&issue), Some("42".to_string())); -} - -#[test] -fn extract_issue_id_falls_back_to_html_url_path() { - let issue = json!({ - "html_url": "https://github.com/owner/repo/issues/7" - }); - assert_eq!(extract_issue_id(&issue), Some("owner/repo#7".to_string())); -} - -#[test] -fn extract_issue_id_none_when_no_id_or_url() { - let issue = json!({ "title": "orphan" }); - assert!(extract_issue_id(&issue).is_none()); -} - -// ── extract_issue_title ────────────────────────────────────────────────────── - -#[test] -fn extract_issue_title_builds_prefixed_title() { - let issue = json!({ - "id": 1u64, - "title": "Fix race condition", - "html_url": "https://github.com/acme/core/issues/99" - }); - assert_eq!( - extract_issue_title(&issue), - Some("GitHub: acme/core#99: Fix race condition".to_string()) - ); -} - -#[test] -fn extract_issue_title_pr_url_also_works() { - let issue = json!({ - "id": 2u64, - "title": "Add feature", - "html_url": "https://github.com/org/repo/pull/101" - }); - assert_eq!( - extract_issue_title(&issue), - Some("GitHub: org/repo#101: Add feature".to_string()) - ); -} - -#[test] -fn extract_issue_title_returns_raw_title_when_no_url() { - let issue = json!({ "title": "Bare title" }); - assert_eq!(extract_issue_title(&issue), Some("Bare title".to_string())); -} - -#[test] -fn extract_issue_title_none_when_no_title() { - let issue = json!({ "id": 1u64 }); - assert!(extract_issue_title(&issue).is_none()); -} - -// ── extract_issue_updated_at ───────────────────────────────────────────────── - -#[test] -fn extract_issue_updated_at_from_top_level() { - let issue = json!({ "updated_at": "2024-05-21T15:30:00Z" }); - assert_eq!( - extract_issue_updated_at(&issue), - Some("2024-05-21T15:30:00Z".to_string()) - ); -} - -#[test] -fn extract_issue_updated_at_from_data_wrapper() { - let issue = json!({ "data": { "updated_at": "2023-01-01T00:00:00Z" } }); - assert_eq!( - extract_issue_updated_at(&issue), - Some("2023-01-01T00:00:00Z".to_string()) - ); -} - -#[test] -fn extract_issue_updated_at_none_when_missing() { - let issue = json!({ "id": 1u64 }); - assert!(extract_issue_updated_at(&issue).is_none()); -} - -// ── extract_user_login ─────────────────────────────────────────────────────── - -#[test] -fn extract_user_login_from_top_level() { - let data = json!({ "login": "octocat" }); - assert_eq!(extract_user_login(&data), Some("octocat".to_string())); -} - -#[test] -fn extract_user_login_from_data_wrapper() { - let data = json!({ "data": { "login": "monalisa" } }); - assert_eq!(extract_user_login(&data), Some("monalisa".to_string())); -} - -#[test] -fn extract_user_login_none_when_missing() { - let data = json!({ "id": 1u64 }); - assert!(extract_user_login(&data).is_none()); -} - -// ── provider metadata ──────────────────────────────────────────────────────── - -#[test] -fn provider_metadata_is_stable() { - let p = GitHubProvider::new(); - assert_eq!(p.toolkit_slug(), "github"); - assert_eq!(p.sync_interval_secs(), Some(30 * 60)); - assert!(p.curated_tools().is_some()); -} - -#[test] -fn curated_tools_contains_core_actions() { - let p = GitHubProvider::new(); - let curated = p.curated_tools().expect("GITHUB_CURATED is registered"); - let slugs: Vec<&str> = curated.iter().map(|t| t.slug).collect(); - assert!(slugs.contains(&"GITHUB_GET_THE_AUTHENTICATED_USER")); - assert!(slugs.contains(&"GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS")); - assert!(slugs.contains(&"GITHUB_LIST_REPOSITORY_ISSUES")); - assert!(slugs.contains(&"GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER")); - assert!(slugs.contains(&"GITHUB_CREATE_A_REPOSITORY_FOR_THE_AUTHENTICATED_USER")); - // DELETE_A_REFERENCE replaces DELETE_A_BRANCH (Composio v3 rename). - assert!(slugs.contains(&"GITHUB_DELETE_A_REFERENCE")); - // CLOSE_AN_ISSUE was removed — callers must use UPDATE_AN_ISSUE with state:"closed". - assert!( - !slugs.contains(&"GITHUB_CLOSE_AN_ISSUE"), - "GITHUB_CLOSE_AN_ISSUE was removed — use GITHUB_UPDATE_AN_ISSUE with state:closed" - ); -} - -#[test] -fn default_impl_matches_new() { - let a = GitHubProvider::new(); - let b = ::default(); - assert_eq!(a.toolkit_slug(), b.toolkit_slug()); - assert_eq!(a.sync_interval_secs(), b.sync_interval_secs()); - assert_eq!( - a.curated_tools().map(<[_]>::len), - b.curated_tools().map(<[_]>::len), - ); -} - -#[test] -fn build_fetch_query_scopes_repo_labels_state_and_assignee() { - let query = build_fetch_query(&TaskFetchFilter { - repo: Some("tinyhumansai/openhuman".to_string()), - labels: vec!["bug".to_string(), "agent harness".to_string()], - state: Some("open".to_string()), - assignee_is_me: true, - ..Default::default() - }); - - assert_eq!( - query, - "repo:tinyhumansai/openhuman label:\"bug\" label:\"agent harness\" assignee:@me state:open" - ); -} - -#[test] -fn build_fetch_query_normalizes_github_repo_urls() { - let query = build_fetch_query(&TaskFetchFilter { - repo: Some("https://github.com/tinyhumansai/openhuman/pull/3267".to_string()), - state: Some("open".to_string()), - ..Default::default() - }); - - assert_eq!(query, "repo:tinyhumansai/openhuman state:open"); -} - -#[test] -fn normalize_github_repo_filter_accepts_common_repo_inputs() { - assert_eq!( - normalize_github_repo_filter("tinyhumansai/openhuman"), - "tinyhumansai/openhuman" - ); - assert_eq!( - normalize_github_repo_filter("https://github.com/tinyhumansai/openhuman.git"), - "tinyhumansai/openhuman" - ); - assert_eq!( - normalize_github_repo_filter("git@github.com:tinyhumansai/openhuman.git"), - "tinyhumansai/openhuman" - ); -} - -#[test] -fn build_fetch_query_falls_back_to_involves_me_when_unscoped() { - // No scoping and no explicit state: fall back to `involves:@me` and bias - // toward open items so closed issues / merged PRs aren't even fetched. - assert_eq!( - build_fetch_query(&TaskFetchFilter::default()), - "involves:@me is:open" - ); -} - -#[test] -fn build_fetch_query_appends_is_open_when_no_explicit_state() { - // Scoped by repo but no explicit state — `is:open` is appended. - let query = build_fetch_query(&TaskFetchFilter { - repo: Some("tinyhumansai/openhuman".to_string()), - ..Default::default() - }); - assert_eq!(query, "repo:tinyhumansai/openhuman is:open"); -} - -#[test] -fn build_fetch_query_respects_explicit_state_without_double_open() { - // Explicit `state` is respected verbatim and `is:open` is NOT added. - let query = build_fetch_query(&TaskFetchFilter { - repo: Some("tinyhumansai/openhuman".to_string()), - state: Some("closed".to_string()), - ..Default::default() - }); - assert_eq!(query, "repo:tinyhumansai/openhuman state:closed"); - assert!(!query.contains("is:open")); -} - -#[test] -fn github_search_arg_pairs_render_cli_and_rest_params() { - let args = json!({ - "q": "repo:tinyhumansai/openhuman state:open", - "sort": "updated", - "order": "desc", - "per_page": 25, - "page": 1, - "include_prs": true, - "skip": null - }); - - let pairs = github_search_arg_pairs(&args).expect("pairs"); - assert!(pairs.contains(&( - "q".to_string(), - "repo:tinyhumansai/openhuman state:open".to_string() - ))); - assert!(pairs.contains(&("per_page".to_string(), "25".to_string()))); - assert!(pairs.contains(&("include_prs".to_string(), "true".to_string()))); - assert!(!pairs.iter().any(|(key, _)| key == "skip")); -} - -// ── slug regression tests (#2768) ─────────────────────────────────────────── -// -// Guard the current Composio action slug values used by the GitHub provider. -// Outdated slugs (e.g. GITHUB_USERS_GET_AUTHENTICATED, GITHUB_LIST_REPOS, -// GITHUB_LIST_ISSUES) were previously scattered across tests; these assertions -// pin the correct values in one place so a slug rename is caught immediately. - -#[test] -fn action_get_authenticated_user_slug_is_current() { - // The Composio v3 slug is GITHUB_GET_THE_AUTHENTICATED_USER. - // Regression: was mistakenly referenced as GITHUB_USERS_GET_AUTHENTICATED - // in tests (see issue #2768). - assert_eq!( - ACTION_GET_AUTHENTICATED_USER, "GITHUB_GET_THE_AUTHENTICATED_USER", - "slug must match Composio v3 catalog; old slug GITHUB_USERS_GET_AUTHENTICATED is retired" - ); -} - -#[test] -fn action_search_issues_slug_is_current() { - assert_eq!( - ACTION_SEARCH_ISSUES, "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS", - "slug must match Composio v3 catalog" - ); -} - -#[test] -fn curated_list_does_not_contain_retired_slugs() { - // Guard against re-introducing removed slugs that no longer exist in the - // Composio v3 GitHub app catalog. - const RETIRED: &[&str] = &[ - "GITHUB_USERS_GET_AUTHENTICATED", // replaced by GITHUB_GET_THE_AUTHENTICATED_USER - "GITHUB_LIST_REPOS", // replaced by GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER - "GITHUB_LIST_ISSUES", // replaced by GITHUB_LIST_REPOSITORY_ISSUES - "GITHUB_COMMIT_MULTIPLE_FILES", // removed from Composio catalog - "GITHUB_CLOSE_AN_ISSUE", // removed; use GITHUB_UPDATE_AN_ISSUE with state=closed - "GITHUB_DELETE_A_BRANCH", // removed; use GITHUB_DELETE_A_REFERENCE - ]; - - let slugs: Vec<&str> = GITHUB_CURATED.iter().map(|t| t.slug).collect(); - for retired in RETIRED { - assert!( - !slugs.contains(retired), - "curated list must not contain retired slug {retired} (see #2768)" - ); - } -} - -#[test] -fn curated_list_contains_current_read_slugs() { - // Verify that the primary read-tier actions are present with their correct - // v3 slug names (not the old v1/v2 names). - let slugs: Vec<&str> = GITHUB_CURATED.iter().map(|t| t.slug).collect(); - let required = [ - "GITHUB_GET_THE_AUTHENTICATED_USER", - "GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER", - "GITHUB_LIST_REPOSITORY_ISSUES", - "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS", - "GITHUB_LIST_PULL_REQUESTS", - "GITHUB_GET_A_PULL_REQUEST", - ]; - for slug in required { - assert!( - slugs.contains(&slug), - "curated list must contain current slug {slug} (see #2768)" - ); - } -} - -#[test] -fn curated_list_contains_current_write_slugs() { - let slugs: Vec<&str> = GITHUB_CURATED.iter().map(|t| t.slug).collect(); - let required = [ - "GITHUB_CREATE_AN_ISSUE", - "GITHUB_UPDATE_AN_ISSUE", - "GITHUB_CREATE_A_PULL_REQUEST", - "GITHUB_MERGE_A_PULL_REQUEST", - ]; - for slug in required { - assert!( - slugs.contains(&slug), - "curated list must contain current write slug {slug} (see #2768)" - ); - } -} - -// ── GithubFetchMode (#3279) ───────────────────────────────────────────────── -// -// The fetch-mode selector makes the local `gh`/REST path a *true fallback* -// (default `Auto`) instead of a hard Composio replacement. These tests pin the -// default and the serde wire contract the UI persists into a source's -// `FilterSpec::Github { fetch_mode }`. - -#[test] -fn github_fetch_mode_defaults_to_auto() { - // `Auto` must be the default so shipped Composio users keep working and - // local/dev setups still get the fallback — neither side regresses. - assert_eq!(GithubFetchMode::default(), GithubFetchMode::Auto); -} - -#[test] -fn task_fetch_filter_default_uses_auto_fetch_mode() { - // A filter built with no explicit mode (the common path) carries `Auto`. - let filter = TaskFetchFilter::default(); - assert_eq!(filter.github_fetch_mode, GithubFetchMode::Auto); -} - -#[test] -fn github_fetch_mode_serializes_snake_case() { - assert_eq!( - serde_json::to_value(GithubFetchMode::Auto).expect("ser auto"), - json!("auto") - ); - assert_eq!( - serde_json::to_value(GithubFetchMode::Composio).expect("ser composio"), - json!("composio") - ); - assert_eq!( - serde_json::to_value(GithubFetchMode::Local).expect("ser local"), - json!("local") - ); -} - -#[test] -fn github_fetch_mode_deserializes_each_variant() { - let auto: GithubFetchMode = serde_json::from_value(json!("auto")).expect("de auto"); - let composio: GithubFetchMode = serde_json::from_value(json!("composio")).expect("de composio"); - let local: GithubFetchMode = serde_json::from_value(json!("local")).expect("de local"); - assert_eq!(auto, GithubFetchMode::Auto); - assert_eq!(composio, GithubFetchMode::Composio); - assert_eq!(local, GithubFetchMode::Local); -} - -#[test] -fn github_fetch_mode_round_trips_through_json() { - for mode in [ - GithubFetchMode::Auto, - GithubFetchMode::Composio, - GithubFetchMode::Local, - ] { - let json = serde_json::to_string(&mode).expect("ser"); - let back: GithubFetchMode = serde_json::from_str(&json).expect("de"); - assert_eq!(back, mode, "round-trip must preserve {mode:?}"); - } -} - -#[test] -fn github_fetch_mode_rejects_unknown_variant() { - let parsed: Result = serde_json::from_value(json!("remote")); - assert!(parsed.is_err(), "unknown mode strings must fail to parse"); -} - -// ── github_search_arg_pairs edge cases (#3279) ────────────────────────────── - -#[test] -fn github_search_arg_pairs_skips_null_and_empty_string_values() { - // Null values are dropped entirely; whitespace-only / empty strings are - // trimmed to empty and also dropped, so they never reach the gh CLI / REST - // query as blank params. - let args = json!({ - "q": "involves:@me", - "empty": "", - "blank": " ", - "missing": null, - "page": 1, - }); - let pairs = github_search_arg_pairs(&args).expect("pairs"); - assert!(pairs.contains(&("q".to_string(), "involves:@me".to_string()))); - assert!(pairs.contains(&("page".to_string(), "1".to_string()))); - assert!(!pairs.iter().any(|(k, _)| k == "empty")); - assert!(!pairs.iter().any(|(k, _)| k == "blank")); - assert!(!pairs.iter().any(|(k, _)| k == "missing")); -} - -#[test] -fn github_search_arg_pairs_errors_when_not_an_object() { - // A non-object value (array, scalar) is a programmer error — surface it - // rather than silently producing an empty arg set. - let err = github_search_arg_pairs(&json!(["not", "an", "object"])) - .expect_err("array args must error"); - assert!(err.contains("JSON object"), "got: {err}"); -} - -// ── github_env_token (#3279) ──────────────────────────────────────────────── - -#[test] -fn github_env_token_reads_env_and_is_none_when_unset() { - // Env-mutation test: this whole suite is the only reader of GH_TOKEN / - // GITHUB_TOKEN, but cargo runs tests in parallel threads sharing the - // process env. Hold a process-wide lock so concurrent token reads don't - // race, and restore the original values on exit. - use std::sync::Mutex; - static ENV_LOCK: Mutex<()> = Mutex::new(()); - let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - - let prev_gh = std::env::var("GH_TOKEN").ok(); - let prev_github = std::env::var("GITHUB_TOKEN").ok(); - - // Neither set → None. - std::env::remove_var("GH_TOKEN"); - std::env::remove_var("GITHUB_TOKEN"); - assert_eq!(github_env_token(), None, "no token vars → None"); - - // GH_TOKEN takes precedence; surrounding whitespace is trimmed. - std::env::set_var("GH_TOKEN", " gh-pat-123 "); - assert_eq!(github_env_token().as_deref(), Some("gh-pat-123")); - - // Falls back to GITHUB_TOKEN when GH_TOKEN is absent. - std::env::remove_var("GH_TOKEN"); - std::env::set_var("GITHUB_TOKEN", "github-pat-456"); - assert_eq!(github_env_token().as_deref(), Some("github-pat-456")); - - // A blank token is treated as unset. - std::env::set_var("GH_TOKEN", " "); - std::env::remove_var("GITHUB_TOKEN"); - assert_eq!(github_env_token(), None, "blank token → None"); - - // Restore original env. - match prev_gh { - Some(v) => std::env::set_var("GH_TOKEN", v), - None => std::env::remove_var("GH_TOKEN"), - } - match prev_github { - Some(v) => std::env::set_var("GITHUB_TOKEN", v), - None => std::env::remove_var("GITHUB_TOKEN"), - } -} - -// ── issue vs pull-request kind detection ───────────────────────────────────── - -#[test] -fn normalize_tags_pull_request_when_pull_request_object_present() { - // GitHub's issues-and-PRs search marks a PR hit with a `pull_request` object. - let pr = json!({ - "id": 42, - "title": "Add retry to fetch", - "state": "open", - "html_url": "https://github.com/o/r/pull/42", - "pull_request": { "url": "https://api.github.com/repos/o/r/pulls/42" } - }); - let nt = normalize_github_issue(&pr).expect("normalizes"); - assert_eq!(nt.kind, TaskKind::PullRequest); -} - -#[test] -fn normalize_tags_issue_when_no_pull_request_object() { - let issue = json!({ - "id": 7, - "title": "Login throws on empty password", - "state": "open", - "html_url": "https://github.com/o/r/issues/7" - }); - let nt = normalize_github_issue(&issue).expect("normalizes"); - assert_eq!(nt.kind, TaskKind::Issue); -} - -#[test] -fn normalize_tags_issue_when_pull_request_is_null() { - // The REST issue payload carries `pull_request: null` for plain issues. - let issue = json!({ - "id": 8, - "title": "Docs typo", - "state": "open", - "html_url": "https://github.com/o/r/issues/8", - "pull_request": null - }); - let nt = normalize_github_issue(&issue).expect("normalizes"); - assert_eq!(nt.kind, TaskKind::Issue); -} - -// ── skip merged PRs / closed issues ────────────────────────────────────────── - -#[test] -fn normalize_skips_closed_issue() { - // A closed issue is already-done work — drop it. - let issue = json!({ - "id": 100, - "title": "Old bug", - "state": "closed", - "html_url": "https://github.com/o/r/issues/100" - }); - assert!( - normalize_github_issue(&issue).is_none(), - "closed issue must be skipped" - ); -} - -#[test] -fn normalize_skips_merged_or_closed_pull_request() { - // A merged/closed PR also reports `state == "closed"` — drop it too. - let pr = json!({ - "id": 101, - "title": "Shipped feature", - "state": "closed", - "html_url": "https://github.com/o/r/pull/101", - "pull_request": { "url": "https://api.github.com/repos/o/r/pulls/101" } - }); - assert!( - normalize_github_issue(&pr).is_none(), - "merged/closed PR must be skipped" - ); -} - -#[test] -fn normalize_keeps_open_item() { - // An open item is kept and tagged with its kind. - let issue = json!({ - "id": 102, - "title": "Active work", - "state": "open", - "html_url": "https://github.com/o/r/issues/102" - }); - let nt = normalize_github_issue(&issue).expect("open item is kept"); - assert_eq!(nt.kind, TaskKind::Issue); - assert_eq!(nt.status.as_deref(), Some("open")); -} - -#[cfg(unix)] -#[test] -fn local_fetch_uses_gh_expands_me_and_normalizes_open_work() { - use std::os::unix::fs::PermissionsExt; - - let _env = crate::test_env_lock::TEST_ENV_LOCK - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - let previous_path = std::env::var_os("PATH"); - let previous_gh = std::env::var_os("GH_TOKEN"); - let previous_github = std::env::var_os("GITHUB_TOKEN"); - struct RestoreEnv { - path: Option, - gh: Option, - github: Option, - } - impl Drop for RestoreEnv { - fn drop(&mut self) { - match self.path.take() { - Some(value) => std::env::set_var("PATH", value), - None => std::env::remove_var("PATH"), - } - match self.gh.take() { - Some(value) => std::env::set_var("GH_TOKEN", value), - None => std::env::remove_var("GH_TOKEN"), - } - match self.github.take() { - Some(value) => std::env::set_var("GITHUB_TOKEN", value), - None => std::env::remove_var("GITHUB_TOKEN"), - } - } - } - let _restore = RestoreEnv { - path: previous_path.clone(), - gh: previous_gh, - github: previous_github, - }; - - let temp = tempfile::tempdir().expect("fake gh directory"); - let gh = temp.path().join("gh"); - std::fs::write( - &gh, - r##"#!/bin/sh -if [ "$1" = "api" ] && [ "$2" = "user" ]; then - printf '%s\n' 'octocat' - exit 0 -fi -case "$*" in - *'assignee:octocat'*) - printf '%s\n' '{"items":[{"id":1,"title":"Closed","state":"closed","html_url":"https://github.com/o/r/issues/1"},{"id":2,"title":"Open issue","state":"open","body":"Issue body","html_url":"https://github.com/o/r/issues/2","labels":[{"name":"bug"}]},{"id":3,"title":"Open PR","state":"open","html_url":"https://github.com/o/r/pull/3","pull_request":{"url":"https://api.github.com/repos/o/r/pulls/3"}},{"id":4,"title":"Beyond max","state":"open","html_url":"https://github.com/o/r/issues/4"}]}' - exit 0 - ;; - *) - printf '%s\n' 'query did not expand @me' >&2 - exit 17 - ;; -esac -"##, - ) - .expect("write fake gh"); - let mut permissions = std::fs::metadata(&gh) - .expect("fake gh metadata") - .permissions(); - permissions.set_mode(0o755); - std::fs::set_permissions(&gh, permissions).expect("make fake gh executable"); - let mut path = temp.path().as_os_str().to_os_string(); - if let Some(previous) = previous_path { - path.push(":"); - path.push(previous); - } - std::env::set_var("PATH", path); - std::env::remove_var("GH_TOKEN"); - std::env::remove_var("GITHUB_TOKEN"); - - let ctx = ProviderContext { - config: Arc::new(tinymemory_api::host::test_support::TestHostConfig::default()) - as Arc, - toolkit: "github".into(), - connection_id: Some("connection-1".into()), - usage: ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - }; - let runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .expect("test runtime"); - let tasks = runtime - .block_on(GitHubProvider::new().fetch_tasks( - &ctx, - &TaskFetchFilter { - assignee_is_me: true, - max: 2, - github_fetch_mode: GithubFetchMode::Local, - extra: json!({"custom": true}), - ..Default::default() - }, - )) - .expect("local GitHub fetch"); - assert_eq!(tasks.len(), 2); - assert_eq!(tasks[0].external_id, "2"); - assert_eq!(tasks[0].kind, TaskKind::Issue); - assert_eq!(tasks[0].labels, vec!["bug"]); - assert_eq!(tasks[1].external_id, "3"); - assert_eq!(tasks[1].kind, TaskKind::PullRequest); -} - -#[tokio::test] -async fn composio_provider_failures_name_the_action_without_network() { - use tinymemory_api::host::test_support::TestHostConfig; - - let temp = tempfile::tempdir().expect("config directory"); - let mut config = TestHostConfig::default(); - config.config_path = temp.path().join("config.toml"); - config.workspace_dir = temp.path().join("workspace"); - config.secrets_encrypt = false; - tinymemory_api::host::MemoryHostConfig::save(&config) - .await - .expect("save unsigned config"); - let ctx = ProviderContext { - config: Arc::new(config) as Arc, - toolkit: "github".into(), - connection_id: Some("connection-1".into()), - usage: ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - }; - let provider = GitHubProvider::new(); - let profile_error = provider - .fetch_user_profile(&ctx) - .await - .expect_err("signed-out profile fetch"); - assert!(profile_error.contains(ACTION_GET_AUTHENTICATED_USER)); - let task_error = provider - .fetch_tasks( - &ctx, - &TaskFetchFilter { - github_fetch_mode: GithubFetchMode::Composio, - ..Default::default() - }, - ) - .await - .expect_err("signed-out task fetch"); - assert!(task_error.contains(ACTION_SEARCH_ISSUES)); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/github/tools.rs b/crates/tinymemory-core/src/sync/composio/providers/github/tools.rs deleted file mode 100644 index cada1872..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/github/tools.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! The curated `github` catalog, re-exported at its historical path. -//! -//! The table moved to [`tinymemory_api::composio::catalogs::github`] with every -//! other catalog — see [`super::super::catalogs`] for why. - -pub use tinymemory_api::composio::catalogs::github::GITHUB_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/gmail/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/gmail/mod.rs deleted file mode 100644 index df067ec2..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/gmail/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -// The Gmail post-processor moved to tinycortex (a pure Value transform, i.e. -// driver-side). Aliased under the old module name so the single call site in -// `provider.rs` stays unchanged. -use tinymemory_sync::gmail_post_process as post_process; -mod provider; -#[cfg(test)] -mod tests; -pub mod tools; - -pub use provider::GmailProvider; -pub use tools::GMAIL_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/gmail/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/gmail/provider.rs deleted file mode 100644 index 701d56f7..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/gmail/provider.rs +++ /dev/null @@ -1,177 +0,0 @@ -//! Gmail provider — incremental sync into the memory tree. -//! -//! On each sync pass: -//! -//! 1. Load persistent [`SyncState`] from the KV store. -//! 2. Check the daily request budget — bail early if exhausted. -//! 3. Fetch a page of recent messages via `GMAIL_FETCH_EMAILS`, adding -//! a date filter when a cursor exists so only newer mail is returned. -//! 4. Run [`ComposioProvider::post_process_action_result`] (bounded -//! HTML→text, normalise, sanitise) on the page so the LLM-facing chunk -//! content is cleaned, not raw. -//! 5. Delegate incremental filtering and document ingestion to tinycortex. -//! 6. Paginate (up to budget) until no more results or all items in the -//! page are already synced. -//! 7. Advance the cursor and save state. -//! -//! Daily budget (`DEFAULT_DAILY_REQUEST_LIMIT`, default 500) caps the -//! number of `execute_tool` calls per calendar day, preventing runaway -//! API usage during large initial backfills. - -use async_trait::async_trait; -use serde_json::{json, Value}; - -use crate::sync::composio::providers::{ - pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, ProviderContext, - ProviderUserProfile, -}; - -pub(super) const ACTION_GET_PROFILE: &str = "GMAIL_GET_PROFILE"; - -pub struct GmailProvider; - -impl GmailProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for GmailProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for GmailProvider { - fn toolkit_slug(&self) -> &'static str { - "gmail" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(super::tools::GMAIL_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - Some(resolve_sync_interval_secs("gmail", 15 * 60)) - } - - fn post_process_action_result( - &self, - slug: &str, - arguments: Option<&serde_json::Value>, - data: &mut serde_json::Value, - ) { - super::post_process::post_process(slug, arguments, data); - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:gmail] fetch_user_profile via {ACTION_GET_PROFILE}" - ); - - let resp = ctx - .execute(ACTION_GET_PROFILE, Some(json!({}))) - .await - .map_err(|e| format!("[composio:gmail] {ACTION_GET_PROFILE} failed: {e:#}"))?; - - if !resp.successful { - let err = resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!("[composio:gmail] {ACTION_GET_PROFILE}: {err}")); - } - - // `data` is the inner Composio payload — paths here are relative - // to it. (The previous `data.*` paths were dead — `pick_str` - // does dotted-path traversal, so `data.emailAddress` looked for - // a nested `data.data.emailAddress` that never exists.) - let data = &resp.data; - let email = pick_str(data, &["emailAddress", "email", "profile.emailAddress"]); - // Don't fall back to the email when no name is returned — that - // produces duplicated `display_name == email` rows in the - // identity registry (#1365). Gmail's `GMAIL_GET_PROFILE` action - // doesn't return a name today, so this stays None. - let display_name = pick_str(data, &["name", "profile.name", "displayName"]); - let profile_url = pick_str( - data, - &["display_url", "profileUrl", "profile_url", "profile.url"], - ); - - let profile = ProviderUserProfile { - toolkit: "gmail".to_string(), - connection_id: ctx.connection_id.clone(), - display_name, - email, - username: None, - avatar_url: None, - profile_url, - extras: data.clone(), - }; - let has_email = profile.email.is_some(); - let email_domain = profile - .email - .as_deref() - .and_then(|e| e.split('@').nth(1)) - .map(|d| d.to_string()); - tracing::info!( - connection_id = ?profile.connection_id, - has_email, - email_domain = ?email_domain, - "[composio:gmail] fetched user profile" - ); - Ok(profile) - } - - /// Incremental sync via the generic - /// `orchestrator`: - /// pagination, dedup, the `max_items` cap, and cursor handling live in - /// `run_sync`; the Gmail-specific primitives — the account-email preamble, - /// server-side `after:` depth window, adaptive page ceiling, all-synced - /// stop, and batch ingest — live in `super::source`. - async fn on_trigger( - &self, - ctx: &ProviderContext, - trigger: &str, - _payload: &Value, - ) -> Result<(), String> { - tracing::info!( - connection_id = ?ctx.connection_id, - trigger = %trigger, - "[composio:gmail] on_trigger" - ); - - if trigger.eq_ignore_ascii_case("GMAIL_NEW_GMAIL_MESSAGE") - || trigger.eq_ignore_ascii_case("GMAIL_NEW_MESSAGE") - { - let Some(connection_id) = ctx.connection_id.as_deref() else { - return Err("[composio:gmail] trigger missing connection_id".to_string()); - }; - if let Err(e) = crate::sync::pipelines::host::run_composio_connection( - "gmail", - connection_id, - ctx.config.as_ref(), - None, - None, - ) - .await - { - tracing::warn!( - error = %e, - "[composio:gmail] trigger-driven sync failed (non-fatal)" - ); - } - } - Ok(()) - } -} - -// Message fetching (the `GMAIL_FETCH_EMAILS` action, the search query, the -// `max_items` cap math and the `sync_depth_days` `after:` floor) is owned -// by `crate::sync::pipelines::composio::GmailSyncPipeline`. What stays here is the -// host-side provider surface: profile lookup and trigger dispatch. diff --git a/crates/tinymemory-core/src/sync/composio/providers/gmail/tests.rs b/crates/tinymemory-core/src/sync/composio/providers/gmail/tests.rs deleted file mode 100644 index 4c6554b8..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/gmail/tests.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! Host-owned Gmail provider surface tests. -//! -//! Pagination, cursor, envelope parsing, and ingest behavior are owned and -//! tested by `crate::sync::pipelines::composio::GmailSyncPipeline`. - -use super::GmailProvider; -use crate::sync::composio::providers::ComposioProvider; - -#[test] -fn provider_metadata_is_stable() { - let provider = GmailProvider::new(); - assert_eq!(provider.toolkit_slug(), "gmail"); - assert_eq!(provider.sync_interval_secs(), Some(15 * 60)); -} - -#[test] -fn default_impl_matches_new() { - let _new = GmailProvider::new(); - let _default = ::default(); -} - -#[test] -fn provider_source_does_not_restrict_to_inbox() { - let source = include_str!("provider.rs"); - assert!( - !source.contains("\"in:inbox"), - "provider query must not exclude sent mail" - ); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/gmail/tools.rs b/crates/tinymemory-core/src/sync/composio/providers/gmail/tools.rs deleted file mode 100644 index 477161ba..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/gmail/tools.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! The curated `gmail` catalog, re-exported at its historical path. -//! -//! The table moved to [`tinymemory_api::composio::catalogs::gmail`] with every -//! other catalog — see [`super::super::catalogs`] for why. - -pub use tinymemory_api::composio::catalogs::gmail::GMAIL_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/helpers.rs b/crates/tinymemory-core/src/sync/composio/providers/helpers.rs deleted file mode 100644 index e0358e20..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/helpers.rs +++ /dev/null @@ -1,56 +0,0 @@ -//! Shared helpers for Composio provider implementations. -//! -//! `pick_str` used to live here. It is a provider payload normaliser, so it -//! moved to `tinymemory_sync::helpers` -//! and is re-exported from this module's parent. The helpers that remain are -//! request-building rather than normalisation, and stay host-side. - -use tinymemory_sync::helpers::pick_str; - -/// Shallow-merge an `extra` JSON object into a (mutable) action-args -/// object. Only object-typed extras are merged; non-object `extra` -/// values are ignored. Backs the `task_sources` advanced free-form -/// filter escape hatch — provider `fetch_tasks` impls call this to fold -/// user-supplied provider-native query fragments into their request -/// arguments. -pub(crate) fn merge_extra(args: &mut serde_json::Value, extra: &serde_json::Value) { - if let (Some(args_obj), Some(extra_obj)) = (args.as_object_mut(), extra.as_object()) { - for (k, v) in extra_obj { - args_obj.insert(k.clone(), v.clone()); - } - } -} - -/// Resolve the first array found among `array_paths` (dotted object -/// paths), then return the first non-empty string at one of `fields` -/// on that array's first element. Complements [`pick_str`], which -/// cannot index into arrays. Used to pull e.g. the first assignee's -/// username out of an `assignees` array. -pub(crate) fn first_array_str( - value: &serde_json::Value, - array_paths: &[&str], - fields: &[&str], -) -> Option { - for path in array_paths { - let mut cur = value; - let mut ok = true; - for segment in path.split('.') { - match cur.get(segment) { - Some(next) => cur = next, - None => { - ok = false; - break; - } - } - } - if !ok { - continue; - } - if let Some(first) = cur.as_array().and_then(|a| a.first()) { - if let Some(found) = pick_str(first, fields) { - return Some(found); - } - } - } - None -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/linear/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/linear/mod.rs deleted file mode 100644 index 7ccce87c..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/linear/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Linear Composio provider — incremental Memory Tree ingest for -//! issues assigned to the connected user. -//! -//! Issue: #2400. - -// The payload normalisers moved to tinycortex (they are pure Value -// transforms, i.e. driver-side). Aliased under the old module name so -// every `normalization::extract_*` call site below stays unchanged. -use tinymemory_sync::linear as normalization; -mod provider; -#[cfg(test)] -mod tests; -pub mod tools; - -pub use provider::LinearProvider; -pub use tools::LINEAR_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/linear/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/linear/provider.rs deleted file mode 100644 index 2863c8d0..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/linear/provider.rs +++ /dev/null @@ -1,241 +0,0 @@ -//! Linear provider — incremental sync of issues assigned to the -//! authenticated user, with per-issue memory_tree ingest. -//! -//! On each sync pass: -//! -//! 1. Load persistent [`SyncState`] from the KV store. -//! 2. Check the daily request budget — bail early if exhausted. -//! 3. Resolve the viewer ID via `LINEAR_LIST_LINEAR_USERS { isMe: true }`. -//! 4. Page through `LINEAR_LIST_LINEAR_ISSUES` filtered to the viewer as -//! assignee, ordered by `updatedAt` descending. Stop early once we hit -//! issues older than the cursor or a page without a next-page cursor. -//! 5. For each issue, ingest into memory_tree if it's new *or* edited -//! since the last sync. -//! 6. Advance the cursor to the newest `updatedAt` seen and save. -//! -//! Privacy posture: we only pull issues the user is assigned to, never -//! the whole workspace's issue graph. This mirrors the -//! "fetch-what-the-user-sees" model `gmail` / `notion` already follow -//! and avoids accidentally ingesting other teammates' private issues. - -use async_trait::async_trait; -use serde_json::json; - -use super::normalization; -use crate::sync::composio::providers::{ - merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, - NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind, -}; - -pub(super) const ACTION_LIST_USERS: &str = "LINEAR_LIST_LINEAR_USERS"; -pub(super) const ACTION_LIST_ISSUES: &str = "LINEAR_LIST_LINEAR_ISSUES"; - -/// Paths for extracting a Linear issue's unique ID. -pub(super) const ISSUE_ID_PATHS: &[&str] = &["id", "data.id", "identifier", "data.identifier"]; - -pub struct LinearProvider; - -impl LinearProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for LinearProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for LinearProvider { - fn toolkit_slug(&self) -> &'static str { - "linear" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(super::tools::LINEAR_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - // 30 minutes — same cadence as ClickUp/Notion. Linear issues change - // more slowly than chat but faster than email. - Some(resolve_sync_interval_secs("linear", 30 * 60)) - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:linear] fetch_user_profile via {ACTION_LIST_USERS}" - ); - - let resp = ctx - .execute(ACTION_LIST_USERS, Some(json!({ "isMe": true }))) - .await - .map_err(|e| format!("[composio:linear] {ACTION_LIST_USERS} failed: {e:#}"))?; - - if !resp.successful { - let err = resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!("[composio:linear] {ACTION_LIST_USERS}: {err}")); - } - - let data = &resp.data; - let viewer = normalization::extract_viewer(data); - let viewer_ref = viewer.as_ref().unwrap_or(data); - - let display_name = pick_str(viewer_ref, &["name", "data.name", "displayName"]); - let email = pick_str(viewer_ref, &["email", "data.email"]); - let username = pick_str(viewer_ref, &["id", "data.id"]); - let avatar_url = pick_str(viewer_ref, &["avatarUrl", "data.avatarUrl"]); - let profile_url = pick_str(viewer_ref, &["url", "data.url"]); - - Ok(ProviderUserProfile { - toolkit: "linear".to_string(), - connection_id: ctx.connection_id.clone(), - display_name, - email, - username, - avatar_url, - profile_url, - extras: data.clone(), - }) - } - - /// Incremental sync via the generic - /// `orchestrator`: - /// viewer resolution, pagination, dedup, the `max_items` cap, the - /// `sync_depth_days` window, and cursor handling live in `run_sync`; the - /// Linear-specific primitives live in `super::source`. - async fn fetch_tasks( - &self, - ctx: &ProviderContext, - filter: &TaskFetchFilter, - ) -> Result, String> { - let max = filter.effective_max(); - tracing::debug!( - connection_id = ?ctx.connection_id, - max, - team_id = ?filter.team_id, - assignee_is_me = filter.assignee_is_me, - "[composio:linear] fetch_tasks" - ); - - let mut args = json!({ - "first": max.min(100) as u64, - "orderBy": "updatedAt", - }); - if filter.assignee_is_me { - let resp = ctx - .execute(ACTION_LIST_USERS, Some(json!({ "isMe": true }))) - .await - .map_err(|e| format!("[composio:linear] {ACTION_LIST_USERS}: {e:#}"))?; - // Fail closed: a failed viewer lookup must not silently widen - // the query beyond "assigned to me". - if !resp.successful { - return Err(format!( - "[composio:linear] {ACTION_LIST_USERS}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - let viewer_id = normalization::extract_viewer_id(&resp.data).ok_or_else(|| { - "[composio:linear] LINEAR_LIST_LINEAR_USERS returned no viewer id".to_string() - })?; - args["assigneeId"] = json!(viewer_id); - } - if let Some(team) = filter - .team_id - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()) - { - args["teamId"] = json!(team); - } - merge_extra(&mut args, &filter.extra); - - let resp = ctx - .execute(ACTION_LIST_ISSUES, Some(args)) - .await - .map_err(|e| format!("[composio:linear] {ACTION_LIST_ISSUES}: {e:#}"))?; - if !resp.successful { - return Err(format!( - "[composio:linear] {ACTION_LIST_ISSUES}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - - let want_state = filter - .state - .as_deref() - .map(|s| s.trim().to_ascii_lowercase()) - .filter(|s| !s.is_empty()); - - let mut out: Vec = Vec::new(); - for issue in normalization::extract_issues(&resp.data) { - if out.len() >= max { - break; - } - let Some(nt) = normalize_linear_issue(&issue) else { - continue; - }; - if let Some(ref want) = want_state { - let matches = nt - .status - .as_deref() - .map(|s| s.to_ascii_lowercase() == *want) - .unwrap_or(false); - if !matches { - continue; - } - } - out.push(nt); - } - tracing::debug!(count = out.len(), "[composio:linear] fetch_tasks complete"); - Ok(out) - } -} - -/// Map a raw Linear issue payload into a [`NormalizedTask`]. -pub(super) fn normalize_linear_issue(issue: &serde_json::Value) -> Option { - let external_id = pick_str(issue, ISSUE_ID_PATHS)?; - let title = normalization::extract_issue_title(issue) - .unwrap_or_else(|| format!("Linear issue {external_id}")); - Some(NormalizedTask { - external_id, - source_id: String::new(), - provider: "linear".to_string(), - kind: TaskKind::Generic, - title, - body: pick_str(issue, &["description", "data.description"]), - url: pick_str(issue, &["url", "data.url"]), - status: pick_str(issue, &["state.name", "data.state.name", "state.type"]), - assignee: pick_str(issue, &["assignee.name", "data.assignee.name"]), - due: pick_str(issue, &["dueDate", "data.dueDate"]), - labels: extract_linear_labels(issue), - priority: pick_str(issue, &["priorityLabel", "data.priorityLabel"]), - updated_at: normalization::extract_issue_updated(issue), - raw: issue.clone(), - }) -} - -/// Extract label names from a Linear issue (`labels.nodes[].name`). -pub(super) fn extract_linear_labels(issue: &serde_json::Value) -> Vec { - let arr = issue - .get("labels") - .or_else(|| issue.get("data").and_then(|d| d.get("labels"))) - .and_then(|l| l.get("nodes")) - .and_then(|v| v.as_array()); - match arr { - Some(items) => items - .iter() - .filter_map(|l| l.get("name").and_then(|n| n.as_str())) - .map(|s| s.to_string()) - .collect(), - None => Vec::new(), - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/linear/tests.rs b/crates/tinymemory-core/src/sync/composio/providers/linear/tests.rs deleted file mode 100644 index f1ac7cb1..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/linear/tests.rs +++ /dev/null @@ -1,226 +0,0 @@ -//! Unit tests for the Linear provider. - -use super::normalization::{ - extract_issue_title, extract_issue_updated, extract_issues, extract_pagination_cursor, - extract_viewer, extract_viewer_id, -}; -use super::LinearProvider; -use crate::sync::composio::providers::{ - ComposioProvider, ComposioUsageHandle, ProviderContext, TaskFetchFilter, -}; -use serde_json::json; -use std::sync::Arc; - -fn context() -> ProviderContext { - ProviderContext { - config: Arc::new(tinymemory_api::host::test_support::TestHostConfig::default()) - as Arc, - toolkit: "linear".into(), - connection_id: Some("connection-1".into()), - usage: ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - } -} - -// ── extract_issues ─────────────────────────────────────────────────── - -#[test] -fn extract_issues_walks_common_shapes() { - let v1 = json!({ "data": { "nodes": [{"id": "i1"}] } }); - let v2 = json!({ "nodes": [{"id": "i2"}, {"id": "i3"}] }); - let v3 = json!({ "data": { "issues": { "nodes": [{"id": "i4"}] } } }); - let v4 = json!({ "foo": "bar" }); - assert_eq!(extract_issues(&v1).len(), 1); - assert_eq!(extract_issues(&v2).len(), 2); - assert_eq!(extract_issues(&v3).len(), 1); - assert_eq!(extract_issues(&v4).len(), 0); -} - -// ── extract_issue_title ────────────────────────────────────────────── - -#[test] -fn extract_issue_title_finds_title_field() { - let issue = json!({ "id": "i1", "title": "Fix the login bug" }); - assert_eq!( - extract_issue_title(&issue), - Some("Fix the login bug".into()) - ); -} - -#[test] -fn extract_issue_title_falls_back_to_wrapped_data() { - let issue = json!({ "data": { "title": "Wrapped issue" } }); - assert_eq!(extract_issue_title(&issue), Some("Wrapped issue".into())); -} - -#[test] -fn extract_issue_title_falls_back_to_identifier() { - let issue = json!({ "identifier": "ENG-99" }); - assert_eq!(extract_issue_title(&issue), Some("ENG-99".into())); -} - -// ── extract_issue_updated ──────────────────────────────────────────── - -#[test] -fn extract_issue_updated_handles_camel_case() { - let issue = json!({ "updatedAt": "2026-03-01T12:00:00.000Z" }); - assert_eq!( - extract_issue_updated(&issue), - Some("2026-03-01T12:00:00.000Z".to_string()) - ); -} - -#[test] -fn extract_issue_updated_handles_wrapped_data() { - let issue = json!({ "data": { "updatedAt": "2026-01-15T08:30:00.000Z" } }); - assert_eq!( - extract_issue_updated(&issue), - Some("2026-01-15T08:30:00.000Z".to_string()) - ); -} - -// ── extract_viewer ─────────────────────────────────────────────────── - -#[test] -fn extract_viewer_finds_first_node() { - let data = json!({ "data": { "nodes": [{ "id": "usr_1", "email": "a@b.com" }] } }); - let v = extract_viewer(&data).expect("viewer found"); - assert_eq!(v["id"], "usr_1"); -} - -#[test] -fn extract_viewer_from_top_level_nodes() { - let data = json!({ "nodes": [{ "id": "usr_2" }] }); - let v = extract_viewer(&data).expect("viewer found"); - assert_eq!(v["id"], "usr_2"); -} - -#[test] -fn extract_viewer_fallback_direct_object() { - let data = json!({ "id": "usr_direct", "name": "Alice" }); - let v = extract_viewer(&data).expect("viewer found"); - assert_eq!(v["id"], "usr_direct"); -} - -#[test] -fn extract_viewer_returns_none_when_absent() { - let data = json!({ "foo": "bar" }); - assert!(extract_viewer(&data).is_none()); -} - -// ── extract_pagination_cursor ──────────────────────────────────────── - -#[test] -fn extract_pagination_cursor_returns_cursor_on_has_next_page() { - let data = json!({ - "data": { - "pageInfo": { "hasNextPage": true, "endCursor": "abc123" } - } - }); - assert_eq!(extract_pagination_cursor(&data), Some("abc123".to_string())); -} - -#[test] -fn extract_pagination_cursor_returns_none_on_last_page() { - let data = json!({ - "pageInfo": { "hasNextPage": false, "endCursor": "xyz" } - }); - assert!(extract_pagination_cursor(&data).is_none()); -} - -#[test] -fn extract_pagination_cursor_returns_none_when_absent() { - let data = json!({ "nodes": [{"id": "i1"}] }); - assert!(extract_pagination_cursor(&data).is_none()); -} - -// ── extract_viewer_id ──────────────────────────────────────────────── - -#[test] -fn extract_viewer_id_from_data_nodes() { - let data = json!({ "data": { "nodes": [{ "id": "usr_abc" }] } }); - assert_eq!(extract_viewer_id(&data), Some("usr_abc".to_string())); -} - -#[test] -fn extract_viewer_id_returns_none_when_absent() { - let data = json!({ "foo": "bar" }); - assert!(extract_viewer_id(&data).is_none()); -} - -// ── provider metadata ──────────────────────────────────────────────── - -#[test] -fn provider_metadata_is_stable() { - let p = LinearProvider::new(); - assert_eq!(p.toolkit_slug(), "linear"); - assert_eq!(p.sync_interval_secs(), Some(30 * 60)); - assert!(p.curated_tools().is_some()); -} - -#[test] -fn curated_tools_contains_core_sync_surface() { - let p = LinearProvider::new(); - let curated = p.curated_tools().expect("LINEAR_CURATED is registered"); - let slugs: Vec<&str> = curated.iter().map(|t| t.slug).collect(); - assert!( - slugs.contains(&"LINEAR_LIST_LINEAR_USERS"), - "LINEAR_LIST_LINEAR_USERS must be in curated catalog" - ); - assert!( - slugs.contains(&"LINEAR_LIST_LINEAR_ISSUES"), - "LINEAR_LIST_LINEAR_ISSUES must be in curated catalog" - ); -} - -#[test] -fn default_impl_matches_new() { - let a = LinearProvider::new(); - let b = ::default(); - assert_eq!(a.toolkit_slug(), b.toolkit_slug()); - assert_eq!(a.sync_interval_secs(), b.sync_interval_secs()); - assert_eq!( - a.curated_tools().map(<[_]>::len), - b.curated_tools().map(<[_]>::len), - ); -} - -#[tokio::test] -async fn provider_calls_fail_contextually_without_a_composio_host() { - let provider = LinearProvider::new(); - let profile = provider.fetch_user_profile(&context()).await.unwrap_err(); - assert!(profile.contains("LINEAR_LIST_LINEAR_USERS")); - let tasks = provider - .fetch_tasks( - &context(), - &TaskFetchFilter { - assignee_is_me: false, - team_id: Some(" team-1 ".into()), - extra: json!({"includeArchived": false}), - max: 3, - ..Default::default() - }, - ) - .await - .unwrap_err(); - assert!(tasks.contains("LINEAR_LIST_LINEAR_ISSUES")); -} - -#[test] -fn issue_normalization_covers_fallbacks_labels_and_missing_id() { - use super::provider::{extract_linear_labels, normalize_linear_issue}; - - assert!(normalize_linear_issue(&json!({"title": "missing id"})).is_none()); - let task = normalize_linear_issue(&json!({ - "identifier": "ENG-7", - "description": "body", - "state": {"name": "Started"}, - "labels": {"nodes": [{"name": "bug"}, {}, {"name": "urgent"}]} - })) - .unwrap(); - assert_eq!(task.external_id, "ENG-7"); - assert_eq!(task.title, "ENG-7"); - assert_eq!(task.labels, vec!["bug", "urgent"]); - assert!(extract_linear_labels(&json!({})).is_empty()); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/linear/tools.rs b/crates/tinymemory-core/src/sync/composio/providers/linear/tools.rs deleted file mode 100644 index 610bac00..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/linear/tools.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! The curated `linear` catalog, re-exported at its historical path. -//! -//! The table moved to [`tinymemory_api::composio::catalogs::linear`] with every -//! other catalog — see [`super::super::catalogs`] for why. - -pub use tinymemory_api::composio::catalogs::linear::LINEAR_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/mod.rs deleted file mode 100644 index 2876da57..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/mod.rs +++ /dev/null @@ -1,105 +0,0 @@ -//! Provider-specific code for Composio toolkits. -//! -//! Each Composio toolkit (gmail, notion, slack, …) can register a -//! [`ComposioProvider`] implementation that knows how to: -//! -//! * Fetch a normalized **user profile** for a connected account. -//! * Run an **initial / periodic sync** that pulls fresh data from the -//! upstream service via the backend-proxied -//! `ComposioClient`. -//! * React to **trigger webhooks** that arrive over the -//! `composio:trigger` Socket.IO bridge. -//! * React to **OAuth handoff completion** so the very first sync can -//! run as soon as a user connects an account. -//! -//! Providers are pure Rust — there is no JS sandbox involved. They are -//! the native counterpart to the QuickJS skill bundles in -//! `tinyhumansai/openhuman-skills`, but specialized for Composio's API -//! surface and run inside the core process directly. -//! -//! ## Registry & dispatch -//! -//! The [`registry`] module owns a process-global `HashMap>`. The composio event bus subscriber -//! (`super::bus::ComposioTriggerSubscriber`) and the periodic sync -//! task both look up providers by toolkit slug and call into them. -//! -//! ## Why a trait, not a giant `match` -//! -//! Each provider has provider-specific shapes (gmail returns -//! emailAddress + messagesTotal, notion returns workspaces + pages, …) -//! and a different idea of what "sync" means. A trait keeps each -//! provider's implementation isolated, individually testable, and -//! easy to add without touching the dispatch layer. - -mod descriptions; -pub(crate) mod helpers; -mod scope_lookup; -pub mod tool_scope; -mod traits; -mod types; -pub mod user_scopes; - -pub mod catalogs; -mod catalogs_compat; -pub mod clickup; -pub mod github; -pub mod gmail; -pub mod linear; -pub mod notion; -pub mod profile; -pub mod profile_md; -pub mod registry; -pub mod slack; -pub mod sync_state; - -// The capability matrix, the curated-catalog lookup and the visibility gate -// all moved to the contract crate (OpenHuman#5560) — see [`catalogs`] and -// [`tinymemory_api::host::composio::capability_matrix`]. They are re-exported -// at the bottom of this file, so every historical `providers::…` path keeps -// resolving and the wire surface is unchanged. - -/// All toolkit slugs that have a curated agent-ready catalog. -/// -/// Source of truth for the UI "preview / agent integration coming soon" badge: -/// any connected toolkit whose slug is NOT in this list can be authorized but -/// lacks a curated tool surface, so the agent can't use it productively. -/// -/// Defined in the contract crate (#5560) because the *host* renders that badge -/// and reaching this crate to spell the list is one of the compile-time links -/// the issue removes. Re-exported here so every historical -/// `providers::agent_ready_toolkits()` call keeps resolving. -pub use tinymemory_api::composio::scopes::agent_ready_toolkits; - -// Historical per-category module paths (`providers::catalogs_business::…`, -// pre-#5560). See [`catalogs_compat`] for why these stay as thin re-exports -// rather than a semver bump. -pub use catalogs_compat::{ - catalogs_business, catalogs_google, catalogs_messaging, catalogs_microsoft, - catalogs_productivity, catalogs_social_media, -}; - -pub use descriptions::toolkit_description; -pub(crate) use helpers::{first_array_str, merge_extra}; -pub use tinymemory_api::composio::catalogs::{catalog_for_toolkit, is_action_visible_with_pref}; -pub use tinymemory_api::host::composio::capability_matrix; -// `pick_str` is a provider payload normaliser and lives in tinycortex; it is -// re-exported here so the ~40 in-tree call sites keep resolving unchanged. -// Note this is deliberately NOT `providers::common::pick_str`, which coerces -// numbers to strings — see the doc comments on both definitions. -pub use registry::{ - all_providers, get_provider, init_default_providers, register_provider, ProviderArc, -}; -pub use scope_lookup::{curated_scope_for, toolkit_has_scope}; -pub(crate) use tinymemory_sync::helpers::pick_str; -pub use tool_scope::{classify_unknown, find_curated, toolkit_from_slug, CuratedTool, ToolScope}; -pub use traits::{resolve_sync_interval_secs, sync_interval_env_var, ComposioProvider}; -pub use types::{ - ComposioUsage, ComposioUsageHandle, GithubFetchMode, NormalizedTask, ProviderContext, - ProviderUserProfile, SyncOutcome, SyncReason, TaskContainer, TaskFetchFilter, TaskKind, -}; -pub use user_scopes::{load_or_default as load_user_scope_or_default, UserScopePref}; - -#[cfg(test)] -#[path = "providers_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/notion/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/notion/mod.rs deleted file mode 100644 index 138fe603..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/notion/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -// The payload normalisers moved to tinycortex (they are pure Value -// transforms, i.e. driver-side). Aliased under the old module name so -// every `normalization::extract_*` call site below stays unchanged. -use tinymemory_sync::notion as normalization; -mod provider; -#[cfg(test)] -mod tests; -pub mod tools; - -pub use provider::NotionProvider; -pub use tools::NOTION_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/notion/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/notion/provider.rs deleted file mode 100644 index ac03bc2a..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/notion/provider.rs +++ /dev/null @@ -1,430 +0,0 @@ -//! Notion provider — incremental sync with per-item persistence. -//! -//! On each sync pass: -//! -//! 1. Load persistent [`SyncState`] from the KV store. -//! 2. Check the daily request budget — bail early if exhausted. -//! 3. Fetch a page of recently edited pages via `NOTION_FETCH_DATA`, -//! sorted by `last_edited_time` descending. When a cursor exists -//! we can stop as soon as we see pages older than the cursor. -//! 4. Deduplicate against `synced_ids` in the state. Pages that have -//! been *edited* since their last sync are re-persisted (the cursor -//! is based on `last_edited_time`, so an edited page appears again). -//! 5. Persist each **new or updated** page as its own memory document. -//! 6. Paginate (up to budget) until no more results or all items in the -//! page are older than the cursor. -//! 7. Advance the cursor and save state. - -use async_trait::async_trait; -use serde_json::{json, Value}; - -use super::normalization; -use crate::sync::composio::providers::{ - first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, - CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskContainer, - TaskFetchFilter, TaskKind, -}; - -pub(crate) const ACTION_GET_ABOUT_ME: &str = "NOTION_GET_ABOUT_ME"; -pub(crate) const ACTION_FETCH_DATA: &str = "NOTION_FETCH_DATA"; -pub(crate) const ACTION_QUERY_DATABASE: &str = "NOTION_QUERY_DATABASE"; - -/// Arguments for the recent-pages `NOTION_FETCH_DATA` fallback `fetch_tasks` -/// uses when a task source names no database. -/// -/// Extracted from the call site so the request shape is assertable: the -/// `ComposioHost` seam is a process-global stub in this crate's tests, so the -/// outgoing arguments are otherwise unobservable and only the error string can -/// be checked. -/// -/// `fetch_type` is required by Composio and its omission is what -/// `Invalid request data provided - Following fields are missing: {'fetch_type'}` -/// reports. Unconditionally `"pages"` for the same reason the periodic sync -/// pipeline sends that value: this arm hardcodes `filter: {value: "page"}`, so -/// no other value is valid here. -/// -/// OpenHuman injects the field for this action in `ensure_notion_fetch_type`, -/// but only on the Backend arm of its Composio client; the Direct (BYOK) arm -/// passes arguments through untouched, so this path cannot rely on it. -pub(crate) fn fetch_data_args(max: usize) -> Value { - json!({ - "fetch_type": "pages", - "page_size": max.min(100) as u32, - "filter": { "value": "page", "property": "object" }, - "sort": { "direction": "descending", "timestamp": "last_edited_time" }, - }) -} -pub(crate) const ACTION_SEARCH_NOTION_PAGE: &str = "NOTION_SEARCH_NOTION_PAGE"; - -/// Paths for extracting a page's unique ID. -pub(crate) const PAGE_ID_PATHS: &[&str] = &["id", "data.id", "pageId", "data.pageId"]; - -/// Paths for extracting the `last_edited_time` used as sync cursor. -pub(crate) const PAGE_EDITED_PATHS: &[&str] = &[ - "last_edited_time", - "data.last_edited_time", - "lastEditedTime", - "data.lastEditedTime", -]; - -pub struct NotionProvider; - -impl NotionProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for NotionProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for NotionProvider { - fn toolkit_slug(&self) -> &'static str { - "notion" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(super::tools::NOTION_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - Some(resolve_sync_interval_secs("notion", 30 * 60)) - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:notion] fetch_user_profile via {ACTION_GET_ABOUT_ME}" - ); - - let resp = ctx - .execute(ACTION_GET_ABOUT_ME, Some(json!({}))) - .await - .map_err(|e| format!("[composio:notion] {ACTION_GET_ABOUT_ME} failed: {e:#}"))?; - - if !resp.successful { - let err = resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!("[composio:notion] {ACTION_GET_ABOUT_ME}: {err}")); - } - - // `data` is already the inner Composio response payload — paths - // here are relative to it. For bot-token connections the - // top-level `name` is the *integration's* name (e.g. "Composio"), - // and the actual owning user lives at `bot.owner.user.*`. Probe - // the bot-owner paths first so identity reflects the user (#1365). - let data = &resp.data; - let display_name = pick_str(data, &["bot.owner.user.name", "user.name", "name"]); - let email = pick_str( - data, - &[ - "bot.owner.user.person.email", - "user.person.email", - "person.email", - "email", - ], - ); - let username = pick_str(data, &["bot.owner.user.id", "user.id", "id"]); - let avatar_url = pick_str( - data, - &["bot.owner.user.avatar_url", "user.avatar_url", "avatar_url"], - ); - let profile_url = pick_str(data, &["url", "profile_url", "profile.url"]); - - Ok(ProviderUserProfile { - toolkit: "notion".to_string(), - connection_id: ctx.connection_id.clone(), - display_name, - email, - username, - avatar_url, - profile_url, - extras: data.clone(), - }) - } - - /// Incremental sync. Notion was the first provider migrated to the generic - /// `orchestrator`: - /// the per-item loop, dedup, `max_items` cap, `sync_depth_days` window, and - /// cursor handling all live in `run_sync`; the Notion-specific primitives - /// (page fetch, dedup key, body fetch, ingest) live in `super::source`. - async fn fetch_tasks( - &self, - ctx: &ProviderContext, - filter: &TaskFetchFilter, - ) -> Result, String> { - let max = filter.effective_max(); - let database_id = filter - .database_id - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - - tracing::debug!( - connection_id = ?ctx.connection_id, - max, - has_database = database_id.is_some(), - "[composio:notion] fetch_tasks" - ); - - // A configured board (database) uses NOTION_QUERY_DATABASE; - // otherwise fall back to NOTION_FETCH_DATA (recent pages), the - // same action the periodic sync uses. - let (action, mut args) = match database_id { - Some(db) => ( - ACTION_QUERY_DATABASE, - json!({ - "database_id": db, - "page_size": max.min(100) as u32, - "sorts": [ { "timestamp": "last_edited_time", "direction": "descending" } ], - }), - ), - None => (ACTION_FETCH_DATA, fetch_data_args(max)), - }; - merge_extra(&mut args, &filter.extra); - - let resp = ctx - .execute(action, Some(args)) - .await - .map_err(|e| format!("[composio:notion] {action}: {e:#}"))?; - if !resp.successful { - return Err(format!( - "[composio:notion] {action}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - - // Optional client-side status filter — Notion status properties - // are user-defined, so we match on the normalized status rather - // than building a server-side property filter. - let want_status = filter - .status - .as_deref() - .map(|s| s.trim().to_ascii_lowercase()) - .filter(|s| !s.is_empty()); - - let mut out: Vec = Vec::new(); - for page in normalization::extract_results(&resp.data) { - if out.len() >= max { - break; - } - let Some(nt) = normalize_notion_page(&page) else { - continue; - }; - if let Some(ref want) = want_status { - let matches = nt - .status - .as_deref() - .map(|s| s.to_ascii_lowercase() == *want) - .unwrap_or(false); - if !matches { - continue; - } - } - out.push(nt); - } - tracing::debug!(count = out.len(), "[composio:notion] fetch_tasks complete"); - Ok(out) - } - - /// List the Notion databases (tables) the connected integration can see, - /// via `NOTION_SEARCH_NOTION_PAGE` filtered to database objects, so the - /// task-source UI can offer a picker for `database_id`. Only databases the - /// integration has been *shared with* in Notion are returned. - async fn list_databases(&self, ctx: &ProviderContext) -> Result, String> { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:notion] list_databases via {ACTION_SEARCH_NOTION_PAGE}" - ); - // Composio's NOTION_SEARCH_NOTION_PAGE *flattens* Notion's native - // `filter: { value, property }` into top-level `filter_value` / - // `filter_property` params and silently drops the nested form (which - // returned only pages). We send the flat params here; the nested - // `filter` is kept too as a belt-and-braces hint for any variant that - // honours it, and the parser still drops any stray `page` items. - let args = json!({ - "query": "", - "filter_value": "database", - "filter_property": "object", - "filter": { "value": "database", "property": "object" }, - "page_size": 100, - }); - let resp = ctx - .execute(ACTION_SEARCH_NOTION_PAGE, Some(args)) - .await - .map_err(|e| format!("[composio:notion] {ACTION_SEARCH_NOTION_PAGE}: {e:#}"))?; - if !resp.successful { - return Err(format!( - "[composio:notion] {ACTION_SEARCH_NOTION_PAGE}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - - tracing::info!( - successful = resp.successful, - data_is_array = resp.data.is_array(), - data_keys = ?resp.data.as_object().map(|o| o.keys().cloned().collect::>()), - "[composio:notion] list_databases raw response shape" - ); - let out = parse_database_results(&resp.data); - tracing::info!( - count = out.len(), - "[composio:notion] list_databases complete" - ); - Ok(out) - } - - async fn on_trigger( - &self, - ctx: &ProviderContext, - trigger: &str, - _payload: &Value, - ) -> Result<(), String> { - tracing::info!( - connection_id = ?ctx.connection_id, - trigger = %trigger, - "[composio:notion] on_trigger" - ); - let Some(connection_id) = ctx.connection_id.as_deref() else { - return Err("[composio:notion] trigger missing connection_id".to_string()); - }; - if let Err(e) = crate::sync::pipelines::host::run_composio_connection( - "notion", - connection_id, - ctx.config.as_ref(), - None, - None, - ) - .await - { - tracing::warn!( - error = %e, - "[composio:notion] trigger-driven sync failed (non-fatal)" - ); - } - Ok(()) - } -} - -/// Map a raw Notion page payload into a [`NormalizedTask`]. -/// -/// Notion databases are user-defined, so property extraction is -/// best-effort against common property names (`Status`, `Assignee`, -/// `Due`). Anything unmatched is simply left `None` — the raw payload is -/// preserved for enrichment. -pub(super) fn normalize_notion_page(page: &serde_json::Value) -> Option { - let external_id = pick_str(page, PAGE_ID_PATHS)?; - let title = normalization::extract_page_title(page) - .unwrap_or_else(|| format!("Notion page {external_id}")); - Some(NormalizedTask { - external_id, - source_id: String::new(), - provider: "notion".to_string(), - kind: TaskKind::Generic, - title, - body: None, - url: pick_str(page, &["url", "data.url"]), - status: pick_str( - page, - &[ - "properties.Status.status.name", - "properties.Status.select.name", - "data.properties.Status.status.name", - ], - ), - assignee: first_array_str( - page, - &[ - "properties.Assignee.people", - "data.properties.Assignee.people", - ], - &["name"], - ), - due: pick_str( - page, - &[ - "properties.Due.date.start", - "data.properties.Due.date.start", - ], - ), - labels: Vec::new(), - priority: pick_str( - page, - &[ - "properties.Priority.select.name", - "data.properties.Priority.select.name", - ], - ), - updated_at: pick_str(page, PAGE_EDITED_PATHS), - raw: page.clone(), - }) -} - -/// Map a `NOTION_SEARCH_NOTION_PAGE` response into the database containers -/// the UI picker needs. -/// -/// We send a server-side `object: database` filter, so the response is -/// already scoped — we therefore *trust* it and only drop items explicitly -/// typed as `page`. This is intentional: Composio's response items don't -/// always carry a top-level `object` field, and an over-strict -/// "keep only object==database" check silently dropped every database. -/// Pure (no I/O) so it is unit-testable. -pub(super) fn parse_database_results(data: &serde_json::Value) -> Vec { - let results = normalization::extract_results(data); - let mut kinds: std::collections::BTreeMap = std::collections::BTreeMap::new(); - let mut out: Vec = Vec::new(); - for item in &results { - let object = pick_str(item, &["object", "data.object"]); - *kinds - .entry(object.clone().unwrap_or_else(|| "".to_string())) - .or_default() += 1; - // Trust the server-side database filter: keep databases / data_sources - // *and* objectless items; only drop items explicitly typed as pages. - if object.as_deref() == Some("page") { - continue; - } - let Some(id) = pick_str(item, PAGE_ID_PATHS) else { - continue; - }; - let title = extract_database_title(item).unwrap_or_else(|| format!("Notion database {id}")); - out.push(TaskContainer { id, title }); - } - tracing::info!( - raw = results.len(), - kept = out.len(), - object_kinds = ?kinds, - "[composio:notion] parse_database_results" - ); - out -} - -/// Extract a Notion database's display title from its top-level `title` -/// rich-text array (`title[].plain_text`), tolerant of the Composio `data` -/// wrapper. Returns `None` for an untitled / shapeless database. -fn extract_database_title(db: &serde_json::Value) -> Option { - let arr = db - .get("title") - .or_else(|| db.get("data").and_then(|d| d.get("title"))) - .and_then(|v| v.as_array())?; - let text: String = arr - .iter() - .filter_map(|t| { - t.get("plain_text").and_then(|p| p.as_str()).or_else(|| { - t.get("text") - .and_then(|x| x.get("content")) - .and_then(|c| c.as_str()) - }) - }) - .collect(); - let text = text.trim(); - (!text.is_empty()).then(|| text.to_string()) -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/notion/tests.rs b/crates/tinymemory-core/src/sync/composio/providers/notion/tests.rs deleted file mode 100644 index 82a017ff..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/notion/tests.rs +++ /dev/null @@ -1,235 +0,0 @@ -//! Unit tests for the Notion provider. - -use super::normalization::{extract_notion_cursor, extract_page_title, extract_results}; -use super::NotionProvider; -use crate::sync::composio::providers::{ - ComposioProvider, ComposioUsageHandle, ProviderContext, TaskFetchFilter, -}; -use serde_json::json; -use std::sync::Arc; - -fn context(connection_id: Option<&str>) -> ProviderContext { - ProviderContext { - config: Arc::new(tinymemory_api::host::test_support::TestHostConfig::default()) - as Arc, - toolkit: "notion".into(), - connection_id: connection_id.map(str::to_string), - usage: ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - } -} - -#[test] -fn extract_results_walks_common_shapes() { - let v1 = json!({ "data": { "results": [{"id": "p1"}] } }); - let v2 = json!({ "results": [{"id": "p2"}, {"id": "p3"}] }); - let v3 = json!({ "data": {} }); - assert_eq!(extract_results(&v1).len(), 1); - assert_eq!(extract_results(&v2).len(), 2); - assert_eq!(extract_results(&v3).len(), 0); -} - -#[test] -fn extract_notion_cursor_finds_nested() { - let v = json!({ "data": { "next_cursor": "abc123" } }); - assert_eq!(extract_notion_cursor(&v), Some("abc123".to_string())); -} - -#[test] -fn extract_notion_cursor_none_when_missing() { - let v = json!({ "data": { "has_more": false } }); - assert_eq!(extract_notion_cursor(&v), None); -} - -#[test] -fn extract_page_title_from_properties() { - let page = json!({ - "id": "page-1", - "properties": { - "Name": { - "type": "title", - "title": [ - { "plain_text": "My " }, - { "plain_text": "Page Title" } - ] - } - } - }); - assert_eq!(extract_page_title(&page), Some("My Page Title".to_string())); -} - -#[test] -fn extract_page_title_fallback_to_top_level() { - let page = json!({ "title": "Fallback Title" }); - assert_eq!( - extract_page_title(&page), - Some("Fallback Title".to_string()) - ); -} - -#[test] -fn extract_page_title_returns_none_when_missing() { - let page = json!({ "id": "p1" }); - assert_eq!(extract_page_title(&page), None); -} - -#[test] -fn provider_metadata_is_stable() { - let p = NotionProvider::new(); - assert_eq!(p.toolkit_slug(), "notion"); - assert_eq!(p.sync_interval_secs(), Some(30 * 60)); -} - -#[test] -fn default_impl_matches_new() { - let _a = NotionProvider::new(); - let _b = ::default(); -} - -// ── parse_database_results (list_databases parser) ─────────────────────────── - -#[test] -fn parse_database_results_keeps_databases_and_extracts_title() { - use super::provider::parse_database_results; - let data = json!({ - "results": [ - { - "object": "database", - "id": "db-1", - "title": [{ "plain_text": "Engineering " }, { "plain_text": "Tasks" }] - }, - // A page hit must be filtered out — list_databases is databases only. - { "object": "page", "id": "pg-9", "title": [{ "plain_text": "Some page" }] }, - // Newest API exposes databases as `data_source`. - { "object": "data_source", "id": "db-2", "title": [{ "plain_text": "Roadmap" }] }, - // Untitled database falls back to a synthesized label. - { "object": "database", "id": "db-3", "title": [] } - ] - }); - let dbs = parse_database_results(&data); - assert_eq!( - dbs.len(), - 3, - "two named databases + one data_source, page dropped" - ); - assert_eq!(dbs[0].id, "db-1"); - assert_eq!(dbs[0].title, "Engineering Tasks"); - assert_eq!(dbs[1].id, "db-2"); - assert_eq!(dbs[1].title, "Roadmap"); - assert_eq!(dbs[2].title, "Notion database db-3"); -} - -#[test] -fn parse_database_results_handles_data_wrapper_and_empty() { - use super::provider::parse_database_results; - let wrapped = json!({ "data": { "results": [ - { "object": "database", "id": "x", "title": [{ "plain_text": "Wrapped" }] } - ] } }); - let dbs = parse_database_results(&wrapped); - assert_eq!(dbs.len(), 1); - assert_eq!(dbs[0].title, "Wrapped"); - - assert!(parse_database_results(&json!({ "results": [] })).is_empty()); -} - -#[tokio::test] -async fn provider_io_methods_fail_with_action_context_without_host() { - let provider = NotionProvider::new(); - assert!(provider - .fetch_user_profile(&context(Some("connection-1"))) - .await - .unwrap_err() - .contains("NOTION_GET_ABOUT_ME")); - assert!(provider - .fetch_tasks( - &context(Some("connection-1")), - &TaskFetchFilter { - database_id: Some(" database-1 ".into()), - max: 4, - extra: json!({"archived": false}), - ..Default::default() - }, - ) - .await - .unwrap_err() - .contains("NOTION_QUERY_DATABASE")); - assert!(provider - .fetch_tasks(&context(Some("connection-1")), &TaskFetchFilter::default()) - .await - .unwrap_err() - .contains("NOTION_FETCH_DATA")); - assert!(provider - .list_databases(&context(Some("connection-1"))) - .await - .unwrap_err() - .contains("NOTION_SEARCH_NOTION_PAGE")); - assert!(provider - .on_trigger(&context(None), "PAGE_UPDATED", &json!({})) - .await - .unwrap_err() - .contains("missing connection_id")); -} - -#[test] -fn page_normalization_covers_properties_and_fallbacks() { - use super::provider::normalize_notion_page; - - assert!(normalize_notion_page(&json!({"title": "missing id"})).is_none()); - let page = normalize_notion_page(&json!({ - "pageId": "page-7", - "properties": { - "Status": {"status": {"name": "In progress"}}, - "Assignee": {"people": [{"name": "Alice"}]}, - "Due": {"date": {"start": "2026-08-21"}}, - "Priority": {"select": {"name": "High"}} - }, - "lastEditedTime": "2026-08-20T12:00:00Z" - })) - .unwrap(); - assert_eq!(page.external_id, "page-7"); - assert_eq!(page.title, "Notion page page-7"); - assert_eq!(page.status.as_deref(), Some("In progress")); - assert_eq!(page.assignee.as_deref(), Some("Alice")); - assert_eq!(page.due.as_deref(), Some("2026-08-21")); - assert_eq!(page.priority.as_deref(), Some("High")); -} - -/// The second `NOTION_FETCH_DATA` call site. -/// -/// `fetch_tasks` falls back to this action when a Notion task source names no -/// database, and it is reached from a shipped, user-configurable feature — -/// OpenHuman's task-sources pipeline resolves `notion` through `get_provider` -/// and calls `fetch_tasks`. It had the same missing `fetch_type` the periodic -/// sync pipeline did, so it failed Composio's input-schema validation the same -/// way on the Direct (BYOK) arm, where nothing injects the field for it. -/// -/// Asserted on the argument builder rather than through `fetch_tasks`: the -/// `ComposioHost` seam is installed once per process as a signed-out stub, so a -/// test cannot observe what the request carried — only that it errored. That is -/// exactly how this site stayed broken while the tests above passed. -#[test] -fn fetch_tasks_recent_pages_fallback_sends_fetch_type() { - let args = super::provider::fetch_data_args(25); - assert_eq!( - args["fetch_type"], "pages", - "NOTION_FETCH_DATA is rejected without `fetch_type`; args were {args}" - ); - // The rest of the shape is unchanged — pinned so the fix cannot be - // "corrected" by rewriting the request into something Composio pages - // differently. - assert_eq!(args["page_size"], 25); - assert_eq!(args["filter"]["value"], "page"); - assert_eq!(args["sort"]["timestamp"], "last_edited_time"); -} - -/// The cap `fetch_tasks` applies before the request goes out. -#[test] -fn fetch_tasks_recent_pages_fallback_caps_page_size_at_the_api_maximum() { - assert_eq!(super::provider::fetch_data_args(5_000)["page_size"], 100); - assert_eq!( - super::provider::fetch_data_args(5_000)["fetch_type"], - "pages", - "the cap must not drop the required field" - ); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/notion/tools.rs b/crates/tinymemory-core/src/sync/composio/providers/notion/tools.rs deleted file mode 100644 index dd0840ec..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/notion/tools.rs +++ /dev/null @@ -1,6 +0,0 @@ -//! The curated `notion` catalog, re-exported at its historical path. -//! -//! The table moved to [`tinymemory_api::composio::catalogs::notion`] with every -//! other catalog — see [`super::super::catalogs`] for why. - -pub use tinymemory_api::composio::catalogs::notion::NOTION_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/profile.rs b/crates/tinymemory-core/src/sync/composio/providers/profile.rs deleted file mode 100644 index d2546717..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/profile.rs +++ /dev/null @@ -1,354 +0,0 @@ -//! Profile persistence — maps [`ProviderUserProfile`] (and provider-specific -//! `extras`) into [`IdentityKind`]-tagged facet rows so the self-identity -//! matcher can join directly against the memory tree's `EntityKind` and the -//! structural sender field on chunks. -//! -//! Schema: `user_profile.facet_type='skill'`, -//! `key = "skill:{toolkit}:{conn_id}:{identity_kind}"`, `value` = -//! canonicalized identifier. Confidence is set per-kind so the matcher can -//! refuse to auto-promote weak signals (display_name) to `is_self`. -//! -//! One [`ProviderUserProfile`] expands to multiple rows — including -//! identifiers carried in `extras` that the previous fixed-fields shape -//! dropped on the floor (e.g. Slack screen-name handle). -//! -//! Callers invoke [`persist_provider_profile`] after every successful -//! `fetch_user_profile` call — from `on_connection_created`, periodic syncs, -//! and the `composio_get_user_profile` / `composio_refresh_all_identities` -//! RPC ops. -//! -//! # Where the vocabulary lives (#5560) -//! -//! [`IdentityKind`], [`canonicalize`], [`ConnectedIdentity`], -//! [`render_connected_identities_section`] and -//! [`normalize_connection_identifier`] are defined in -//! [`tinymemory_api::composio::profile`] and re-exported here. -//! -//! Canonicalisation had to go down because equality of canonical forms is the -//! matcher's *only* test, and the two calls it compares are on opposite sides -//! of the module boundary: the value is canonicalised here when a profile is -//! persisted, and again in OpenHuman when a candidate identifier is checked -//! against it. Two implementations would fail open — a user's own messages -//! would quietly stop being recognised as theirs. The same argument covers the -//! identifier normalisation, which produces the key segment a row is *stored* -//! under: a delete that spelled it differently would leave rows behind and keep -//! treating a disconnected account as the user. -//! -//! Everything that touches the facet store stayed here, because the contract -//! crate holds no storage. - -use crate::learning_candidate::{ - self as learning_candidate, CueFamily, EvidenceRef, FacetClass, LearningCandidate, -}; -use crate::store::profile::FacetType; -use serde_json::Value; -use std::collections::BTreeMap; - -use tinymemory_api::composio::profile::normalize_connection_identifier as normalize_token; - -/// The identity vocabulary, defined in the contract crate. -/// -/// Re-exported at this path so every historical -/// `providers::profile::IdentityKind` reference keeps resolving. See the module -/// docs for why the shapes went down and the store access stayed. -pub use tinymemory_api::composio::profile::{ - canonicalize, normalize_connection_identifier, render_connected_identities_section, - ConnectedIdentity, IdentityKind, ProviderUserProfile, -}; - -// ──────────────────────────────────────────────────────────────────────── -// Persist -// ──────────────────────────────────────────────────────────────────────── - -/// Persist a provider profile as one facet row per (kind, value). Returns -/// the number of rows written. Silently no-ops if the memory client isn't -/// ready (startup race / unauthenticated CLI). -pub fn persist_provider_profile(profile: &ProviderUserProfile) -> usize { - let Some(client) = crate::global::client_if_ready() else { - tracing::debug!( - toolkit = %profile.toolkit, - "[composio:profile] memory client not ready, skipping persist" - ); - return 0; - }; - let store = client.profile_store(); - - let now = now_secs(); - let toolkit = normalize_token(&profile.toolkit); - let identifier = profile - .connection_id - .as_deref() - .map(normalize_token) - .filter(|v| !v.is_empty()) - .unwrap_or_else(|| "default".to_string()); - - let rows = expand_identity_rows(&toolkit, profile); - - let mut written = 0usize; - for (kind, value) in rows { - let key = format!("skill:{toolkit}:{identifier}:{}", kind.as_str()); - let facet_id = format!("skill-{toolkit}-{identifier}-{}", kind.as_str()); - - if let Err(e) = store.upsert_provider_facet( - &facet_id, - &FacetType::Workflow, - &key, - &value, - kind.confidence(), - None, - now, - ) { - tracing::warn!( - toolkit = %toolkit, - identifier = %identifier, - kind = kind.as_str(), - error = %e, - "[composio:profile] profile_upsert failed (non-fatal)" - ); - continue; - } - - // Phase 3 (#566): also emit a LearningCandidate so the stability detector - // can score provider data alongside other evidence on the next rebuild. - // We use the `identity/` key prefix for provider identity fields. - if kind.is_matchable() { - let identity_key = format!("{}:{}", normalize_token(&toolkit), kind.as_str()); - let candidate = LearningCandidate { - class: FacetClass::Identity, - key: identity_key, - value: value.clone(), - cue_family: CueFamily::Structural, - evidence: EvidenceRef::Provider { - toolkit: toolkit.clone(), - connection_id: identifier.clone(), - field: kind.as_str().to_string(), - }, - initial_confidence: kind.confidence(), - observed_at: now, - }; - learning_candidate::global().push(candidate); - } - - written += 1; - } - - if written > 0 { - tracing::debug!( - toolkit = %toolkit, - identifier = %identifier, - rows_written = written, - "[composio:profile] persisted identity rows (+ emitted Identity candidates)" - ); - } - written -} - -/// Expand a [`ProviderUserProfile`] (and provider-specific `extras`) into -/// the canonical (kind, value) rows. **All per-toolkit quirks live here**; -/// the matcher only sees normalized tuples. -fn expand_identity_rows( - toolkit: &str, - profile: &ProviderUserProfile, -) -> Vec<(IdentityKind, String)> { - let mut rows: Vec<(IdentityKind, String)> = Vec::new(); - let mut push = |kind: IdentityKind, raw: Option<&str>| { - if let Some(v) = raw.and_then(|s| canonicalize(kind, s)) { - rows.push((kind, v)); - } - }; - - push(IdentityKind::DisplayName, profile.display_name.as_deref()); - push(IdentityKind::Email, profile.email.as_deref()); - push(IdentityKind::AvatarUrl, profile.avatar_url.as_deref()); - push(IdentityKind::ProfileUrl, profile.profile_url.as_deref()); - - match toolkit { - "slack" => { - // After the auth.test + users.info fix in slack/provider.rs: - // profile.username == Slack user_id (e.g. U123ABC) - // extras.handle == Slack screen_name (e.g. "cyrus") - // extras.team_* → workspace context, not identity - push(IdentityKind::UserId, profile.username.as_deref()); - push(IdentityKind::Handle, json_str(&profile.extras, "handle")); - } - "notion" => { - // Notion's `username` is the user UUID - // (`data.bot.owner.user.id` per notion/provider.rs). - push(IdentityKind::UserId, profile.username.as_deref()); - } - "gmail" => { - // Email + display_name only — no platform user_id worth matching. - } - _ => { - // Unknown toolkit: best-effort. If `username` is set treat it - // as a handle so weak-match logic (medium confidence) applies. - push(IdentityKind::Handle, profile.username.as_deref()); - } - } - - rows -} - -fn json_str<'a>(v: &'a Value, key: &str) -> Option<&'a str> { - v.get(key).and_then(|x| x.as_str()) -} - -// ──────────────────────────────────────────────────────────────────────── -// Read paths -// ──────────────────────────────────────────────────────────────────────── - -/// Load all provider-sourced identities, grouped by `(source, conn_id)`. -/// Rows whose last segment is not a known [`IdentityKind`] are silently -/// skipped — that includes legacy `username` rows from before the rewrite. -pub fn load_connected_identities() -> Vec { - let Some(client) = crate::global::client_if_ready() else { - tracing::debug!("[composio:profile] load_connected_identities: memory client not ready"); - return Vec::new(); - }; - let facets = match client.profile_store().facets_by_type(&FacetType::Workflow) { - Ok(f) => f, - Err(error) => { - tracing::warn!( - error = %error, - "[composio:profile] load_connected_identities: profile_facets_by_type failed" - ); - return Vec::new(); - } - }; - - let mut grouped: BTreeMap<(String, String), ConnectedIdentity> = BTreeMap::new(); - for facet in facets { - let Some((source, identifier, kind_str)) = parse_skill_identity_key(&facet.key) else { - continue; - }; - let Some(kind) = IdentityKind::parse(&kind_str) else { - continue; - }; - let entry = grouped - .entry((source.clone(), identifier.clone())) - .or_insert_with(|| ConnectedIdentity { - source, - identifier, - ..Default::default() - }); - match kind { - IdentityKind::DisplayName => entry.display_name = Some(facet.value), - IdentityKind::Email => entry.email = Some(facet.value), - IdentityKind::Handle => entry.handle = Some(facet.value), - IdentityKind::Phone => entry.phone = Some(facet.value), - IdentityKind::UserId => entry.user_id = Some(facet.value), - IdentityKind::AvatarUrl => entry.avatar_url = Some(facet.value), - IdentityKind::ProfileUrl => entry.profile_url = Some(facet.value), - } - } - grouped.into_values().collect() -} - -/// Direct self-check for the entity matcher and the chunk-build hook. -/// Returns true if any connection of `toolkit` has a row with this -/// `(kind, value)` after canonicalization. Non-matchable kinds -/// (avatar_url, profile_url) always return false. -pub fn is_self_identity(toolkit: &str, kind: IdentityKind, raw_value: &str) -> bool { - if !kind.is_matchable() { - return false; - } - let Some(canonical) = canonicalize(kind, raw_value) else { - return false; - }; - let Some(client) = crate::global::client_if_ready() else { - return false; - }; - let key_pattern = format!("skill:{}:%:{}", normalize_token(toolkit), kind.as_str()); - client - .profile_store() - .skill_identity_matches(&key_pattern, &canonical) -} - -/// Cross-toolkit variant — matches against every connected provider's -/// rows of this kind. Used for marking memory-tree entity rows: an email -/// in a Slack message that matches the user's Gmail address is still -/// "me," regardless of which source produced the chunk. -pub fn is_self_identity_any_toolkit(kind: IdentityKind, raw_value: &str) -> bool { - if !kind.is_matchable() { - return false; - } - let Some(canonical) = canonicalize(kind, raw_value) else { - return false; - }; - let Some(client) = crate::global::client_if_ready() else { - return false; - }; - let key_pattern = format!("skill:%:%:{}", kind.as_str()); - client - .profile_store() - .skill_identity_matches(&key_pattern, &canonical) -} - -/// Delete every row for a `(source, conn_id)` pair — used on disconnect. -pub fn delete_connected_identity_facets(source: &str, identifier: &str) -> usize { - // `persist_provider_profile` writes keys with `normalize_token`-applied - // segments; compare against the same normalized form here so a caller - // passing the raw toolkit/connection_id still matches stored rows - // (otherwise rows would survive disconnect and the user-tagger would - // keep treating the removed account as the user — #1381 review). - let source = normalize_token(source); - let identifier = normalize_token(identifier); - let Some(client) = crate::global::client_if_ready() else { - tracing::debug!( - source = %source, - identifier = %identifier, - "[composio:profile] delete_connected_identity_facets: memory client not ready" - ); - return 0; - }; - let store = client.profile_store(); - let Ok(facets) = store.facets_by_type(&FacetType::Workflow) else { - return 0; - }; - let mut deleted = 0usize; - for facet in facets { - let Some((s, i, _kind)) = parse_skill_identity_key(&facet.key) else { - continue; - }; - if s == source && i == identifier { - // Same swallow as before: a disconnect must not fail because one - // row was already gone. - if store.delete_by_facet_id(&facet.facet_id).unwrap_or(false) { - deleted += 1; - } - } - } - deleted -} - -// ──────────────────────────────────────────────────────────────────────── -// Helpers -// ──────────────────────────────────────────────────────────────────────── - -fn parse_skill_identity_key(key: &str) -> Option<(String, String, String)> { - let mut parts = key.split(':'); - let prefix = parts.next()?; - let source = parts.next()?; - let identifier = parts.next()?; - let kind = parts.next()?; - if prefix != "skill" || parts.next().is_some() { - return None; - } - Some((source.to_string(), identifier.to_string(), kind.to_string())) -} - -fn now_secs() -> f64 { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs_f64()) - .unwrap_or(0.0) -} - -// ──────────────────────────────────────────────────────────────────────── -// Tests -// ──────────────────────────────────────────────────────────────────────── - -#[cfg(test)] -#[path = "profile_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/profile_md.rs b/crates/tinymemory-core/src/sync/composio/providers/profile_md.rs deleted file mode 100644 index a5006910..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/profile_md.rs +++ /dev/null @@ -1,427 +0,0 @@ -//! `PROFILE.md` markdown bridge — mirrors managed facet blocks into -//! `{workspace_dir}/PROFILE.md` so the agent prompt loader -//! (`agent/prompts/mod.rs::UserFilesSection`) picks them up on the next -//! turn. -//! -//! ## Block convention -//! -//! Each managed section lives between a pair of HTML comment markers: -//! -//! ```md -//! -//! ##
-//! -//! -//! -//! -//! ``` -//! -//! Anything outside the markers is left untouched, so user-authored prose -//! or hand-edited bullets are preserved across provider reconnects or -//! cache rebuilds. -//! -//! All operations are best-effort — errors are logged rather than -//! propagated, matching the PII-discipline pattern used in -//! `on_connection_created`. - -use super::ProviderUserProfile; -use std::fs; -use std::io; -use std::path::Path; - -// ── Legacy connected-accounts constants (kept for internal helpers) ─────────── - -const CA_BLOCK: &str = "connected-accounts"; -const CA_HEADING: &str = "## Connected Accounts"; -const FILE_HEADER: &str = "# User Profile\n"; - -/// All managed block names, in the order they are appended when a new -/// `PROFILE.md` is created. -pub const BLOCKS: &[&str] = &[ - "connected-accounts", // written by provider path (merge_provider_into_profile_md) - "style", - "identity", - "tooling", - "vetoes", - "goals", -]; - -// ── Public API ──────────────────────────────────────────────────────────────── - -/// Upsert the per-toolkit bullet for `profile` inside the managed -/// `connected-accounts` block of `{workspace_dir}/PROFILE.md`. -/// -/// Creates the file with a `# User Profile` header if it does not exist. -/// Idempotent — re-connecting the same toolkit replaces the existing -/// bullet rather than duplicating it. -pub fn merge_provider_into_profile_md( - workspace_dir: &Path, - profile: &ProviderUserProfile, -) -> io::Result<()> { - let toolkit = normalize_token(&profile.toolkit); - if toolkit.is_empty() { - return Ok(()); - } - // Require a real connection_id so the bullet keys match what the - // disconnect path (`composio_delete_connection`) will look up. - let identifier = profile - .connection_id - .as_deref() - .map(normalize_token) - .filter(|v| !v.is_empty()); - let identifier = match identifier { - Some(id) => id, - None => { - tracing::debug!( - toolkit = %toolkit, - "[composio:profile_md] skipping merge — connection_id missing or empty" - ); - return Ok(()); - } - }; - - let bullet = match render_provider_bullet(&toolkit, &identifier, profile) { - Some(b) => b, - None => return Ok(()), - }; - - let path = workspace_dir.join("PROFILE.md"); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - let existing = match fs::read_to_string(&path) { - Ok(s) => s, - Err(e) if e.kind() == io::ErrorKind::NotFound => String::new(), - Err(e) => return Err(e), - }; - - let updated = upsert_provider_bullet(&existing, &toolkit, &identifier, &bullet); - fs::write(&path, updated)?; - tracing::debug!( - target_file = "PROFILE.md", - toolkit = %toolkit, - identifier = %identifier, - "[composio:profile_md] merged provider profile into PROFILE.md" - ); - Ok(()) -} - -/// Remove the per-toolkit bullet for `(source, identifier)` from the -/// managed Connected Accounts block. If the block becomes empty the whole -/// block is dropped. Missing file or missing block are no-ops. -pub fn remove_provider_from_profile_md( - workspace_dir: &Path, - source: &str, - identifier: &str, -) -> io::Result<()> { - let path = workspace_dir.join("PROFILE.md"); - let existing = match fs::read_to_string(&path) { - Ok(s) => s, - Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), - Err(e) => return Err(e), - }; - let toolkit = normalize_token(source); - let identifier = normalize_token(identifier); - if toolkit.is_empty() || identifier.is_empty() { - return Ok(()); - } - let updated = remove_provider_bullet(&existing, &toolkit, &identifier); - if updated != existing { - fs::write(&path, updated)?; - tracing::debug!( - target_file = "PROFILE.md", - toolkit = %toolkit, - identifier = %identifier, - "[composio:profile_md] removed provider bullet from PROFILE.md" - ); - } - Ok(()) -} - -/// Upsert a generic managed block. -/// -/// * `block_name` — one of [`BLOCKS`] (e.g. `"style"`, `"identity"`). -/// * `section_heading` — heading rendered inside the block (e.g. `"## Style"`). -/// * `body_markdown` — pre-rendered content (bullets, prose). Must not -/// contain the block markers themselves. -/// -/// Creates `PROFILE.md` if it does not exist. If the block is absent it is -/// appended at the end of the file. If the block exists its body is replaced -/// in-place — content outside the markers is left byte-for-byte untouched. -/// -/// An empty `body_markdown` renders a `*(no entries yet)*` placeholder -/// instead of deleting the block; this preserves the block's position for the -/// next write. -/// -/// Idempotent: calling with the same inputs twice produces the same file. -pub fn replace_managed_block( - workspace_dir: &Path, - block_name: &str, - section_heading: &str, - body_markdown: String, -) -> io::Result<()> { - let path = workspace_dir.join("PROFILE.md"); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - - let existing = match fs::read_to_string(&path) { - Ok(s) => s, - Err(e) if e.kind() == io::ErrorKind::NotFound => String::new(), - Err(e) => return Err(e), - }; - - let updated = upsert_block(&existing, block_name, section_heading, &body_markdown); - fs::write(&path, updated)?; - tracing::debug!( - block_name = %block_name, - "[composio:profile_md] replaced managed block '{}' in PROFILE.md", - block_name - ); - Ok(()) -} - -// ── Connected-accounts internals ────────────────────────────────────────────── - -fn render_provider_bullet( - toolkit: &str, - identifier: &str, - profile: &ProviderUserProfile, -) -> Option { - let mut fields: Vec = Vec::new(); - if let Some(v) = profile.display_name.as_deref().map(sanitize) { - if !v.is_empty() { - fields.push(v); - } - } - if let Some(v) = profile.email.as_deref().map(sanitize) { - if !v.is_empty() { - fields.push(v); - } - } - if let Some(v) = profile.username.as_deref().map(sanitize) { - if !v.is_empty() { - fields.push(format!("@{v}")); - } - } - if let Some(v) = profile.profile_url.as_deref().map(sanitize) { - if !v.is_empty() { - fields.push(v); - } - } - if fields.is_empty() { - return None; - } - let marker = bullet_marker(toolkit, identifier); - Some(format!( - "- {marker} **{title}** ({identifier}): {fields}", - title = title_case(toolkit), - identifier = identifier, - fields = fields.join(" | ") - )) -} - -fn bullet_marker(toolkit: &str, identifier: &str) -> String { - format!("") -} - -/// Insert or replace `bullet` inside the connected-accounts managed block. -fn upsert_provider_bullet(existing: &str, toolkit: &str, identifier: &str, bullet: &str) -> String { - let marker = bullet_marker(toolkit, identifier); - let start_tag = block_start(CA_BLOCK); - let end_tag = block_end(CA_BLOCK); - let (prefix, block_body, suffix) = split_any_block(existing, &start_tag, &end_tag); - - let mut lines: Vec = block_body - .lines() - .filter(|l| !l.contains(&marker)) - .map(|l| l.to_string()) - .collect(); - lines.push(bullet.to_string()); - - let mut bullets = lines - .into_iter() - .filter(|l| l.trim_start().starts_with("- ") -} - -/// Build the end marker for `block_name`. -pub fn block_end(block_name: &str) -> String { - format!("") -} - -/// Insert or replace a generic managed block in `existing`. -/// -/// If the block is absent it is appended. If it exists its body (between the -/// markers) is replaced. Content outside the markers is returned unchanged. -fn upsert_block( - existing: &str, - block_name: &str, - section_heading: &str, - body_markdown: &str, -) -> String { - let start_tag = block_start(block_name); - let end_tag = block_end(block_name); - - let body = if body_markdown.trim().is_empty() { - "*(no entries yet)*".to_string() - } else { - body_markdown.to_string() - }; - - let block = format!("{start_tag}\n{section_heading}\n\n{body}\n\n{end_tag}"); - - let (prefix, _old_body, suffix) = split_any_block(existing, &start_tag, &end_tag); - - if prefix == existing { - // Block was absent — append. - assemble(existing, &block, "") - } else { - assemble(&prefix, &block, &suffix) - } -} - -/// Split `existing` around the markers `[start_tag, end_tag]`. -/// -/// Returns `(prefix, block_body, suffix)`. If no block is present, -/// `prefix` is the full string and `block_body` / `suffix` are empty. -/// `block_body` is the content *between* the markers (excluding the -/// markers themselves). -fn split_any_block(existing: &str, start_tag: &str, end_tag: &str) -> (String, String, String) { - if let (Some(start), Some(end)) = (existing.find(start_tag), existing.find(end_tag)) { - if end > start { - let prefix = existing[..start].to_string(); - let body = existing[start + start_tag.len()..end].to_string(); - let suffix_start = end + end_tag.len(); - let suffix = existing[suffix_start..].to_string(); - return (prefix, body, suffix); - } - } - (existing.to_string(), String::new(), String::new()) -} - -/// Assemble `prefix + block + suffix`, normalising the newlines immediately -/// adjacent to the managed block while leaving the user's bytes elsewhere -/// untouched. -fn assemble(prefix: &str, block: &str, suffix: &str) -> String { - if block.is_empty() { - // Removing the block entirely. - let p = prefix.trim_end_matches('\n'); - let s = suffix.trim_start_matches('\n'); - let mut out = String::with_capacity(p.len() + s.len() + 2); - out.push_str(p); - if !p.is_empty() { - out.push('\n'); - if !s.is_empty() { - out.push('\n'); - } - } - out.push_str(s); - if !out.is_empty() && !out.ends_with('\n') { - out.push('\n'); - } - return out; - } - - let mut out = String::new(); - if prefix.trim().is_empty() { - // Seed with a header on first creation. - out.push_str(FILE_HEADER); - out.push('\n'); - } else { - let p = prefix.trim_end_matches('\n'); - out.push_str(p); - out.push_str("\n\n"); - } - out.push_str(block); - if suffix.is_empty() { - out.push('\n'); - } else { - let s = suffix.trim_start_matches('\n'); - if s.is_empty() { - // Suffix was only whitespace — end with single newline, no blank line. - out.push('\n'); - } else { - out.push_str("\n\n"); - out.push_str(s); - if !out.ends_with('\n') { - out.push('\n'); - } - } - } - out -} - -// ── Token / string helpers ──────────────────────────────────────────────────── - -fn normalize_token(raw: &str) -> String { - let mut out = String::with_capacity(raw.len()); - for ch in raw.chars() { - let lower = ch.to_ascii_lowercase(); - if lower.is_ascii_alphanumeric() || lower == '-' || lower == '_' { - out.push(lower); - } else { - out.push('_'); - } - } - out.trim_matches('_').to_string() -} - -fn title_case(raw: &str) -> String { - let mut chars = raw.chars(); - match chars.next() { - Some(first) => first.to_ascii_uppercase().to_string() + chars.as_str(), - None => String::new(), - } -} - -fn sanitize(raw: &str) -> String { - let replaced = raw.replace(['\n', '\r', '\t'], " ").replace('|', "/"); - replaced.split_whitespace().collect::>().join(" ") -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -#[path = "profile_md_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/profile_md_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/profile_md_tests.rs deleted file mode 100644 index 998b44f7..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/profile_md_tests.rs +++ /dev/null @@ -1,293 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; -use tempfile::TempDir; - -// ── merge_provider_into_profile_md (legacy API, unchanged) ─────────────── - -fn sample(toolkit: &str, conn: &str) -> ProviderUserProfile { - ProviderUserProfile { - toolkit: toolkit.into(), - connection_id: Some(conn.into()), - display_name: Some("Jane Doe".into()), - email: Some("jane@example.com".into()), - username: Some("janedoe".into()), - avatar_url: None, - profile_url: Some("https://example.com/jane".into()), - extras: serde_json::Value::Null, - } -} - -#[test] -fn creates_file_when_missing() { - let tmp = TempDir::new().unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(body.starts_with("# User Profile"), "body was:\n{body}"); - let start = block_start(CA_BLOCK); - let end = block_end(CA_BLOCK); - assert!(body.contains(&start)); - assert!(body.contains(CA_HEADING)); - assert!(body.contains("**Gmail** (c-1):")); - assert!(body.contains("jane@example.com")); - assert!(body.contains("@janedoe")); - assert!(body.contains(&end)); -} - -#[test] -fn upsert_is_idempotent_for_same_toolkit_connection() { - let tmp = TempDir::new().unwrap(); - let mut p = sample("gmail", "c-1"); - merge_provider_into_profile_md(tmp.path(), &p).unwrap(); - p.display_name = Some("Jane D.".into()); - merge_provider_into_profile_md(tmp.path(), &p).unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - let occurrences = body.matches("acct:gmail:c-1").count(); - assert_eq!(occurrences, 1, "duplicate bullet:\n{body}"); - assert!(body.contains("Jane D.")); - assert!(!body.contains("Jane Doe")); -} - -#[test] -fn multiple_toolkits_render_separate_bullets() { - let tmp = TempDir::new().unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("twitter", "c-2")).unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(body.contains("acct:gmail:c-1")); - assert!(body.contains("acct:twitter:c-2")); - let start = block_start(CA_BLOCK); - let end = block_end(CA_BLOCK); - assert_eq!(body.matches(&start).count(), 1); - assert_eq!(body.matches(&end).count(), 1); -} - -#[test] -fn preserves_user_authored_content_outside_block() { - let tmp = TempDir::new().unwrap(); - let path = tmp.path().join("PROFILE.md"); - fs::write( - &path, - "# User Profile\n\nSome bio paragraph from LinkedIn.\n\n## Key facts\n- a\n- b\n", - ) - .unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - let body = fs::read_to_string(&path).unwrap(); - assert!(body.contains("Some bio paragraph from LinkedIn.")); - assert!(body.contains("## Key facts")); - assert!(body.contains("- a")); - assert!(body.contains("acct:gmail:c-1")); -} - -#[test] -fn skips_when_no_useful_fields() { - let tmp = TempDir::new().unwrap(); - let p = ProviderUserProfile { - toolkit: "gmail".into(), - connection_id: Some("c-1".into()), - display_name: Some(" ".into()), - email: None, - username: Some("".into()), - avatar_url: None, - profile_url: None, - extras: serde_json::Value::Null, - }; - merge_provider_into_profile_md(tmp.path(), &p).unwrap(); - assert!(!tmp.path().join("PROFILE.md").exists()); -} - -#[test] -fn remove_drops_specific_bullet() { - let tmp = TempDir::new().unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("twitter", "c-2")).unwrap(); - remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(!body.contains("acct:gmail:c-1")); - assert!(body.contains("acct:twitter:c-2")); -} - -#[test] -fn remove_drops_block_when_empty() { - let tmp = TempDir::new().unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - let start = block_start(CA_BLOCK); - let end = block_end(CA_BLOCK); - assert!(!body.contains(&start), "block remained:\n{body}"); - assert!(!body.contains(&end)); - assert!(body.starts_with("# User Profile")); -} - -#[test] -fn remove_is_noop_when_file_missing() { - let tmp = TempDir::new().unwrap(); - remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); - assert!(!tmp.path().join("PROFILE.md").exists()); -} - -#[test] -fn skips_when_connection_id_missing() { - let tmp = TempDir::new().unwrap(); - let p = ProviderUserProfile { - toolkit: "gmail".into(), - connection_id: None, - display_name: Some("Jane".into()), - email: Some("jane@example.com".into()), - username: None, - avatar_url: None, - profile_url: None, - extras: serde_json::Value::Null, - }; - merge_provider_into_profile_md(tmp.path(), &p).unwrap(); - assert!(!tmp.path().join("PROFILE.md").exists()); -} - -#[test] -fn preserves_indentation_and_blank_lines_around_block() { - let tmp = TempDir::new().unwrap(); - let path = tmp.path().join("PROFILE.md"); - let original = "# User Profile\n\n indented bio line\n\n## Notes\n- alpha\n- beta\n\n"; - fs::write(&path, original).unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - let body = fs::read_to_string(&path).unwrap(); - assert!(body.contains(" indented bio line")); - assert!(body.contains("## Notes\n- alpha\n- beta")); - let start = block_start(CA_BLOCK); - let end = block_end(CA_BLOCK); - assert!(body.contains(&start) && body.contains(&end)); - remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); - let after = fs::read_to_string(&path).unwrap(); - assert!(after.contains(" indented bio line")); - assert!(after.contains("## Notes\n- alpha\n- beta")); - assert!(!after.contains(&start)); -} - -#[test] -fn sanitize_strips_pipes_and_newlines() { - assert_eq!(sanitize("foo\nbar"), "foo bar"); - assert_eq!(sanitize("a | b"), "a / b"); - assert_eq!(sanitize(" multi space "), "multi space"); -} - -// ── replace_managed_block ───────────────────────────────────────────────── - -#[test] -fn replace_managed_block_creates_file_if_missing() { - let tmp = TempDir::new().unwrap(); - replace_managed_block( - tmp.path(), - "style", - "## Style", - "- **verbosity**: terse".into(), - ) - .unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(body.contains("# User Profile"), "missing header:\n{body}"); - assert!(body.contains(&block_start("style"))); - assert!(body.contains("## Style")); - assert!(body.contains("- **verbosity**: terse")); - assert!(body.contains(&block_end("style"))); -} - -#[test] -fn replace_managed_block_appends_block_when_absent() { - let tmp = TempDir::new().unwrap(); - let path = tmp.path().join("PROFILE.md"); - fs::write(&path, "# User Profile\n\nSome existing text.\n").unwrap(); - replace_managed_block( - tmp.path(), - "identity", - "## Identity", - "- **name**: Alice".into(), - ) - .unwrap(); - let body = fs::read_to_string(&path).unwrap(); - // Existing content preserved. - assert!(body.contains("Some existing text.")); - // New block appended. - assert!(body.contains(&block_start("identity"))); - assert!(body.contains("## Identity")); - assert!(body.contains("- **name**: Alice")); -} - -#[test] -fn replace_managed_block_replaces_body_in_place() { - let tmp = TempDir::new().unwrap(); - replace_managed_block( - tmp.path(), - "style", - "## Style", - "- **verbosity**: verbose".into(), - ) - .unwrap(); - replace_managed_block( - tmp.path(), - "style", - "## Style", - "- **verbosity**: terse".into(), - ) - .unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(body.contains("terse")); - assert!(!body.contains("verbose")); - // Only one start marker. - assert_eq!(body.matches(&block_start("style")).count(), 1); -} - -#[test] -fn replace_managed_block_preserves_other_blocks_and_user_text() { - let tmp = TempDir::new().unwrap(); - // Write two blocks. - replace_managed_block( - tmp.path(), - "style", - "## Style", - "- **verbosity**: terse".into(), - ) - .unwrap(); - replace_managed_block( - tmp.path(), - "identity", - "## Identity", - "- **name**: Bob".into(), - ) - .unwrap(); - // Update only style. - replace_managed_block( - tmp.path(), - "style", - "## Style", - "- **verbosity**: verbose".into(), - ) - .unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - // Identity block untouched. - assert!(body.contains("- **name**: Bob")); - // Style updated. - assert!(body.contains("verbose")); - assert!(!body.contains("terse")); -} - -#[test] -fn replace_managed_block_empty_body_renders_placeholder() { - let tmp = TempDir::new().unwrap(); - replace_managed_block(tmp.path(), "goals", "## Goals", String::new()).unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(body.contains("*(no entries yet)*")); - // Block markers still present. - assert!(body.contains(&block_start("goals"))); - assert!(body.contains(&block_end("goals"))); -} - -#[test] -fn replace_managed_block_idempotent_on_repeat_invocation() { - let tmp = TempDir::new().unwrap(); - let content = "- **verbosity**: terse".to_string(); - replace_managed_block(tmp.path(), "style", "## Style", content.clone()).unwrap(); - let body1 = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - replace_managed_block(tmp.path(), "style", "## Style", content).unwrap(); - let body2 = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert_eq!(body1, body2, "second write should be idempotent"); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/profile_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/profile_tests.rs deleted file mode 100644 index 28eafaeb..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/profile_tests.rs +++ /dev/null @@ -1,378 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; -use crate::store::profile::{self, profile_load_all, PROFILE_INIT_SQL}; -use parking_lot::Mutex; -use rusqlite::Connection; -use serde_json::json; -use std::sync::Arc; - -fn setup_db() -> Arc> { - let conn = Connection::open_in_memory().unwrap(); - conn.execute_batch(PROFILE_INIT_SQL).unwrap(); - Arc::new(Mutex::new(conn)) -} - -// ── IdentityKind ─────────────────────────────────────────────── - -#[test] -fn identity_kind_round_trips_through_str() { - for kind in [ - IdentityKind::UserId, - IdentityKind::Email, - IdentityKind::Handle, - IdentityKind::Phone, - IdentityKind::DisplayName, - IdentityKind::AvatarUrl, - IdentityKind::ProfileUrl, - ] { - assert_eq!(IdentityKind::parse(kind.as_str()), Some(kind)); - } -} - -#[test] -fn identity_kind_parse_rejects_unknown() { - assert_eq!(IdentityKind::parse("username"), None); - assert_eq!(IdentityKind::parse(""), None); - assert_eq!(IdentityKind::parse("UserId"), None); -} - -#[test] -fn matchable_kinds_exclude_url_fields() { - assert!(IdentityKind::UserId.is_matchable()); - assert!(IdentityKind::Email.is_matchable()); - assert!(IdentityKind::Handle.is_matchable()); - assert!(IdentityKind::Phone.is_matchable()); - assert!(IdentityKind::DisplayName.is_matchable()); - assert!(!IdentityKind::AvatarUrl.is_matchable()); - assert!(!IdentityKind::ProfileUrl.is_matchable()); -} - -#[test] -fn confidence_orders_hard_above_weak() { - assert!(IdentityKind::UserId.confidence() > IdentityKind::Email.confidence()); - assert!(IdentityKind::Email.confidence() > IdentityKind::Handle.confidence()); - assert!(IdentityKind::Handle.confidence() > IdentityKind::DisplayName.confidence()); -} - -// ── canonicalize ────────────────────────────────────────────── - -#[test] -fn canonicalize_email_lowercases_and_trims() { - assert_eq!( - canonicalize(IdentityKind::Email, " Cyrus@Example.COM "), - Some("cyrus@example.com".to_string()) - ); -} - -#[test] -fn canonicalize_handle_strips_at_and_lowercases() { - assert_eq!( - canonicalize(IdentityKind::Handle, "@Cyrus"), - Some("cyrus".to_string()) - ); - assert_eq!( - canonicalize(IdentityKind::Handle, "cyrus"), - Some("cyrus".to_string()) - ); -} - -#[test] -fn canonicalize_phone_keeps_only_digits_and_plus() { - assert_eq!( - canonicalize(IdentityKind::Phone, "+1 (555) 123-4567"), - Some("+15551234567".to_string()) - ); -} - -#[test] -fn canonicalize_display_name_collapses_whitespace() { - assert_eq!( - canonicalize(IdentityKind::DisplayName, " Cyrus Smith "), - Some("Cyrus Smith".to_string()) - ); -} - -#[test] -fn canonicalize_user_id_preserved_as_is() { - // Slack user_ids are case-sensitive; do not lowercase. - assert_eq!( - canonicalize(IdentityKind::UserId, "U123ABC"), - Some("U123ABC".to_string()) - ); -} - -#[test] -fn canonicalize_empty_returns_none() { - assert_eq!(canonicalize(IdentityKind::Email, ""), None); - assert_eq!(canonicalize(IdentityKind::Email, " "), None); -} - -// ── expand_identity_rows ────────────────────────────────────── - -fn fixture_profile(toolkit: &str, username: Option<&str>, extras: Value) -> ProviderUserProfile { - ProviderUserProfile { - toolkit: toolkit.into(), - connection_id: Some("conn-1".into()), - display_name: Some("Cyrus Smith".into()), - email: Some("cyrus@example.com".into()), - username: username.map(str::to_string), - avatar_url: None, - profile_url: Some("https://example.com/cyrus".into()), - extras, - } -} - -#[test] -fn expand_slack_promotes_username_to_user_id_and_extras_handle() { - let p = fixture_profile("slack", Some("U123ABC"), json!({ "handle": "cyrus" })); - let rows = expand_identity_rows("slack", &p); - - assert!(rows.contains(&(IdentityKind::UserId, "U123ABC".to_string()))); - assert!(rows.contains(&(IdentityKind::Handle, "cyrus".to_string()))); - assert!(rows.contains(&(IdentityKind::Email, "cyrus@example.com".to_string()))); - assert!(rows.contains(&(IdentityKind::DisplayName, "Cyrus Smith".to_string()))); - assert!(rows.contains(&( - IdentityKind::ProfileUrl, - "https://example.com/cyrus".to_string() - ))); -} - -#[test] -fn expand_gmail_skips_username_with_no_user_id_concept() { - let p = fixture_profile("gmail", None, Value::Null); - let rows = expand_identity_rows("gmail", &p); - - assert!(rows - .iter() - .all(|(k, _)| !matches!(k, IdentityKind::UserId | IdentityKind::Handle))); - assert!(rows.contains(&(IdentityKind::Email, "cyrus@example.com".to_string()))); -} - -#[test] -fn expand_notion_treats_username_as_user_id() { - let p = fixture_profile( - "notion", - Some("f3c1a8e2-b9b7-4a8d-9d5b-31a2e9f44e2f"), - Value::Null, - ); - let rows = expand_identity_rows("notion", &p); - - assert!(rows.contains(&( - IdentityKind::UserId, - "f3c1a8e2-b9b7-4a8d-9d5b-31a2e9f44e2f".to_string() - ))); -} - -#[test] -fn expand_unknown_toolkit_falls_back_to_handle() { - let p = fixture_profile("hypothetical", Some("alice"), Value::Null); - let rows = expand_identity_rows("hypothetical", &p); - - assert!(rows.contains(&(IdentityKind::Handle, "alice".to_string()))); -} - -#[test] -fn expand_empty_profile_emits_nothing_matchable() { - let p = ProviderUserProfile { - toolkit: "gmail".into(), - connection_id: Some("c-1".into()), - display_name: None, - email: None, - username: None, - avatar_url: None, - profile_url: None, - extras: Value::Null, - }; - let rows = expand_identity_rows("gmail", &p); - assert!(rows.is_empty()); -} - -// ── upsert wiring (uses the underlying profile_upsert directly) ─ - -#[test] -fn upsert_writes_kind_tagged_key() { - let conn = setup_db(); - - profile::profile_upsert( - &conn, - "skill-slack-conn-1-user_id", - &FacetType::Workflow, - "skill:slack:conn-1:user_id", - "U123ABC", - IdentityKind::UserId.confidence(), - None, - 1000.0, - ) - .unwrap(); - - let facets = profile_load_all(&conn).unwrap(); - let row = facets - .iter() - .find(|f| f.key == "skill:slack:conn-1:user_id") - .expect("row exists"); - assert_eq!(row.value, "U123ABC"); - assert!((row.confidence - 1.00).abs() < f64::EPSILON); -} - -#[test] -fn upsert_repeated_increments_evidence() { - let conn = setup_db(); - - for now in [1000.0, 2000.0] { - profile::profile_upsert( - &conn, - "skill-notion-default-email", - &FacetType::Workflow, - "skill:notion:default:email", - "user@workspace.com", - IdentityKind::Email.confidence(), - None, - now, - ) - .unwrap(); - } - - let facets = profile_load_all(&conn).unwrap(); - assert_eq!(facets.len(), 1); - assert_eq!(facets[0].evidence_count, 2); -} - -// ── parse_skill_identity_key ────────────────────────────────── - -#[test] -fn parse_key_round_trip() { - let parsed = parse_skill_identity_key("skill:slack:conn_1:user_id"); - assert_eq!( - parsed, - Some(( - "slack".to_string(), - "conn_1".to_string(), - "user_id".to_string() - )) - ); -} - -#[test] -fn parse_key_rejects_wrong_prefix() { - assert!(parse_skill_identity_key("preference:slack:c:email").is_none()); -} - -#[test] -fn parse_key_rejects_extra_segments() { - assert!(parse_skill_identity_key("skill:slack:c:email:extra").is_none()); -} - -// ── render ──────────────────────────────────────────────────── - -#[test] -fn render_includes_handle_with_at_and_omits_user_id() { - let rendered = render_connected_identities_section(&[ConnectedIdentity { - source: "slack".into(), - identifier: "T01ABC".into(), - display_name: Some("Cyrus Smith".into()), - email: Some("cyrus@example.com".into()), - handle: Some("cyrus".into()), - phone: None, - user_id: Some("U123ABC".into()), - avatar_url: None, - profile_url: None, - }]); - assert!(rendered.contains("## Connected Identities")); - assert!(rendered.contains("- Slack (T01ABC): Cyrus Smith | cyrus@example.com | @cyrus")); - assert!( - !rendered.contains("U123ABC"), - "user_id should not appear in prompt" - ); -} - -#[test] -fn render_empty_list_returns_empty_string() { - assert_eq!(render_connected_identities_section(&[]), ""); -} - -#[test] -fn render_sanitizes_untrusted_fields_and_skips_empty_identities() { - let rendered = render_connected_identities_section(&[ - ConnectedIdentity { - source: "linear".into(), - identifier: " conn\n42 ".into(), - display_name: Some(" Alice\tExample ".into()), - email: Some("alice|example.com".into()), - handle: Some("\r alice ".into()), - phone: None, - user_id: None, - avatar_url: None, - profile_url: Some(" https://example.com/a|b ".into()), - }, - ConnectedIdentity { - source: "slack".into(), - identifier: "unused".into(), - display_name: Some(" \n\t ".into()), - ..Default::default() - }, - ]); - - assert_eq!( - rendered, - "## Connected Identities\n\n- Linear (conn 42): Alice Example | alice/example.com | @alice | https://example.com/a/b\n" - ); -} - -#[test] -fn render_returns_empty_when_every_identity_has_only_non_rendered_fields() { - let rendered = render_connected_identities_section(&[ConnectedIdentity { - source: "slack".into(), - identifier: "conn".into(), - phone: Some("+15551234567".into()), - user_id: Some("U123".into()), - avatar_url: Some("https://example.com/avatar".into()), - ..Default::default() - }]); - - assert!(rendered.is_empty()); -} - -#[test] -fn helper_parsing_and_normalization_cover_malformed_inputs() { - for key in ["", "skill", "skill:slack", "skill:slack:conn"] { - assert_eq!(parse_skill_identity_key(key), None); - } - assert_eq!( - normalize_connection_identifier(" Team.Name/@Me "), - "team_name__me" - ); - assert_eq!(normalize_connection_identifier("___"), ""); - // `title_case` went down with the renderer that is its only caller - // (#5560); its behaviour is asserted through - // `render_connected_identities_section` in the contract crate's tests, - // which is the only way it is observable. -} - -#[test] -fn canonicalize_preserves_urls_and_removes_empty_phone_noise() { - assert_eq!( - canonicalize(IdentityKind::ProfileUrl, " https://example.com/Me "), - Some("https://example.com/Me".into()) - ); - assert_eq!( - canonicalize(IdentityKind::Phone, "extension only"), - Some(String::new()) - ); -} - -// ── now_secs sanity ─────────────────────────────────────────── - -#[test] -fn now_secs_returns_recent_unix_seconds() { - let t = now_secs(); - assert!(t > 1_000_000_000.0); -} - -#[test] -fn persist_returns_zero_when_memory_client_not_ready() { - // Exercise the early-return branch. Global client may or may - // not be initialised in the test binary depending on ordering. - let p = fixture_profile("gmail", None, Value::Null); - let _ = persist_provider_profile(&p); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/providers_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/providers_tests.rs deleted file mode 100644 index 6134ee49..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/providers_tests.rs +++ /dev/null @@ -1,318 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; -use serde_json::json; - -#[test] -fn pick_str_finds_first_non_empty_match() { - let v = json!({ - "data": { "user": { "email": " user@example.com ", "name": "" } }, - "fallback": "fallback@example.com" - }); - // first path empty -> falls through - assert_eq!( - pick_str(&v, &["data.user.name", "data.user.email"]), - Some("user@example.com".to_string()) - ); - // missing path -> falls through to fallback - assert_eq!( - pick_str(&v, &["data.missing", "fallback"]), - Some("fallback@example.com".to_string()) - ); - // nothing matches - assert_eq!(pick_str(&v, &["nope.nope"]), None); -} - -#[test] -fn sync_outcome_elapsed_ms_is_safe_when_finish_lt_start() { - let mut o = SyncOutcome { - started_at_ms: 100, - finished_at_ms: 50, - ..Default::default() - }; - assert_eq!(o.elapsed_ms(), 0); - o.finished_at_ms = 250; - assert_eq!(o.elapsed_ms(), 150); -} - -#[test] -fn pick_str_returns_none_for_non_string_values() { - let v = json!({ "count": 42, "flag": true, "empty": "", "whitespace": " " }); - assert_eq!(pick_str(&v, &["count"]), None); - assert_eq!(pick_str(&v, &["flag"]), None); - assert_eq!(pick_str(&v, &["empty"]), None); - assert_eq!(pick_str(&v, &["whitespace"]), None); -} - -#[test] -fn pick_str_respects_path_order() { - let v = json!({ "a": "first", "b": "second" }); - assert_eq!(pick_str(&v, &["a", "b"]), Some("first".into())); - assert_eq!(pick_str(&v, &["b", "a"]), Some("second".into())); -} - -#[test] -fn sync_reason_as_str_matches_enum_variant() { - assert_eq!(SyncReason::ConnectionCreated.as_str(), "connection_created"); - assert_eq!(SyncReason::Periodic.as_str(), "periodic"); - assert_eq!(SyncReason::Manual.as_str(), "manual"); -} - -#[test] -fn sync_reason_serde_is_snake_case() { - let s = serde_json::to_string(&SyncReason::ConnectionCreated).unwrap(); - assert_eq!(s, "\"connection_created\""); - let back: SyncReason = serde_json::from_str(&s).unwrap(); - assert_eq!(back, SyncReason::ConnectionCreated); -} - -// Note: `toolkit_has_scope` tests now live in `scope_lookup.rs` -// alongside the implementation. - -#[test] -fn catalog_for_toolkit_resolves_new_microsoft_and_todoist_slugs() { - // Newly added catalogs (#2283): OneDrive, Excel, Todoist must be - // discoverable both by their canonical UI slug AND by the - // prefix that `toolkit_from_slug` extracts from action slugs. - assert!(catalog_for_toolkit("one_drive").is_some()); - assert!(catalog_for_toolkit("onedrive").is_some()); - // ONE_DRIVE_GET_FILE → toolkit_from_slug() → "one" - assert!(catalog_for_toolkit("one").is_some()); - assert!(catalog_for_toolkit("excel").is_some()); - assert!(catalog_for_toolkit("todoist").is_some()); -} - -#[test] -fn agent_ready_toolkits_includes_new_catalogs_and_is_sorted() { - let slugs = agent_ready_toolkits(); - assert!(slugs.contains(&"one_drive")); - assert!(slugs.contains(&"excel")); - assert!(slugs.contains(&"todoist")); - // Spot-check legacy entries still present. - assert!(slugs.contains(&"gmail")); - assert!(slugs.contains(&"slack")); - // Uncurated toolkit must NOT appear — guarantees the UI badge - // logic can rely on this set to flag "preview" toolkits. - assert!(!slugs.contains(&"sharepoint")); - assert!(!slugs.contains(&"clickup")); - // Stable order across builds — the RPC consumer caches it. - let mut expected = slugs.clone(); - expected.sort_unstable(); - assert_eq!(slugs, expected); -} - -#[test] -fn capability_matrix_includes_new_catalog_only_toolkits() { - let matrix = capability_matrix(); - for slug in ["one_drive", "excel", "todoist"] { - let row = matrix - .iter() - .find(|entry| entry.toolkit == slug) - .unwrap_or_else(|| panic!("{slug} capability row missing")); - assert!(!row.native_provider, "{slug} should not be native"); - assert!(row.curated_tools, "{slug} should be catalogued"); - assert!( - row.curated_tool_count > 0, - "{slug} catalog should be non-empty" - ); - assert!( - row.tool_execution, - "{slug} tool execution should be enabled" - ); - // No profile/sync/memory ingest — catalog-only. - assert!(!row.user_profile); - assert!(!row.initial_sync); - assert!(!row.periodic_sync); - assert!(!row.memory_ingest); - } -} - -#[test] -fn capability_matrix_distinguishes_native_from_catalog_only_toolkits() { - let matrix = capability_matrix(); - - let gmail = matrix - .iter() - .find(|entry| entry.toolkit == "gmail") - .expect("gmail capability row"); - assert!(gmail.native_provider); - assert!(gmail.curated_tools); - assert!(gmail.curated_tool_count > 0); - assert!(gmail.user_profile); - assert!(gmail.initial_sync); - assert!(gmail.periodic_sync); - assert_eq!(gmail.sync_interval_secs, Some(15 * 60)); - assert!(gmail.trigger_webhooks); - assert!(gmail.memory_ingest); - - let google_calendar = matrix - .iter() - .find(|entry| entry.toolkit == "googlecalendar") - .expect("googlecalendar capability row"); - assert!(!google_calendar.native_provider); - assert!(google_calendar.curated_tools); - assert!(google_calendar.curated_tool_count > 0); - assert!(google_calendar.tool_execution); - assert!(!google_calendar.user_profile); - assert!(!google_calendar.initial_sync); - assert!(!google_calendar.periodic_sync); - assert_eq!(google_calendar.sync_interval_secs, None); - assert!(!google_calendar.memory_ingest); -} - -#[test] -fn capability_matrix_includes_clickup_as_native_memory_provider() { - // Locks in the per-issue #2288 registration: a ClickUp row must - // appear in the capability matrix with the same native-provider - // flags Gmail/Notion/Slack already carry (`memory_ingest`, - // `periodic_sync`, non-zero `sync_interval_secs`). If a future - // change drops one of the four registration touchpoints - // (CAPABILITY_TOOLKITS, has_native_provider, - // native_provider_sync_interval, catalog_for_toolkit) this test - // fails loud rather than silently degrading the provider to - // catalog-only status. - let matrix = capability_matrix(); - let clickup = matrix - .iter() - .find(|entry| entry.toolkit == "clickup") - .expect("clickup capability row"); - assert!(clickup.native_provider, "clickup must be native"); - assert!(clickup.curated_tools, "clickup must have a curated catalog"); - assert!( - clickup.curated_tool_count > 0, - "clickup catalog must be non-empty" - ); - assert!(clickup.user_profile); - assert!(clickup.initial_sync); - assert!(clickup.periodic_sync); - assert_eq!(clickup.sync_interval_secs, Some(30 * 60)); - assert!(clickup.memory_ingest); -} - -#[test] -fn capability_matrix_includes_linear_as_native_memory_provider() { - // Per-issue #2400 registration: a Linear row must appear in - // the capability matrix as a native memory-ingest provider, - // matching gmail / notion / slack / clickup. If a future - // change drops one of the five registration touchpoints - // (CAPABILITY_TOOLKITS, has_native_provider, - // native_provider_sync_interval, catalog_for_toolkit, - // toolkit_description) this test fails loud rather than - // silently degrading the provider to catalog-only status. - let matrix = capability_matrix(); - let linear = matrix - .iter() - .find(|entry| entry.toolkit == "linear") - .expect("linear capability row"); - assert!(linear.native_provider, "linear must be native"); - assert!(linear.curated_tools, "linear must have a curated catalog"); - assert!( - linear.curated_tool_count > 0, - "linear catalog must be non-empty" - ); - assert!(linear.user_profile); - assert!(linear.initial_sync); - assert!(linear.periodic_sync); - assert_eq!(linear.sync_interval_secs, Some(30 * 60)); - assert!(linear.memory_ingest); -} - -#[test] -fn capability_matrix_includes_github_as_native_memory_provider() { - let matrix = capability_matrix(); - let github = matrix - .iter() - .find(|entry| entry.toolkit == "github") - .expect("github capability row"); - assert!(github.native_provider, "github must be native"); - assert!(github.curated_tools, "github must have a curated catalog"); - assert!( - github.curated_tool_count > 0, - "github catalog must be non-empty" - ); - assert!(github.user_profile); - assert!(github.initial_sync); - assert!(github.periodic_sync); - assert_eq!(github.sync_interval_secs, Some(30 * 60)); - assert!(github.memory_ingest); -} - -#[test] -fn toolkit_description_known_slugs_are_distinct_and_non_empty() { - let known = [ - "gmail", - "notion", - "github", - "slack", - "discord", - "google_calendar", - "google_drive", - "google_docs", - "google_sheets", - "outlook", - "microsoft_teams", - "linear", - "jira", - "trello", - "asana", - "dropbox", - "twitter", - "spotify", - "telegram", - "whatsapp", - "twilio", - "shopify", - "stripe", - "hubspot", - "salesforce", - "airtable", - "figma", - "youtube", - "calendar", - ]; - let fallback = toolkit_description("__definitely_unknown_slug__"); - for slug in known { - let desc = toolkit_description(slug); - assert!(!desc.is_empty(), "{slug} description must not be empty"); - assert_ne!( - desc, fallback, - "known slug `{slug}` must not map to the generic fallback" - ); - } -} - -#[test] -fn toolkit_description_unknown_slug_uses_generic_fallback() { - assert_eq!( - toolkit_description("not_a_real_toolkit_123"), - "Interact with this connected service via its available actions" - ); - assert_eq!( - toolkit_description(""), - "Interact with this connected service via its available actions" - ); -} - -#[test] -fn toolkit_description_is_case_sensitive() { - // The match is lowercase-only by convention; an uppercase slug - // should fall through to the generic description. Explicitly - // documenting this guards against accidental case-insensitive - // matching sneaking in later. - let fallback = toolkit_description("__fallback__"); - assert_eq!(toolkit_description("GMAIL"), fallback); - assert_eq!(toolkit_description("Notion"), fallback); -} - -#[test] -fn provider_user_profile_default_is_empty() { - let p = ProviderUserProfile::default(); - assert!(p.toolkit.is_empty()); - assert!(p.connection_id.is_none()); - assert!(p.display_name.is_none()); - assert!(p.email.is_none()); - assert!(p.username.is_none()); - assert!(p.avatar_url.is_none()); - assert!(p.profile_url.is_none()); - assert!(p.extras.is_null()); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/registry.rs b/crates/tinymemory-core/src/sync/composio/providers/registry.rs deleted file mode 100644 index b42d1acc..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/registry.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! Process-global registry of [`ComposioProvider`] implementations. -//! -//! There is exactly one provider per toolkit slug — the trait is not -//! a fan-out fan-in dispatch, it is a 1:1 mapping. This keeps trigger -//! routing simple (`HashMap::get(toolkit)` → call) and avoids the -//! "which subscriber wins" ambiguity that would come with multiple -//! providers per toolkit. -//! -//! The registry is initialised once at startup via -//! [`init_default_providers`] and is intentionally write-rare: tests -//! can register additional providers ad-hoc, but the production path -//! only writes during the startup hook. - -use std::collections::HashMap; -use std::sync::{Arc, OnceLock, RwLock}; - -use super::ComposioProvider; - -/// Reference-counted handle to a registered provider. -pub type ProviderArc = Arc; - -/// Backing storage for the global registry. -/// -/// `RwLock>` is fine here — registration happens at -/// startup and lookups are very fast (no contention in steady state). -type Registry = RwLock>; - -static REGISTRY: OnceLock = OnceLock::new(); - -fn registry() -> &'static Registry { - REGISTRY.get_or_init(|| RwLock::new(HashMap::new())) -} - -/// Register or replace a provider for its toolkit slug. -/// -/// Idempotent — re-registering the same toolkit overwrites the -/// previous entry, which is what tests rely on for setup/teardown. -pub fn register_provider(provider: ProviderArc) { - let slug = provider.toolkit_slug().to_string(); - if slug.is_empty() { - tracing::warn!("[composio:registry] refusing to register provider with empty slug"); - return; - } - let mut guard = registry() - .write() - .expect("composio provider registry poisoned"); - let was_present = guard.insert(slug.clone(), provider).is_some(); - if was_present { - tracing::debug!(toolkit = %slug, "[composio:registry] replaced existing provider"); - } else { - tracing::info!(toolkit = %slug, "[composio:registry] provider registered"); - } -} - -/// Look up the provider for a toolkit slug, if one is registered. -pub fn get_provider(toolkit: &str) -> Option { - let key = toolkit.trim(); - if key.is_empty() { - return None; - } - let guard = registry() - .read() - .expect("composio provider registry poisoned"); - guard.get(key).cloned() -} - -/// Snapshot of every registered provider, in unspecified order. Used -/// by the periodic sync scheduler to walk every toolkit. -pub fn all_providers() -> Vec { - let guard = registry() - .read() - .expect("composio provider registry poisoned"); - guard.values().cloned().collect() -} - -/// Register the built-in providers shipped with the core. Called once -/// from `start_channels` / `bootstrap_core_runtime` startup paths. -/// -/// Idempotent: re-running just re-registers (no-op in practice). -pub fn init_default_providers() { - register_provider(Arc::new(super::clickup::ClickUpProvider::new())); - register_provider(Arc::new(super::github::GitHubProvider::new())); - register_provider(Arc::new(super::gmail::GmailProvider::new())); - register_provider(Arc::new(super::linear::LinearProvider::new())); - register_provider(Arc::new(super::notion::NotionProvider::new())); - register_provider(Arc::new(super::slack::SlackProvider::new())); - tracing::info!( - count = all_providers().len(), - "[composio:registry] default providers initialised" - ); -} - -#[cfg(test)] -#[path = "registry_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/registry_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/registry_tests.rs deleted file mode 100644 index 9a466bea..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/registry_tests.rs +++ /dev/null @@ -1,58 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; -use crate::sync::composio::providers::{ProviderContext, ProviderUserProfile}; -use async_trait::async_trait; - -struct DummyProvider { - slug: &'static str, -} - -#[async_trait] -impl ComposioProvider for DummyProvider { - fn toolkit_slug(&self) -> &'static str { - self.slug - } - async fn fetch_user_profile( - &self, - _ctx: &ProviderContext, - ) -> Result { - Ok(ProviderUserProfile::default()) - } -} - -#[test] -fn register_and_lookup_roundtrip() { - register_provider(Arc::new(DummyProvider { - slug: "test_dummy_a", - })); - let p = get_provider("test_dummy_a").expect("provider should be registered"); - assert_eq!(p.toolkit_slug(), "test_dummy_a"); -} - -#[test] -fn lookup_unknown_returns_none() { - assert!(get_provider("__definitely_not_a_real_toolkit__").is_none()); -} - -#[test] -fn register_replaces_existing() { - register_provider(Arc::new(DummyProvider { - slug: "test_dummy_b", - })); - register_provider(Arc::new(DummyProvider { - slug: "test_dummy_b", - })); - // Still exactly one entry under that slug. - let count_with_b = all_providers() - .iter() - .filter(|p| p.toolkit_slug() == "test_dummy_b") - .count(); - assert_eq!(count_with_b, 1); -} - -#[test] -fn empty_slug_is_rejected() { - register_provider(Arc::new(DummyProvider { slug: "" })); - assert!(get_provider("").is_none()); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/scope_lookup.rs b/crates/tinymemory-core/src/sync/composio/providers/scope_lookup.rs deleted file mode 100644 index 922c2e88..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/scope_lookup.rs +++ /dev/null @@ -1,13 +0,0 @@ -//! Scope-lookup helpers, re-exported at their historical path. -//! -//! Moved to [`tinymemory_api::composio::catalogs`] with the catalogs they walk. -//! -//! One behaviour note, because it looks like a change and is not: the versions -//! here consulted `get_provider(..).curated_tools()` before falling back to -//! `catalog_for_toolkit`. Every native provider's `curated_tools()` returns -//! exactly the slice `catalog_for_toolkit` returns for the same toolkit — true -//! of all six — so the provider hop was pure indirection and the contract -//! versions drop it. That is what lets a host answer "may this action run" -//! without a provider registry. - -pub use tinymemory_api::composio::catalogs::{curated_scope_for, toolkit_has_scope}; diff --git a/crates/tinymemory-core/src/sync/composio/providers/slack/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/slack/mod.rs deleted file mode 100644 index 39b745a5..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/slack/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Composio-backed Slack provider. -//! -//! The provider is wired into the periodic-sync scheduler (see -//! [`super::registry::init_default_providers`]) and fires -//! `SLACK_LIST_CONVERSATIONS` + `SLACK_FETCH_CONVERSATION_HISTORY` -//! against the user's Composio-authorized Slack connection. The reusable -//! synchronization and ingestion engine is owned by tinycortex. - -// The Slack post-processor moved to tinycortex (a pure Value transform, i.e. -// driver-side). Re-exported under the old module name — `pub`, not a plain -// `use`, because `tests/raw_coverage/memory_threads_raw_coverage_e2e.rs` -// imports this path directly. -pub use tinymemory_sync::slack_post_process as post_process; -pub mod types; - -mod provider; - -pub use provider::{run_backfill_via_search, SlackProvider, BACKFILL_DAYS}; -pub use types::{SlackChannel, SlackMessage}; diff --git a/crates/tinymemory-core/src/sync/composio/providers/slack/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/slack/provider.rs deleted file mode 100644 index 4ecc844f..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/slack/provider.rs +++ /dev/null @@ -1,300 +0,0 @@ -//! Composio-backed Slack provider. -//! -//! Drives Slack history ingestion **without** a user-managed bot token -//! — authorization lives in the user's Composio Slack connection, and -//! the actual API calls fan out through [`ComposioClient::execute_tool`] -//! against Composio's action catalog (`SLACK_LIST_CONVERSATIONS`, -//! `SLACK_FETCH_CONVERSATION_HISTORY`, `SLACK_FETCH_TEAM_INFO`, …). -//! -//! The product provider retains profile lookup, trigger handling, and response -//! normalization. Channel enumeration, cursors, history paging, and memory -//! ingestion execute in tinycortex's Slack sync pipeline. -//! -//! ## Idempotency -//! -//! Source id is `slack:{connection_id}` — stable per workspace. Chunk -//! IDs are stable, so repeated synchronization updates the same documents. - -use crate::sync::composio::providers::{ - pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, ProviderContext, - ProviderUserProfile, SyncOutcome, -}; -use async_trait::async_trait; -use serde_json::{json, Value}; - -/// Composio action slug for team/workspace profile fetch. -const ACTION_FETCH_TEAM_INFO: &str = "SLACK_FETCH_TEAM_INFO"; -/// Composio action slug for Slack `auth.test` — returns the authed -/// user's id, handle, and team. Required for self-identity capture. -const ACTION_AUTH_TEST: &str = "SLACK_TEST_AUTH"; -/// Composio action slug for Slack `users.info` — returns the user's -/// profile (email, real_name, avatar). Optional; needs `users:read.email` -/// scope for the email field. -const ACTION_USERS_INFO: &str = "SLACK_RETRIEVE_DETAILED_USER_INFORMATION"; - -/// Default backfill window (days) applied when a channel has no -/// cursor yet. -pub const BACKFILL_DAYS: i64 = 6; - -/// Sync cadence for provider catalog scheduling. -const SYNC_INTERVAL_SECS: u64 = 15 * 60; - -pub struct SlackProvider; - -impl SlackProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for SlackProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for SlackProvider { - fn toolkit_slug(&self) -> &'static str { - "slack" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(crate::sync::composio::providers::catalogs::SLACK_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - Some(resolve_sync_interval_secs("slack", SYNC_INTERVAL_SECS)) - } - - fn post_process_action_result( - &self, - slug: &str, - arguments: Option<&serde_json::Value>, - data: &mut serde_json::Value, - ) { - super::post_process::post_process(slug, arguments, data); - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:slack] fetch_user_profile via {ACTION_AUTH_TEST}" - ); - - // Step 1 — auth.test: required. Returns user_id (canonical sender - // id on Slack messages), the user's handle, and the team. - let auth_resp = ctx - .execute(ACTION_AUTH_TEST, Some(json!({}))) - .await - .map_err(|e| format!("[composio:slack] {ACTION_AUTH_TEST} failed: {e:#}"))?; - - if !auth_resp.successful { - let err = auth_resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!("[composio:slack] {ACTION_AUTH_TEST}: {err}")); - } - - // `auth_data` is the inner Composio payload — paths are relative - // to it. Slack's auth.test returns user_id/user/team/team_id at - // the top of `data`. - let auth_data = &auth_resp.data; - let user_id = pick_str(auth_data, &["user_id"]); - let handle = pick_str(auth_data, &["user"]); - let team_id = pick_str(auth_data, &["team_id"]); - let team_name = pick_str(auth_data, &["team"]); - - // Step 2 — users.info: optional. Needs `users:read.email` scope - // for `email`; falls back to `auth.test` data on missing-scope or - // any other failure so the profile still carries user_id+handle. - let mut display_name: Option = None; - let mut email: Option = None; - let mut avatar_url: Option = None; - - if let Some(uid) = user_id.as_deref() { - match ctx - .execute(ACTION_USERS_INFO, Some(json!({ "user": uid }))) - .await - { - Ok(info) if info.successful => { - let d = &info.data; - email = pick_str(d, &["user.profile.email", "profile.email"]); - display_name = pick_str( - d, - &[ - "user.profile.real_name", - "user.real_name", - "user.profile.display_name", - ], - ); - avatar_url = pick_str(d, &["user.profile.image_192", "user.profile.image_72"]); - } - Ok(info) => { - tracing::info!( - connection_id = ?ctx.connection_id, - error = ?info.error, - "[composio:slack] {ACTION_USERS_INFO} returned non-success — \ - falling back to auth.test data only (likely missing users:read scope)" - ); - } - Err(e) => { - tracing::info!( - connection_id = ?ctx.connection_id, - error = %e, - "[composio:slack] {ACTION_USERS_INFO} call failed — \ - falling back to auth.test data only" - ); - } - } - } - - // Step 3 — team_info: optional. Adds workspace context to `extras` - // (email_domain, icon) so the prompt section / UI can show it. - let (team_email_domain, team_icon) = - match ctx.execute(ACTION_FETCH_TEAM_INFO, Some(json!({}))).await { - Ok(resp) if resp.successful => { - let d = &resp.data; - let domain = pick_str(d, &["team.email_domain", "email_domain"]); - let icon = pick_str(d, &["team.icon.image_132", "team.icon.image_68"]); - (domain, icon) - } - _ => (None, None), - }; - - // Display name preference: users.info real_name > auth.test handle - // > team_name (last-resort so the prompt isn't empty). - let final_display_name = display_name - .clone() - .or_else(|| handle.clone()) - .or_else(|| team_name.clone()); - - // Profile URL: users.info doesn't return one for the user - // directly; the workspace URL is acceptable as a navigational - // fallback. (Slack user profile pages are workspace-scoped and - // not stably linkable from auth.test alone.) - let profile_url = pick_str(auth_data, &["url"]); - - let avatar_url = avatar_url.or(team_icon); - - let profile = ProviderUserProfile { - toolkit: "slack".to_string(), - connection_id: ctx.connection_id.clone(), - display_name: final_display_name, - email, - // username carries the platform-canonical sender id so the - // self-identity matcher can compare against Slack message - // sender_user_id directly. Handle moves into `extras` — - // `expand_identity_rows` lifts it back out as IdentityKind::Handle. - username: user_id, - avatar_url, - profile_url, - extras: json!({ - "handle": handle, - "team_id": team_id, - "team_name": team_name, - "team_email_domain": team_email_domain, - }), - }; - - let has_email = profile.email.is_some(); - let email_domain = profile - .email - .as_deref() - .and_then(|e| e.split('@').nth(1)) - .map(|d| d.to_string()); - tracing::info!( - connection_id = ?profile.connection_id, - has_email, - email_domain = ?email_domain, - has_user_id = profile.username.is_some(), - "[composio:slack] fetched user profile" - ); - Ok(profile) - } - - /// Slack rides the generic orchestrator. Channel enumeration + the user - /// directory backfill happen in `super::source::SlackSource::preamble`; - /// per-channel `conversations.history` pagination, the per-channel `oldest` - /// watermark, dedup, the `max_items` cap, and per-channel error tolerance - /// all live in `run_sync`. The Slack-specific primitives live in - /// `super::source`. - async fn on_trigger( - &self, - ctx: &ProviderContext, - trigger: &str, - _payload: &Value, - ) -> Result<(), String> { - if trigger.to_ascii_uppercase().contains("MESSAGE") { - let Some(connection_id) = ctx.connection_id.as_deref() else { - return Err("[composio:slack] trigger missing connection_id".to_string()); - }; - if let Err(e) = crate::sync::pipelines::host::run_composio_connection( - "slack", - connection_id, - ctx.config.as_ref(), - None, - None, - ) - .await - { - tracing::warn!( - error = %e, - "[composio:slack] trigger-driven sync failed (non-fatal)" - ); - } - } - Ok(()) - } -} - -// ── Search-based backfill (one-shot) ──────────────────────────────── - -/// Compatibility wrapper for the tinycortex workspace-wide search pipeline. -pub async fn run_backfill_via_search( - ctx: &ProviderContext, - backfill_days: i64, -) -> Result { - let connection_id = ctx - .connection_id - .as_deref() - .ok_or_else(|| "[composio:slack] search backfill missing connection_id".to_string())?; - let started_at_ms = now_ms(); - let outcome = crate::sync::pipelines::host::run_slack_search_backfill( - connection_id, - backfill_days, - ctx.config.as_ref(), - ) - .await - .map_err(|error| error.to_string())?; - Ok(SyncOutcome { - toolkit: "slack".into(), - connection_id: Some(connection_id.into()), - reason: "manual".into(), - items_ingested: outcome.records_ingested as usize, - started_at_ms, - finished_at_ms: now_ms(), - summary: outcome - .note - .unwrap_or_else(|| "slack search-backfill complete".into()), - details: serde_json::json!({ - "more_pending": outcome.more_pending, - "actions_called": outcome.actions_called, - "provider_cost_usd": outcome.provider_cost_usd, - }), - }) -} - -fn now_ms() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64 -} -#[cfg(test)] -#[path = "provider_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/slack/provider_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/slack/provider_tests.rs deleted file mode 100644 index e14e59c9..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/slack/provider_tests.rs +++ /dev/null @@ -1,81 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; -use std::sync::Arc; - -fn context(connection_id: Option<&str>) -> ProviderContext { - ProviderContext { - config: Arc::new(tinymemory_api::host::test_support::TestHostConfig::default()) - as Arc, - toolkit: "slack".into(), - connection_id: connection_id.map(str::to_string), - usage: crate::sync::composio::providers::ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - } -} - -#[test] -fn toolkit_slug_is_stable() { - assert_eq!(SlackProvider::new().toolkit_slug(), "slack"); -} - -#[test] -fn sync_interval_matches_constant() { - assert_eq!( - SlackProvider::new().sync_interval_secs(), - Some(SYNC_INTERVAL_SECS) - ); -} - -#[test] -fn curated_tools_returns_slack_catalog() { - let tools = SlackProvider::new().curated_tools().unwrap(); - assert!(tools - .iter() - .any(|t| t.slug == "SLACK_FETCH_CONVERSATION_HISTORY")); - assert!(tools.iter().any(|t| t.slug == "SLACK_LIST_CONVERSATIONS")); -} - -#[test] -fn post_process_action_result_delegates_to_post_process_module() { - let provider = SlackProvider::new(); - let mut data = serde_json::json!({ - "channels": [{"id": "C1", "name": "eng", "is_private": false}] - }); - // Calling with an unknown slug should be a no-op. - provider.post_process_action_result("SLACK_UNKNOWN_ACTION", None, &mut data); - assert!( - data.get("channels").is_some(), - "no-op slug must not mutate data" - ); -} - -#[tokio::test] -async fn profile_and_backfill_failures_name_the_missing_boundary() { - let provider = SlackProvider::new(); - let profile = provider - .fetch_user_profile(&context(None)) - .await - .unwrap_err(); - assert!(profile.contains(ACTION_AUTH_TEST)); - let backfill = run_backfill_via_search(&context(None), BACKFILL_DAYS) - .await - .unwrap_err(); - assert!(backfill.contains("missing connection_id")); -} - -#[tokio::test] -async fn trigger_filters_non_messages_and_requires_connection_for_messages() { - let provider = SlackProvider::new(); - provider - .on_trigger(&context(None), "CHANNEL_CREATED", &serde_json::json!({})) - .await - .unwrap(); - let error = provider - .on_trigger(&context(None), "MESSAGE_CREATED", &serde_json::json!({})) - .await - .unwrap_err(); - assert!(error.contains("missing connection_id")); - assert!(now_ms() > 0); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/slack/types.rs b/crates/tinymemory-core/src/sync/composio/providers/slack/types.rs deleted file mode 100644 index 42714427..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/slack/types.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! Canonical types for the Composio-backed Slack provider. -//! -//! These types are independent of the Composio/Slack API payload shape. -//! They remain as compatibility wire types for product RPC responses and -//! backfill reporting; tinycortex owns runtime parsing and ingestion. -//! -//! The old `Bucket` struct (6-hour UTC window) has been removed — the -//! memory tree's L0 seal cascade handles batching after PR #1348, so -//! tinycortex owns batching and incremental persistence. - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; - -/// A single message fetched from Slack's `conversations.history` or -/// `search.messages`. -/// -/// The Slack API represents `ts` as a decimal string like -/// `"1714003200.123456"` where the integer part is Unix seconds and the -/// fractional part is a per-workspace message sequence. We retain the -/// original string in `ts_raw` so it can round-trip back to the API -/// (e.g. as the `oldest` cursor on the next poll, and as the permalink -/// suffix for provenance). -/// -/// `channel_name`, `is_private`, `author_id`, and `permalink` are added -/// vs the old `memory::slack_ingestion::types::SlackMessage` because we no -/// longer carry a separate `SlackChannel` through the ingest path — -/// per-message context is self-contained. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct SlackMessage { - /// Channel ID this message belongs to (e.g. `"C0123456"`). - pub channel_id: String, - /// Human-readable channel name (e.g. `"eng"`). Injected by the enricher - /// from the channel directory; may be empty for search results whose - /// channel was not listed. - pub channel_name: String, - /// `true` if this is a private channel the bot has been invited to. - pub is_private: bool, - /// Resolved display name of the author. Falls back to the raw user id - /// when the user directory doesn't have an entry for this id. - pub author: String, - /// Raw Slack user id (e.g. `"U01234"`). Retained alongside the resolved - /// `author` so downstream code can still look up or log the stable id. - pub author_id: String, - /// Message body (plain text; may contain Slack-flavoured markdown). - pub text: String, - /// Canonical timestamp derived from `ts_raw`. - pub timestamp: DateTime, - /// Raw Slack `ts` string (used for API cursors + archive URLs). - pub ts_raw: String, - /// Root thread `ts` if this message is a reply; `None` for top-level - /// messages. Retained for future thread-aware ingestion. - pub thread_ts: Option, - /// Resolved HTTPS permalink, if Composio includes it in the response. - /// Falls back to the `slack://archives/…` scheme in ingest. - pub permalink: Option, -} - -/// A Slack channel visible to the bot, as returned by `conversations.list`. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct SlackChannel { - /// Channel ID (stable across renames). - pub id: String, - /// Human-readable name (e.g. `"eng"` → rendered as `"#eng"` in headers). - /// May change if admins rename the channel. - pub name: String, - /// `true` if this is a private channel the bot has been invited to. - pub is_private: bool, -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/sync_state.rs b/crates/tinymemory-core/src/sync/composio/providers/sync_state.rs deleted file mode 100644 index d39e4364..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/sync_state.rs +++ /dev/null @@ -1,138 +0,0 @@ -//! Cursor, dedup and daily-budget state for Composio sync (#18 §B2). -//! -//! Engine-neutral, persisted through the [`SyncStateStore`] KV seam — any -//! provider whose KV family can get and set a JSON value can carry sync state. -//! -//! The engine keeps its own copy of the shape for its internal pipelines until -//! §B1's orchestrator move retires them. The two persist under the same KV -//! namespace with the same serde form; the pin tests either side hold this copy -//! to that contract. -//! -//! # Where the shape lives (#5560) -//! -//! [`SyncState`], [`DailyBudget`], the namespaces and [`extract_item_id`] are -//! defined in [`tinymemory_api::composio::state`] and re-exported here. Both -//! sides read them: the module advances the cursor and spends the budget, while -//! OpenHuman renders "312 of 500 requests used today" and, on disconnect, walks -//! the dedup set to decide what to forget. A host-side twin would decode today -//! and diverge on the first added field — and because this shape is -//! *persisted*, divergence is a stranded cursor and a re-ingested inbox rather -//! than a wire error someone notices. -//! -//! What stayed here is the I/O: the [`SyncStateStore`] seam and the two methods -//! that use it, offered as the [`PersistedSyncState`] extension trait because -//! an inherent `impl` has to live in the crate that defines the type. Call -//! sites are unchanged — `SyncState::load(store, …)` and `state.save(store)` -//! still resolve — but the trait has to be in scope, so the four call sites in -//! this crate import it alongside the type. - -use async_trait::async_trait; - -/// The persisted sync-state shape, defined in the contract crate. -/// -/// Re-exported at this path so every historical -/// `providers::sync_state::SyncState` reference keeps resolving. -pub use tinymemory_api::composio::state::{ - extract_item_id, DailyBudget, SyncState, DEFAULT_DAILY_REQUEST_LIMIT, KV_NAMESPACE, - STATE_NAMESPACE, -}; - -/// The key/value seam a [`SyncState`] is persisted through. -/// -/// Deliberately narrower than a memory client: get and set one JSON value by -/// `(namespace, key)`. That is the whole requirement, and stating it as two -/// methods is what lets a non-TinyCortex driver carry Composio sync state -/// without implementing anything else. -#[async_trait] -pub trait SyncStateStore: Send + Sync { - /// Read the value at `(namespace, key)`, or `None` when nothing is stored. - /// - /// # Errors - /// - /// Returns an error when the underlying store cannot be reached. "Nothing - /// stored" is `Ok(None)`, not an error — a first sync is the normal case. - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result>; - - /// Write `value` at `(namespace, key)`, replacing anything already there. - /// - /// # Errors - /// - /// Returns an error when the underlying store rejects or cannot persist the - /// write. - async fn set( - &self, - namespace: &str, - key: &str, - value: &serde_json::Value, - ) -> anyhow::Result<()>; -} - -/// Loading and saving a [`SyncState`] through a [`SyncStateStore`]. -/// -/// An extension trait rather than an inherent `impl` because the type is -/// defined in the contract crate, which holds no I/O and publishes no traits. -/// The method names and signatures are the ones the inherent versions had, so -/// existing call sites only need this trait in scope. -#[async_trait] -pub trait PersistedSyncState: Sized { - /// Load the state for one `(toolkit, connection)` pair. - /// - /// A connection with nothing stored yields a fresh state rather than an - /// error — that is a first sync, not a failure. A loaded state has its - /// daily budget rolled forward before it is returned, so what a caller - /// spends and later writes back is today's row rather than yesterday's. - /// - /// # Errors - /// - /// Returns an error when the store cannot be reached, or when the stored - /// value is not a decodable state. Both are genuine faults: silently - /// starting from a fresh state would re-ingest everything the connection - /// had already synced. - async fn load( - store: &dyn SyncStateStore, - toolkit: &str, - connection_id: &str, - ) -> anyhow::Result; - - /// Persist this state under its `(toolkit, connection)` key. - /// - /// # Errors - /// - /// Returns an error when the state cannot be serialised or the store - /// rejects the write. - async fn save(&self, store: &dyn SyncStateStore) -> anyhow::Result<()>; -} - -#[async_trait] -impl PersistedSyncState for SyncState { - async fn load( - store: &dyn SyncStateStore, - toolkit: &str, - connection_id: &str, - ) -> anyhow::Result { - let key = Self::key(toolkit, connection_id); - match store.get(STATE_NAMESPACE, &key).await? { - Some(value) => { - let mut state: Self = serde_json::from_value(value)?; - state.daily_budget.roll_over_if_stale(); - Ok(state) - } - None => Ok(Self::new(toolkit, connection_id)), - } - } - - async fn save(&self, store: &dyn SyncStateStore) -> anyhow::Result<()> { - let value = serde_json::to_value(self)?; - store - .set( - STATE_NAMESPACE, - &Self::key(&self.toolkit, &self.connection_id), - &value, - ) - .await - } -} - -#[cfg(test)] -#[path = "sync_state_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/sync_state_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/sync_state_tests.rs deleted file mode 100644 index d68b7a51..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/sync_state_tests.rs +++ /dev/null @@ -1,104 +0,0 @@ -//! Tests for the surrounding module. - -use std::collections::HashMap; -use std::sync::Mutex; - -use super::*; - -#[derive(Default)] -struct MemoryStateStore(Mutex>); - -#[async_trait] -impl SyncStateStore for MemoryStateStore { - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - Ok(self - .0 - .lock() - .unwrap() - .get(&format!("{namespace}:{key}")) - .cloned()) - } - - async fn set( - &self, - namespace: &str, - key: &str, - value: &serde_json::Value, - ) -> anyhow::Result<()> { - self.0 - .lock() - .unwrap() - .insert(format!("{namespace}:{key}"), value.clone()); - Ok(()) - } -} - -#[tokio::test] -async fn state_round_trips_cursor_dedup_and_budget() { - let store = MemoryStateStore::default(); - let mut state = SyncState::new("gmail", "conn-1"); - state.advance_cursor("cursor-2"); - state.mark_synced("message-1"); - state.record_requests(3); - state.save(&store).await.unwrap(); - - let loaded = SyncState::load(&store, "gmail", "conn-1").await.unwrap(); - assert_eq!(loaded.cursor.as_deref(), Some("cursor-2")); - assert!(loaded.is_synced("message-1")); - assert_eq!(loaded.daily_budget.requests_used, 3); -} - -/// The namespace is durable: every persisted Composio sync cursor lives -/// under this string, so a change strands all of them. The engine's copy -/// must agree; failing here means a coordinated migration, never a local -/// edit. -#[test] -fn the_state_namespace_is_pinned() { - assert_eq!( - KV_NAMESPACE, "composio-sync-state", - "the Composio sync-state KV namespace changed; every persisted \ - cursor is stored under the old value and needs migrating" - ); - assert_eq!(STATE_NAMESPACE, KV_NAMESPACE); -} - -/// The engine persists the same state with its own copy of this type. -/// Pins the serialised shape so the copies cannot drift silently. -#[test] -fn state_line_format_is_pinned() { - let mut state = SyncState::new("gmail", "conn-1"); - state.daily_budget.date = "2026-01-02".into(); - state.daily_budget.requests_used = 3; - state.advance_cursor("c2"); - state.mark_synced("m1"); - state.item_versions.insert("m1".into(), "v1".into()); - state.set_last_seen_id("m1"); - state.set_last_sync_at_ms(1_000); - let value = serde_json::to_value(&state).unwrap(); - assert_eq!( - value, - serde_json::json!({ - "toolkit": "gmail", - "connection_id": "conn-1", - "cursor": "c2", - "synced_ids": ["m1"], - "item_versions": {"m1": "v1"}, - "daily_budget": {"date": "2026-01-02", "requests_used": 3, "limit": 500}, - "last_seen_id": "m1", - "last_sync_at_ms": 1000 - }) - ); -} - -#[test] -fn stale_budget_reports_full_and_resets_on_record() { - let mut budget = DailyBudget { - date: "2000-01-01".into(), - requests_used: 499, - limit: 500, - }; - assert_eq!(budget.remaining(), 500); - budget.record_requests(1); - assert_eq!(budget.requests_used, 1); - assert_eq!(budget.remaining(), 499); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/tool_scope.rs b/crates/tinymemory-core/src/sync/composio/providers/tool_scope.rs deleted file mode 100644 index 8044c622..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/tool_scope.rs +++ /dev/null @@ -1,32 +0,0 @@ -//! Per-action scope classification (read / write / admin) plus the -//! [`CuratedTool`] catalog type that providers use to whitelist the -//! actions they want surfaced to the agent. -//! -//! Composio publishes 60+ actions per toolkit; most are noise for the -//! agent's planning loop. Each provider exports a hand-curated -//! [`CuratedTool`] slice via [`super::ComposioProvider::curated_tools`] -//! that pares the surface down to a useful subset and tags every action -//! with a [`ToolScope`] so per-user scope preferences can gate execution. -//! -//! # Where the definitions live (#5560) -//! -//! All of it moved to [`tinymemory_api::composio::scopes`] and is re-exported -//! here at its historical path. The reason is that the *same verdict* has to be -//! reached on both sides of the module boundary: OpenHuman filters the agent's -//! visible tool list with [`classify_unknown`] and [`find_curated`], and the -//! sync pipelines gate execution with them inside the module. Two copies of the -//! verb-precedence rule would be two different answers to "may this action -//! run", which is not a shape mismatch but a permissions bug. -//! -//! The curated catalogs themselves stay in this crate — see -//! [`super::catalogs`] and the per-toolkit modules. They are provider data -//! rather than contract vocabulary, they change whenever a provider does, and -//! nothing about them has to cross a frame. - -pub use tinymemory_api::composio::scopes::{ - classify_unknown, find_curated, toolkit_from_slug, CuratedTool, ToolScope, -}; - -#[cfg(test)] -#[path = "tool_scope_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/tool_scope_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/tool_scope_tests.rs deleted file mode 100644 index b5c9ffdf..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/tool_scope_tests.rs +++ /dev/null @@ -1,84 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; - -#[test] -fn classify_unknown_picks_admin_for_destructive_verbs() { - assert_eq!(classify_unknown("GMAIL_DELETE_EMAIL"), ToolScope::Admin); - assert_eq!(classify_unknown("GMAIL_TRASH_EMAIL"), ToolScope::Admin); - assert_eq!(classify_unknown("GMAIL_MODIFY_LABELS"), ToolScope::Admin); -} - -#[test] -fn classify_unknown_picks_write_for_mutating_verbs() { - assert_eq!(classify_unknown("GMAIL_SEND_EMAIL"), ToolScope::Write); - assert_eq!(classify_unknown("NOTION_CREATE_PAGE"), ToolScope::Write); - assert_eq!(classify_unknown("NOTION_UPDATE_PAGE"), ToolScope::Write); -} - -#[test] -fn classify_unknown_defaults_to_read() { - assert_eq!(classify_unknown("GMAIL_FETCH_EMAILS"), ToolScope::Read); - assert_eq!(classify_unknown("NOTION_SEARCH"), ToolScope::Read); - assert_eq!(classify_unknown("GMAIL_GET_PROFILE"), ToolScope::Read); -} - -#[test] -fn classify_unknown_admin_takes_precedence_over_write() { - // MODIFY_LABELS contains no write verb but DELETE_DRAFT does — make - // sure the admin check wins. - assert_eq!(classify_unknown("GMAIL_DELETE_DRAFT"), ToolScope::Admin); -} - -#[test] -fn toolkit_from_slug_extracts_lowercase_prefix() { - assert_eq!( - toolkit_from_slug("GMAIL_SEND_EMAIL"), - Some("gmail".to_string()) - ); - assert_eq!( - toolkit_from_slug("NOTION_FETCH_DATA"), - Some("notion".to_string()) - ); - assert_eq!(toolkit_from_slug(""), None); - assert_eq!( - toolkit_from_slug("noUnderscore"), - Some("nounderscore".into()) - ); -} - -#[test] -fn toolkit_from_slug_handles_known_multi_segment_toolkits() { - assert_eq!( - toolkit_from_slug("ZOHO_MAIL_SEND_EMAIL"), - Some("zoho_mail".to_string()) - ); - assert_eq!( - toolkit_from_slug("ONE_DRIVE_GET_FILE"), - Some("one_drive".to_string()) - ); - assert_eq!( - toolkit_from_slug("MICROSOFT_TEAMS_SEND_MESSAGE"), - Some("microsoft_teams".to_string()) - ); -} - -#[test] -fn find_curated_is_case_insensitive() { - let catalog = &[CuratedTool { - slug: "GMAIL_SEND_EMAIL", - scope: ToolScope::Write, - }]; - assert!(find_curated(catalog, "gmail_send_email").is_some()); - assert!(find_curated(catalog, "GMAIL_SEND_EMAIL").is_some()); - assert!(find_curated(catalog, "GMAIL_DELETE_EMAIL").is_none()); -} - -#[test] -fn tool_scope_serializes_lowercase() { - assert_eq!(serde_json::to_string(&ToolScope::Read).unwrap(), "\"read\""); - assert_eq!( - serde_json::to_string(&ToolScope::Admin).unwrap(), - "\"admin\"" - ); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/traits.rs b/crates/tinymemory-core/src/sync/composio/providers/traits.rs deleted file mode 100644 index df61fb15..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/traits.rs +++ /dev/null @@ -1,311 +0,0 @@ -//! The core provider trait for Composio toolkit implementations. - -use async_trait::async_trait; - -use super::tool_scope::CuratedTool; -use super::types::{ - NormalizedTask, ProviderContext, ProviderUserProfile, SyncOutcome, SyncReason, TaskContainer, - TaskFetchFilter, -}; - -/// Native provider implementation for a specific Composio toolkit. -/// -/// All methods are async and return `Result<_, String>` so the bus -/// subscriber + RPC layer can forward errors as user-visible strings -/// without `anyhow` round-tripping. -#[async_trait] -pub trait ComposioProvider: Send + Sync { - /// Toolkit slug (e.g. `"gmail"`). Must match the slug Composio / - /// the backend allowlist uses — the registry keys on this. - fn toolkit_slug(&self) -> &'static str; - - /// Suggested periodic sync interval in seconds. Return `None` to - /// opt out of the periodic scheduler entirely (e.g. for write-only - /// providers like Slack send-message). - fn sync_interval_secs(&self) -> Option { - Some(15 * 60) - } - - /// Curated whitelist of Composio actions this provider considers - /// useful for the agent, classified by [`super::tool_scope::ToolScope`]. - /// - /// When `Some(&[...])`, the meta-tool layer hides every action not - /// in this list from `composio_list_tools` and rejects execution of - /// any slug not in this list (or whose scope is disabled in the - /// user's pref). - /// - /// Default: `None` — toolkits without a curated catalog (e.g. - /// integrations not yet hand-tuned) pass through all actions and - /// rely on the [`super::tool_scope::classify_unknown`] heuristic for - /// scope gating. - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - None - } - - /// Fetch a normalized user profile for the current connection in - /// `ctx`. Most providers implement this by calling a provider - /// "get profile / about me" action via `super::super::ops::composio_execute`. - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result; - - /// Compatibility entry point backed exclusively by tinycortex. - async fn sync(&self, ctx: &ProviderContext, reason: SyncReason) -> Result { - let connection_id = ctx.connection_id.as_deref().ok_or_else(|| { - format!( - "[composio:{}] sync missing connection_id", - self.toolkit_slug() - ) - })?; - let started_at_ms = now_ms(); - let outcome = crate::sync::pipelines::host::run_composio_connection( - self.toolkit_slug(), - connection_id, - ctx.config.as_ref(), - ctx.max_items, - ctx.sync_depth_days, - ) - .await - .map_err(|error| error.to_string())?; - Ok(SyncOutcome { - toolkit: self.toolkit_slug().into(), - connection_id: Some(connection_id.into()), - reason: reason.as_str().into(), - items_ingested: outcome.records_ingested as usize, - started_at_ms, - finished_at_ms: now_ms(), - summary: outcome.note.unwrap_or_else(|| "sync completed".into()), - details: serde_json::json!({ - "more_pending": outcome.more_pending, - "actions_called": outcome.actions_called, - "provider_cost_usd": outcome.provider_cost_usd, - }), - }) - } - - /// Fetch a filtered set of work items as structured - /// [`NormalizedTask`]s — the read path that powers the - /// `task_sources` domain. - /// - /// Unlike [`Self::sync`], this does **not** persist anything into - /// the memory store; it *returns* normalized tasks so the caller can - /// enrich them and route them onto the agent's todo board. `filter` - /// is provider-agnostic — implementations read only the fields that - /// apply to their toolkit and translate them into their own action - /// slug + arguments, then map the upstream payload back into - /// `NormalizedTask`. Implementations must honour - /// [`TaskFetchFilter::effective_max`] as an upper bound on the - /// number of tasks returned. - /// - /// Default impl: `Err` — providers without a task surface (e.g. - /// gmail, slack) opt out, exactly as - /// [`Self::sync_interval_secs`] returning `None` opts out of the - /// periodic scheduler. - async fn fetch_tasks( - &self, - ctx: &ProviderContext, - filter: &TaskFetchFilter, - ) -> Result, String> { - let _ = (ctx, filter); - Err(format!( - "[composio:{}] provider has no task-fetch surface", - self.toolkit_slug() - )) - } - - /// List the selectable containers the connected account exposes — - /// today Notion databases — so the task-source UI can offer a picker - /// instead of a raw-id text field. - /// - /// Default impl: `Err` — providers without a container surface opt out, - /// mirroring [`Self::fetch_tasks`]. - async fn list_databases(&self, ctx: &ProviderContext) -> Result, String> { - let _ = ctx; - Err(format!( - "[composio:{}] provider has no database/container surface", - self.toolkit_slug() - )) - } - - /// Standardized identity callback for provider implementations. - /// - /// Providers can override this to customize how identity fragments - /// are persisted. Default behavior stores a normalized identity - /// fragment in profile facets via `skill:{source}:{identifier}:{field}` - /// keys and returns the number of facets written. - fn identity_set(&self, profile: &ProviderUserProfile) -> usize { - super::profile::persist_provider_profile(profile) - } - - /// Hook fired when an OAuth handoff completes - /// (the host's `DomainEvent::ComposioConnectionCreated`). - /// - /// Default impl: fetch and persist the user profile. Initial memory - /// ingestion is dispatched separately through tinycortex by the bus. - /// Providers can override to add provider-specific bootstrapping - /// (e.g. registering Composio triggers, seeding labels, …). - async fn on_connection_created(&self, ctx: &ProviderContext) -> Result<(), String> { - let toolkit = self.toolkit_slug(); - tracing::info!( - toolkit = %toolkit, - connection_id = ?ctx.connection_id, - "[composio:provider] on_connection_created: fetching user profile" - ); - match self.fetch_user_profile(ctx).await { - Ok(profile) => { - // PII discipline: do not log raw display_name or email. - // We log only presence indicators and the email domain - // (non-PII) so the trace is debuggable without leaking - // the user's identity. Provider-specific impls follow - // the same convention. - let has_display_name = profile.display_name.is_some(); - let has_email = profile.email.is_some(); - let email_domain = profile - .email - .as_deref() - .and_then(|e| e.split('@').nth(1)) - .map(|d| d.to_string()); - tracing::info!( - toolkit = %toolkit, - has_display_name, - has_email, - email_domain = ?email_domain, - "[composio:provider] user profile fetched" - ); - - // Persist profile fields into the local user_profile - // facet table so display_name / email / avatar are - // available to the agent context and UI without a - // round-trip to the upstream provider. - let facets = self.identity_set(&profile); - tracing::debug!( - toolkit = %toolkit, - facets_written = facets, - "[composio:provider] identity_set persisted profile facets" - ); - - // Mirror the same identity fragment into PROFILE.md so - // it lands in the agent's prompt context on the next - // turn (the facets table feeds queries; PROFILE.md - // feeds the system prompt). - if let Err(e) = super::profile_md::merge_provider_into_profile_md( - ctx.config.workspace_dir(), - &profile, - ) { - tracing::warn!( - toolkit = %toolkit, - error = %e, - "[composio:provider] PROFILE.md merge failed (non-fatal)" - ); - } - } - Err(e) => { - tracing::warn!( - toolkit = %toolkit, - error = %e, - "[composio:provider] user profile fetch failed (continuing to sync)" - ); - } - } - Ok(()) - } - - /// Hook fired immediately after a Composio action executed against - /// this toolkit returns a **successful** response. The provider may - /// mutate `data` in place to reshape the upstream payload before it - /// is handed back to the agent / RPC caller (e.g. convert Gmail's - /// HTML message bodies to markdown to save context tokens). - /// - /// `slug` is the full action slug (e.g. `"GMAIL_FETCH_EMAILS"`) so - /// providers can dispatch per action. `arguments` is the caller's - /// original argument object — providers can read opt-out flags from - /// it (e.g. `raw_html: true` to preserve raw HTML). - /// - /// Errors from upstream are not routed here; only `successful` - /// responses. Default impl is a no-op so providers that have nothing - /// to rewrite don't need to override. - fn post_process_action_result( - &self, - slug: &str, - arguments: Option<&serde_json::Value>, - data: &mut serde_json::Value, - ) { - let _ = (slug, arguments, data); - } - - /// Hook fired when a Composio trigger webhook arrives for this - /// toolkit. `payload` is the raw provider payload as forwarded by - /// the backend. Implementations should be defensive — payload - /// shapes vary across triggers. - /// - /// Default impl: log and no-op. Most providers will want to - /// override this to react to specific triggers. - async fn on_trigger( - &self, - ctx: &ProviderContext, - trigger: &str, - payload: &serde_json::Value, - ) -> Result<(), String> { - tracing::debug!( - toolkit = %self.toolkit_slug(), - trigger = %trigger, - connection_id = ?ctx.connection_id, - payload_bytes = payload.to_string().len(), - "[composio:provider] on_trigger (default no-op)" - ); - Ok(()) - } -} - -fn now_ms() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64 -} - -/// Build the env var name read by [`resolve_sync_interval_secs`] for a -/// given toolkit slug. Exposed so tests (and `.env.example`) can stay in -/// lockstep with the runtime lookup without re-implementing the casing. -pub fn sync_interval_env_var(toolkit: &str) -> String { - format!( - "OPENHUMAN_COMPOSIO_{}_SYNC_INTERVAL_SECS", - toolkit.to_ascii_uppercase() - ) -} - -/// Resolve the effective periodic sync interval (seconds) for a provider. -/// Reads `OPENHUMAN_COMPOSIO__SYNC_INTERVAL_SECS` if set; -/// otherwise returns `default_secs`. A non-positive or unparseable value -/// is rejected with a `warn` and the default is used — `0` would burn the -/// scheduler in a tight loop, so it is never honoured. -/// -/// Each provider's `sync_interval_secs()` impl calls this with its own -/// compile-time default so operators can independently slow down a -/// chatty toolkit (e.g. Slack) without rebuilding. -pub fn resolve_sync_interval_secs(toolkit: &str, default_secs: u64) -> u64 { - let key = sync_interval_env_var(toolkit); - match std::env::var(&key) { - Ok(s) => match s.trim().parse::() { - Ok(n) if n >= 1 => n, - _ => { - static WARNED: std::sync::Once = std::sync::Once::new(); - WARNED.call_once(|| { - tracing::warn!( - env = %key, - value = %s, - default = default_secs, - "[composio:provider] sync-interval env override not a positive u64; using default" - ); - }); - default_secs - } - }, - Err(_) => default_secs, - } -} - -#[cfg(test)] -#[path = "traits_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/traits_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/traits_tests.rs deleted file mode 100644 index 45bf4338..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/traits_tests.rs +++ /dev/null @@ -1,96 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; - -#[test] -fn sync_interval_env_var_uppercases_slug() { - assert_eq!( - sync_interval_env_var("slack"), - "OPENHUMAN_COMPOSIO_SLACK_SYNC_INTERVAL_SECS" - ); - assert_eq!( - sync_interval_env_var("GitHub"), - "OPENHUMAN_COMPOSIO_GITHUB_SYNC_INTERVAL_SECS" - ); -} - -/// RAII guard for env var save/restore so the test does not leak -/// state to siblings within the same process. -struct EnvGuard { - key: String, - previous: Option, -} - -impl EnvGuard { - fn set(key: &str, value: &str) -> Self { - let previous = std::env::var(key).ok(); - std::env::set_var(key, value); - Self { - key: key.to_string(), - previous, - } - } - fn unset(key: &str) -> Self { - let previous = std::env::var(key).ok(); - std::env::remove_var(key); - Self { - key: key.to_string(), - previous, - } - } -} - -impl Drop for EnvGuard { - fn drop(&mut self) { - match self.previous.take() { - Some(v) => std::env::set_var(&self.key, v), - None => std::env::remove_var(&self.key), - } - } -} - -// Bundled into a single `#[test]` so cargo's per-test parallelism -// does not race on the shared env var. Each scenario explicitly -// drops its guard before the next so the env is in a known state. -#[test] -fn resolve_sync_interval_honors_per_toolkit_env() { - let _lock = crate::test_env_lock::TEST_ENV_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - - let key = sync_interval_env_var("slack"); - let default = 15 * 60; - - // Unset → default. - let _g = EnvGuard::unset(&key); - assert_eq!(resolve_sync_interval_secs("slack", default), default); - drop(_g); - - // Valid override slows the cadence. - let _g = EnvGuard::set(&key, "3600"); - assert_eq!(resolve_sync_interval_secs("slack", default), 3600); - drop(_g); - - // Whitespace tolerated. - let _g = EnvGuard::set(&key, " 1800 "); - assert_eq!(resolve_sync_interval_secs("slack", default), 1800); - drop(_g); - - // Zero rejected (would spin the scheduler). - let _g = EnvGuard::set(&key, "0"); - assert_eq!(resolve_sync_interval_secs("slack", default), default); - drop(_g); - - // Garbage rejected. - let _g = EnvGuard::set(&key, "soon"); - assert_eq!(resolve_sync_interval_secs("slack", default), default); - drop(_g); - - // Per-toolkit scoping: a different toolkit's var does not bleed - // into slack's lookup. - let gmail_key = sync_interval_env_var("gmail"); - let _slack_unset = EnvGuard::unset(&key); - let _gmail_set = EnvGuard::set(&gmail_key, "120"); - assert_eq!(resolve_sync_interval_secs("slack", default), default); - assert_eq!(resolve_sync_interval_secs("gmail", default), 120); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/types.rs b/crates/tinymemory-core/src/sync/composio/providers/types.rs deleted file mode 100644 index 6d251b1e..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/types.rs +++ /dev/null @@ -1,206 +0,0 @@ -//! Shared types for Composio provider implementations. -//! -//! # What is here, and what moved down (#5560) -//! -//! The *values* a provider exchanges — the run report, the task envelope, the -//! normalized profile — are defined in the contract crate -//! ([`tinymemory_api::composio`]) and re-exported below at their historical -//! paths. OpenHuman names every one of them in its own signatures, so they had -//! to be reachable without a compile-time link to this crate; they are inert -//! serde data, so moving them cost nothing. -//! -//! What stayed is [`ProviderContext`], and it stayed because it is not a value: -//! it holds an `Arc`, resolves a Composio client through the host seam -//! on every call, and awaits an HTTP round-trip. None of that may enter the -//! contract crate. - -use std::sync::Arc; - -// Test-only: the tests below build a `TestHostConfig` and call -// `MemoryHostConfig` methods on it directly. Production code in this module -// goes through `crate::Config`, so neither name is needed in a non-test build. -#[cfg(test)] -use tinymemory_api::host::{test_support::TestHostConfig, MemoryHostConfig}; - -use crate::composio_host::{self, ComposioExecuteResponse}; -use crate::config_loader as config_rpc; -use crate::Config; - -/// The Composio sync vocabulary, defined in the contract crate. -/// -/// Re-exported at this path because roughly a hundred call sites here and in -/// OpenHuman already spell these `providers::SyncOutcome`, -/// `providers::NormalizedTask` and so on, and the move delivers the decoupling -/// without spending that churn. -pub use tinymemory_api::composio::{ - ComposioUsage, ComposioUsageHandle, GithubFetchMode, NormalizedTask, ProviderUserProfile, - SyncOutcome, SyncReason, TaskContainer, TaskFetchFilter, TaskKind, -}; - -/// Per-call context handed to provider methods. -/// -/// `connection_id` is `None` when a method runs in a "no specific connection" -/// mode (e.g. an across-the-board periodic sync that already iterated). For -/// per-connection paths it is always populated. -/// -/// **Mode-aware dispatch (#1710)**: pre-fix, `ProviderContext` cached a -/// pre-baked `ComposioClient` built once at construction time. Toggling -/// `composio.mode = "direct"` mid-session left provider syncs still routing -/// through the backend tinyhumans tenant. The current shape keeps an -/// [`Arc`] and resolves the underlying client per call through -/// [`ProviderContext::execute`], mirroring the agent-tool migration in the -/// host's `integrations::composio::tools::ComposioExecuteTool`. -/// -/// This is the one item in this module that is *not* contract vocabulary: a -/// context is a live handle onto the host seam, not something a frame can -/// carry. See the module docs. -#[derive(Clone)] -pub struct ProviderContext { - pub config: Arc, - pub toolkit: String, - pub connection_id: Option, - /// Accumulates Composio billable-action usage across this context's - /// lifetime. Defaulted at every construction site; only the sync path - /// (`run_connection_sync`) reads it back. Non-sync callers (agent tools, - /// task-source fetches) leave it at zero — harmless. - pub usage: ComposioUsageHandle, - /// Maximum items to fetch in a single sync pass. - /// - /// Set from the corresponding `MemorySourceEntry.max_items` field at - /// sync-dispatch time. `None` means no cap beyond the provider's own - /// internal upper bounds. - pub max_items: Option, - /// Maximum sync depth window in days. - /// - /// Set from `MemorySourceEntry.sync_depth_days`. When `Some(n)`, the - /// provider only fetches items from the last `n` days. `None` means - /// no additional depth restriction beyond the provider's cursor. - pub sync_depth_days: Option, -} - -impl ProviderContext { - /// Build a context from the current config + a toolkit slug. - /// - /// Returns `None` only when we want to short-circuit early on the - /// "user clearly not signed in" path. In the post-#1710 shape this - /// is determined by attempting a factory resolve via - /// [`composio_host::is_available`] and treating a `false` there as - /// "skip silently" — the same UX as the pre-fix - /// `build_composio_client(...).is_some()` probe, but routed - /// through the mode-aware factory so direct-mode users (no backend - /// session token, BYO key in keychain) aren't falsely treated as - /// signed-out. - pub fn from_config( - config: Arc, - toolkit: impl Into, - connection_id: Option, - ) -> Option { - // Probe the factory: any successful resolve (Backend OR Direct) - // means the user has *some* viable Composio client. Direct-mode - // users typically have no backend session token, which would - // make a `build_composio_client` probe return None and falsely - // skip them. - if composio_host::is_available(&*config) { - Some(Self { - config, - toolkit: toolkit.into(), - connection_id, - usage: ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - }) - } else { - tracing::debug!( - "[composio:provider_context] from_config: no viable Composio client; \ - treating as not-signed-in" - ); - None - } - } - - /// Resolve the underlying composio client via the mode-aware - /// factory and dispatch a single action. This is the canonical - /// way for provider implementations to execute a Composio action - /// — going through here ensures the live `composio.mode` toggle is - /// honoured on every call (#1710). - /// - /// Returns the same [`ComposioExecuteResponse`] shape that - /// `ComposioClient::execute_tool` used to return so existing - /// provider call-sites can swap `ctx.client.execute_tool(...)` for - /// `ctx.execute(...)` with no other changes. - pub async fn execute( - &self, - action: &str, - arguments: Option, - ) -> anyhow::Result { - // [#1710 Wave 4] Reload config fresh per execute so a mid-session - // `composio.mode` toggle takes effect at the very next call. The - // Arc snapshot held by `self` was taken at agent-init time - // and is otherwise stale relative to subsequent set_api_key / - // clear_api_key RPCs. - // - // Use `reload_config_snapshot_with_timeout` (anchored to the snapshot's - // `config_path`) rather than `load_config_with_timeout` (which - // re-resolves `OPENHUMAN_WORKSPACE` from the process env). The config - // path is stable for the lifetime of a `ProviderContext` — it is set - // at context creation from the agent's scoped config — so reading from - // it always reaches the correct user workspace and avoids a data-race - // in tests that share the process env. - let live_config = config_rpc::reload_config_snapshot_with_timeout(&*self.config) - .await - .map_err(|e| { - tracing::warn!( - action = %action, - toolkit = %self.toolkit, - error = %e, - "[composio:provider_context] execute: reload_config failed" - ); - anyhow::anyhow!("composio provider_context: failed to reload live config: {e}") - })?; - // Mode dispatch (backend tenant vs the user's own direct v3 tenant) - // lives in the host's `ComposioHost` impl — this side just asks. - let result = composio_host::execute( - &*live_config, - action, - arguments, - &live_config.composio().entity_id, - self.connection_id.as_deref(), - ) - .await - .map_err(|e| anyhow::anyhow!(e)); - - // Tally billable-action usage at the single chokepoint every provider - // routes through (#3111). We count any *completed* round-trip — even a - // provider-reported failure (`successful == false`) is a billable call - // — and sum the backend-reported `cost_usd`. Transport errors (the - // `Err` arm) never reached Composio, so they don't count. The lock is - // held only for the increment, never across an `.await`. - if let Ok(ref resp) = result { - if let Ok(mut usage) = self.usage.lock() { - usage.actions_called = usage.actions_called.saturating_add(1); - usage.cost_usd += resp.cost_usd; - } - } - result - } - - /// Memory client handle if the global memory singleton is ready. - /// Used by providers that want to persist sync snapshots. - /// - /// Under `cfg(test)` the global singleton is not booted, so build a - /// workspace-scoped client directly instead. - /// Memory client handle if the global memory singleton is ready. - /// Used by providers that want to persist sync snapshots. - #[cfg(not(test))] - pub fn memory_client(&self) -> Option { - crate::global::client_if_ready() - } -} - -#[cfg(test)] -#[path = "types_test_support.rs"] -mod test_support; - -#[cfg(test)] -#[path = "types_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/types_test_support.rs b/crates/tinymemory-core/src/sync/composio/providers/types_test_support.rs deleted file mode 100644 index 6d76a213..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/types_test_support.rs +++ /dev/null @@ -1,11 +0,0 @@ -//! Test-only workspace-scoped provider context behavior. - -use super::*; - -impl ProviderContext { - pub fn memory_client(&self) -> Option { - crate::store::MemoryClient::from_workspace_dir(self.config.workspace_dir().clone()) - .ok() - .map(std::sync::Arc::new) - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/types_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/types_tests.rs deleted file mode 100644 index 924c35d2..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/types_tests.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; - -/// The whole #3111 tally relies on the `usage` handle being *shared* -/// across `ProviderContext` clones: a provider's `sync` runs against a -/// clone (or the same ctx passed by `&`), accumulates via `execute`, and -/// `run_connection_sync` reads the count back from its own handle. Pin -/// that the `Arc>` is genuinely shared so a clone's increments -/// are visible from the original — if this regressed to a per-clone -/// counter, the audit cost would silently always read zero. -#[test] -fn usage_handle_is_shared_across_context_clones() { - let ctx = ProviderContext { - config: Arc::new(TestHostConfig::default()) as Arc, - toolkit: "gmail".to_string(), - connection_id: None, - usage: ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - }; - let cloned = ctx.clone(); - - // Simulate two `execute` round-trips accumulating on the clone. - { - let mut usage = cloned.usage.lock().expect("lock usage"); - usage.actions_called = usage.actions_called.saturating_add(2); - usage.cost_usd += 0.015; - } - - // The original handle must observe the clone's tally. - let observed = ctx.usage.lock().expect("lock usage"); - assert_eq!(observed.actions_called, 2); - assert!((observed.cost_usd - 0.015).abs() < 1e-9); -} - -/// `ComposioUsage` defaults to a zero tally — the value -/// `run_connection_sync` returns for a sync that fired no Composio -/// actions, and what non-sync `ProviderContext` callers carry. -#[test] -fn composio_usage_defaults_to_zero() { - let usage = ComposioUsage::default(); - assert_eq!(usage.actions_called, 0); - assert_eq!(usage.cost_usd, 0.0); -} - -// `ProviderContext::execute` and `ProviderContext::backend_client` reload -// config from `ctx.config.config_path()` (via `reload_config_snapshot_with_timeout`) -// rather than from the process-global `OPENHUMAN_WORKSPACE`. Tests -// therefore only need to persist the config to `config_path` — no env var -// manipulation required. - -#[tokio::test] -async fn provider_context_execute_backend_branch_without_session_errors_cleanly() { - // Default `Config` (mode = "backend") with no stored session - // token: the factory should return a backend-session error from - // `ctx.execute`. Verifies the backend branch is reachable and - // the error surface is sensible. - let tmp = tempfile::tempdir().expect("tempdir"); - - let mut config = TestHostConfig::default(); - config.config_path = tmp.path().join("config.toml"); - config.workspace_dir = tmp.path().join("workspace"); - config.secrets_encrypt = false; - config.save().await.expect("save fake config to disk"); - - let ctx = ProviderContext { - config: Arc::new(config) as Arc, - toolkit: "gmail".to_string(), - connection_id: None, - usage: ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - }; - let res = ctx.execute("GMAIL_FETCH_EMAILS", None).await; - let err = res.expect_err("no backend session must error"); - let msg = err.to_string(); - assert!( - msg.contains("backend") || msg.contains("session"), - "expected backend-session error, got: {msg}" - ); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/user_scopes.rs b/crates/tinymemory-core/src/sync/composio/providers/user_scopes.rs deleted file mode 100644 index 78b70872..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/user_scopes.rs +++ /dev/null @@ -1,136 +0,0 @@ -//! Per-user, per-toolkit scope preferences. -//! -//! For each Composio toolkit a user has connected (or could connect), -//! we store a [`UserScopePref`] that records whether the agent is -//! allowed to call **read**, **write**, and / or **admin**-classified -//! actions for that toolkit. Defaults are `read=true, write=true, -//! admin=false` — the agent can use the integration productively out of -//! the box, but destructive / permission-changing actions require -//! explicit opt-in. -//! -//! Storage uses the same KV surface as [`super::sync_state`] -//! (`MemoryClient::kv_get` / `kv_set`) under a dedicated namespace so -//! prefs survive process restarts without any extra file management. -//! -//! # Where the shape lives (#5560) -//! -//! [`UserScopePref`] itself is defined in the contract crate and re-exported -//! here: OpenHuman reads a preference to decide which Composio actions to show -//! the user and which to offer the agent, so the *shape* — and the defaults -//! that decide what a brand-new connection may do — has to be nameable without -//! a compile-time link to this crate. -//! -//! The three functions below stayed, because reading and writing a preference -//! is I/O against a memory client and the contract crate holds none. - -use crate::store::MemoryClientRef; - -/// The preference shape, defined in the contract crate. -/// -/// Re-exported at this path so every historical -/// `providers::user_scopes::UserScopePref` reference keeps resolving. -pub use tinymemory_api::composio::scopes::UserScopePref; - -/// KV namespace for scope prefs. Separate from `composio-sync-state` so -/// the two never collide. -const KV_NAMESPACE: &str = "composio-user-scopes"; - -fn kv_key(toolkit: &str) -> String { - toolkit.trim().to_ascii_lowercase() -} - -/// Load the scope pref for `toolkit`. Returns the default -/// (`read+write`, no `admin`) when nothing is stored or when the KV -/// store can't be reached — the agent should always be able to use -/// connected integrations productively, even if pref storage is -/// temporarily unavailable. -pub async fn load(memory: &MemoryClientRef, toolkit: &str) -> UserScopePref { - let key = kv_key(toolkit); - if key.is_empty() { - return UserScopePref::default(); - } - match memory.kv_get(Some(KV_NAMESPACE), &key).await { - Ok(Some(value)) => match serde_json::from_value::(value) { - Ok(pref) => { - tracing::debug!( - toolkit = %key, - read = pref.read, - write = pref.write, - admin = pref.admin, - "[composio][scopes] pref loaded" - ); - pref - } - Err(e) => { - tracing::warn!( - toolkit = %key, - error = %e, - "[composio][scopes] pref deserialize failed, falling back to default" - ); - UserScopePref::default() - } - }, - Ok(None) => { - tracing::debug!( - toolkit = %key, - "[composio][scopes] no pref stored, using default (read+write)" - ); - UserScopePref::default() - } - Err(e) => { - tracing::warn!( - toolkit = %key, - error = %e, - "[composio][scopes] kv_get failed, falling back to default" - ); - UserScopePref::default() - } - } -} - -/// Persist a scope pref for `toolkit`. -pub async fn save( - memory: &MemoryClientRef, - toolkit: &str, - pref: UserScopePref, -) -> Result<(), String> { - let key = kv_key(toolkit); - if key.is_empty() { - return Err("user_scopes: toolkit must not be empty".to_string()); - } - let value = serde_json::to_value(pref) - .map_err(|e| format!("[composio][scopes] serialize failed: {e}"))?; - memory.kv_set(Some(KV_NAMESPACE), &key, &value).await?; - tracing::info!( - toolkit = %key, - read = pref.read, - write = pref.write, - admin = pref.admin, - "[composio][scopes] pref saved" - ); - Ok(()) -} - -/// Best-effort load that resolves the active memory client itself. Used -/// from the meta-tool layer where we don't have a `MemoryClientRef` in -/// scope. Falls back to the default pref when memory isn't initialised. -pub async fn load_or_default(toolkit: &str) -> UserScopePref { - match crate::global::client_if_ready() { - Some(client) => load(&client, toolkit).await, - None => { - // Match the normalized key form `load()` logs so traces - // grouped by `key` correlate across both code paths. - let key = kv_key(toolkit); - tracing::debug!( - toolkit = %toolkit, - key = %key, - "[composio][scopes] memory not ready, using default pref" - ); - UserScopePref::default() - } - } -} - -#[cfg(test)] -#[path = "user_scopes_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/user_scopes_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/user_scopes_tests.rs deleted file mode 100644 index eea0890d..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/user_scopes_tests.rs +++ /dev/null @@ -1,106 +0,0 @@ -use super::*; -// `allows` is defined on the contract's type, so `ToolScope` is no longer -// imported by the module under test and has to be named here (#5560). -use super::super::tool_scope::ToolScope; -use crate::store::MemoryClient; -use std::sync::Arc; -use tempfile::TempDir; - -fn make_client() -> (TempDir, Arc) { - crate::test_seams::init(); - let tmp = TempDir::new().unwrap(); - let client = Arc::new( - MemoryClient::from_workspace_dir(tmp.path().join("workspace")) - .expect("memory client should initialize for user-scope tests"), - ); - (tmp, client) -} - -#[test] -fn default_is_read_write_no_admin() { - let p = UserScopePref::default(); - assert!(p.read); - assert!(p.write); - assert!(!p.admin); -} - -#[test] -fn allows_matches_scope() { - let p = UserScopePref { - read: true, - write: false, - admin: false, - }; - assert!(p.allows(ToolScope::Read)); - assert!(!p.allows(ToolScope::Write)); - assert!(!p.allows(ToolScope::Admin)); -} - -#[test] -fn round_trip_serde() { - let p = UserScopePref { - read: true, - write: true, - admin: true, - }; - let v = serde_json::to_value(p).unwrap(); - let back: UserScopePref = serde_json::from_value(v).unwrap(); - assert_eq!(p, back); -} - -#[test] -fn missing_fields_default_to_true_for_read_write() { - // Forward-compat: if we ever drop a field, existing stored - // documents still deserialize sensibly. - let v = serde_json::json!({}); - let p: UserScopePref = serde_json::from_value(v).unwrap(); - assert_eq!(p, UserScopePref::default()); -} - -#[tokio::test] -async fn save_and_load_round_trip_uses_normalized_toolkit_key() { - let (_tmp, client) = make_client(); - let pref = UserScopePref { - read: true, - write: false, - admin: true, - }; - - save(&client, " GMail ", pref).await.unwrap(); - - let loaded = load(&client, "gmail").await; - assert_eq!(loaded, pref); - - let raw = client - .kv_get(Some(KV_NAMESPACE), "gmail") - .await - .unwrap() - .expect("normalized toolkit key should be used"); - assert_eq!(raw.get("write").and_then(|v| v.as_bool()), Some(false)); - assert_eq!(raw.get("admin").and_then(|v| v.as_bool()), Some(true)); -} - -#[tokio::test] -async fn load_falls_back_to_default_when_stored_payload_is_invalid() { - let (_tmp, client) = make_client(); - client - .kv_set( - Some(KV_NAMESPACE), - "gmail", - &serde_json::json!("not-an-object"), - ) - .await - .unwrap(); - - let loaded = load(&client, "gmail").await; - assert_eq!(loaded, UserScopePref::default()); -} - -#[tokio::test] -async fn save_rejects_blank_toolkit() { - let (_tmp, client) = make_client(); - let err = save(&client, " ", UserScopePref::default()) - .await - .unwrap_err(); - assert!(err.contains("toolkit must not be empty")); -} diff --git a/crates/tinymemory-core/src/sync/mod.rs b/crates/tinymemory-core/src/sync/mod.rs index 089b03cb..8b043f7b 100644 --- a/crates/tinymemory-core/src/sync/mod.rs +++ b/crates/tinymemory-core/src/sync/mod.rs @@ -27,8 +27,7 @@ //! single shape to call; everything else stays local. pub mod audit; -pub mod composio; pub mod mcp; -pub mod pipelines; pub mod sync_status; +pub mod usage; pub mod workspace; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/client.rs b/crates/tinymemory-core/src/sync/pipelines/composio/client.rs deleted file mode 100644 index a3dd900e..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/client.rs +++ /dev/null @@ -1,398 +0,0 @@ -//! Minimal direct/proxied Composio action client. - -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; - -use crate::sync::pipelines::traits::{ComposioMode, ComposioSyncConfig}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ExecuteResponse { - #[serde(default)] - pub data: serde_json::Value, - #[serde(default)] - pub successful: bool, - #[serde(default)] - pub error: Option, - #[serde(rename = "costUsd", default)] - pub cost_usd: f64, - #[serde(rename = "markdownFormatted", default)] - pub markdown_formatted: Option, - #[serde(skip, default = "one_attempt")] - pub attempts: u32, -} - -fn one_attempt() -> u32 { - 1 -} - -#[derive(Debug, thiserror::Error)] -#[error("{message}")] -pub struct ExecuteError { - pub attempts: u32, - message: String, -} - -/// Time to establish a TCP/TLS connection to Composio or the proxy. -const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); -/// Whole-request ceiling. Composio actions that page a large mailbox can run -/// long, so this is generous, but it is finite. -const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120); - -#[derive(Clone)] -pub struct ComposioClient { - http: reqwest::Client, - config: ComposioSyncConfig, -} - -#[async_trait] -pub trait ActionExecutor: Send + Sync { - async fn execute( - &self, - action: &str, - arguments: serde_json::Value, - connection_id: Option<&str>, - ) -> anyhow::Result; -} - -#[async_trait] -impl ActionExecutor for ComposioClient { - async fn execute( - &self, - action: &str, - arguments: serde_json::Value, - connection_id: Option<&str>, - ) -> anyhow::Result { - ComposioClient::execute(self, action, arguments, connection_id).await - } -} - -impl ComposioClient { - /// A client with explicit connect and request timeouts. - /// - /// `reqwest::Client::new()` has none: a hung Composio or proxy connection - /// would stall the sync task indefinitely and hold the sync-state - /// mutation window open with it. The builder is fallible only on TLS - /// backend initialisation, which cannot happen with the rustls feature this - /// crate compiles; if it ever did, an untimed fallback would silently drop - /// the guarantee, so it panics loudly instead of degrading. - pub fn new(config: ComposioSyncConfig) -> Self { - let http = reqwest::Client::builder() - .connect_timeout(CONNECT_TIMEOUT) - .timeout(REQUEST_TIMEOUT) - .build() - .unwrap_or_else(|error| { - panic!("Composio HTTP client failed to build (TLS backend unavailable): {error}") - }); - Self { http, config } - } - - pub fn with_http_client(mut self, http: reqwest::Client) -> Self { - self.http = http; - self - } - - pub async fn execute( - &self, - action: &str, - arguments: serde_json::Value, - connection_id: Option<&str>, - ) -> anyhow::Result { - let action = action.trim(); - anyhow::ensure!(!action.is_empty(), "Composio action must not be empty"); - const MAX_ATTEMPTS: u32 = 3; - for attempt in 1..=MAX_ATTEMPTS { - let result = match self.config.mode { - ComposioMode::Direct => { - self.execute_direct(action, arguments.clone(), connection_id) - .await - } - ComposioMode::Proxied => self.execute_proxied(action, arguments.clone()).await, - }; - match result { - Ok(mut response) - if response.successful - || !retryable_provider_error(response.error.as_deref()) - || attempt == MAX_ATTEMPTS => - { - response.attempts = attempt; - return Ok(response); - } - Ok(_) => tracing::warn!( - action, - attempt, - "[sync:composio] retrying provider rate limit" - ), - Err(error) if retryable_transport_error(&error) && attempt < MAX_ATTEMPTS => { - tracing::warn!(action, attempt, %error, "[sync:composio] retrying transient transport failure"); - } - Err(error) => { - return Err(ExecuteError { - attempts: attempt, - message: error.to_string(), - } - .into()) - } - } - tokio::time::sleep(std::time::Duration::from_millis( - 250 * 2u64.pow(attempt - 1), - )) - .await; - } - unreachable!("retry loop always returns") - } - - async fn execute_direct( - &self, - action: &str, - arguments: serde_json::Value, - connection_id: Option<&str>, - ) -> anyhow::Result { - let key = self - .config - .api_key - .as_ref() - .filter(|key| !key.is_empty()) - .map(|key| key.expose().to_owned()) - .filter(|key| !key.trim().is_empty()) - .ok_or_else(|| anyhow::anyhow!("Composio direct API key is not configured"))?; - let url = format!( - "{}/tools/execute/{action}", - self.config.base_url.trim_end_matches('/') - ); - let mut body = serde_json::json!({ "arguments": arguments }); - if let Some(entity_id) = self - .config - .entity_id - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - { - body["user_id"] = serde_json::json!(entity_id); - } - if let Some(connection_id) = connection_id - .map(str::trim) - .filter(|value| !value.is_empty()) - { - body["connected_account_id"] = serde_json::json!(connection_id); - } - - let response = self - .http - .post(url) - .header("x-api-key", key) - .json(&body) - .send() - .await - .map_err(|error| anyhow::anyhow!("Composio direct transport error: {error}"))?; - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - anyhow::bail!(describe_failure("direct", status, &body)); - } - let raw: serde_json::Value = decode_response(response, "direct").await?; - Ok(decode_direct_response(raw)) - } - - async fn execute_proxied( - &self, - action: &str, - arguments: serde_json::Value, - ) -> anyhow::Result { - let bearer = self - .config - .bearer_token - .as_ref() - .filter(|token| !token.is_empty()) - .ok_or_else(|| anyhow::anyhow!("Composio proxy bearer token is not configured"))?; - let url = format!( - "{}/agent-integrations/composio/execute", - self.config.base_url.trim_end_matches('/') - ); - let response = self - .http - .post(url) - .bearer_auth(bearer.expose()) - .json(&serde_json::json!({ "tool": action, "arguments": arguments })) - .send() - .await - .map_err(|error| anyhow::anyhow!("Composio proxy transport error: {error}"))?; - let status = response.status(); - if !status.is_success() { - let body = response.text().await.unwrap_or_default(); - anyhow::bail!(describe_failure("proxy", status, &body)); - } - let raw: serde_json::Value = response - .json() - .await - .map_err(|error| anyhow::anyhow!("Composio proxy response decode failed: {error}"))?; - decode_proxy_response(raw) - } -} - -/// Shape a direct-API payload into an [`ExecuteResponse`]. -/// -/// A payload that reports an `error` is not a success, whatever the -/// `successful` flag says or omits: consumers gate document creation on the -/// flag, and an error body must never be stored as content. -fn decode_direct_response(raw: serde_json::Value) -> ExecuteResponse { - let flagged = raw - .get("successful") - .and_then(serde_json::Value::as_bool) - .or_else(|| raw.get("success").and_then(serde_json::Value::as_bool)) - .unwrap_or(true); - let error = raw - .get("error") - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|error| !error.is_empty()) - .map(str::to_owned); - let successful = flagged && error.is_none(); - let data = raw.get("data").cloned().unwrap_or(raw); - ExecuteResponse { - data, - successful, - error, - cost_usd: 0.0, - markdown_formatted: None, - attempts: 1, - } -} - -fn decode_proxy_response(raw: serde_json::Value) -> anyhow::Result { - let payload = if raw.get("successful").is_some() { - raw - } else { - raw.get("data").cloned().unwrap_or(raw) - }; - serde_json::from_value(payload) - .map_err(|error| anyhow::anyhow!("Composio proxy response decode failed: {error}")) -} - -fn retryable_provider_error(error: Option<&str>) -> bool { - error.is_some_and(|error| { - let lower = error.to_ascii_lowercase(); - lower.contains("ratelimit") - || lower.contains("rate limit") - || lower.contains("too many requests") - }) -} - -/// Whether a failed execute is worth retrying with backoff. -/// -/// Retryable: rate limiting and upstream unavailability (429/502/503/504), -/// and transport failures (connect/read errors, timeouts) — reported by the -/// request paths as `"… transport error: …"`. NOT retryable: any other HTTP -/// status. 400/401/403/404 are permanent — an invalid API key must fail once, -/// not storm three times — and used to be caught by a `"request failed"` -/// needle that both status-bail messages also matched. -fn retryable_transport_error(error: &anyhow::Error) -> bool { - let message = error.to_string(); - // Anchored on the status clause this module actually emits. A bare - // "HTTP 429" needle would now be forgeable: the failure message carries the - // response body, and a body that merely mentions another status must not - // turn a permanent 400 into a retry. - [ - "failed with HTTP 429", - "failed with HTTP 502", - "failed with HTTP 503", - "failed with HTTP 504", - "transport error", - ] - .iter() - .any(|needle| message.contains(needle)) -} - -/// Longest body snippet echoed for a response whose shape we do not recognise. -const FAILURE_BODY_LIMIT: usize = 400; - -/// Describe a non-2xx Composio response, using the body rather than throwing it away. -/// -/// Composio answers with a structured error whose `message` and `suggested_fix` -/// name the actual problem and how to correct it — an entity-id mismatch says -/// which id to use instead. Discarding it left callers with a bare status line -/// and no route to a fix. -/// -/// The `failed with HTTP {status}` clause is load-bearing: [`retryable_transport_error`] -/// keys off it, so it stays first and stays verbatim. -/// -/// Only the known error fields are surfaced. An unrecognised body is truncated -/// instead of echoed whole, so an unexpected payload cannot pour arbitrary -/// content into logs. -fn describe_failure(surface: &str, status: reqwest::StatusCode, body: &str) -> String { - let head = format!("Composio {surface} request failed with HTTP {status}"); - match failure_detail(body) { - Some(detail) => format!("{head}: {detail}"), - None => head, - } -} - -/// Pull the human-meaningful part out of a Composio error body. -fn failure_detail(body: &str) -> Option { - let trimmed = body.trim(); - if trimmed.is_empty() { - return None; - } - - if let Ok(parsed) = serde_json::from_str::(trimmed) { - if let Some(detail) = structured_detail(&parsed) { - return Some(detail); - } - } - - Some(truncate(trimmed, FAILURE_BODY_LIMIT)) -} - -/// `{"error": {"message": .., "slug": .., "suggested_fix": ..}}`, or a bare -/// `{"error": "..."}`. -fn structured_detail(parsed: &serde_json::Value) -> Option { - let error = parsed.get("error")?; - - if let Some(text) = error.as_str() { - let text = text.trim(); - return (!text.is_empty()).then(|| truncate(text, FAILURE_BODY_LIMIT)); - } - - let field = |name: &str| { - error - .get(name) - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - }; - - let message = field("message")?; - let mut detail = truncate(message, FAILURE_BODY_LIMIT); - if let Some(slug) = field("slug") { - detail.push_str(&format!(" [{slug}]")); - } - if let Some(fix) = field("suggested_fix") { - detail.push_str(&format!( - " — suggested fix: {}", - truncate(fix, FAILURE_BODY_LIMIT) - )); - } - Some(detail) -} - -/// Cut on a char boundary so a multi-byte body cannot panic the error path. -fn truncate(value: &str, limit: usize) -> String { - if value.chars().count() <= limit { - return value.to_owned(); - } - let kept: String = value.chars().take(limit).collect(); - format!("{kept}…") -} - -async fn decode_response( - response: reqwest::Response, - mode: &str, -) -> anyhow::Result { - response - .json() - .await - .map_err(|error| anyhow::anyhow!("Composio {mode} response decode failed: {error}")) -} - -#[cfg(test)] -#[path = "client_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/client_tests.rs b/crates/tinymemory-core/src/sync/pipelines/composio/client_tests.rs deleted file mode 100644 index 2209586b..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/client_tests.rs +++ /dev/null @@ -1,163 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; - -/// 4xx is permanent: an invalid key must fail once, not retry with -/// backoff. Only rate-limit/upstream statuses and transport failures -/// (connect/read errors, timeouts) are worth another attempt. -#[test] -fn retry_classification_is_by_status_not_by_substring() { - let retry = |m: &str| retryable_transport_error(&anyhow::anyhow!("{m}")); - assert!(retry( - "Composio direct request failed with HTTP 429 Too Many Requests" - )); - assert!(retry( - "Composio proxy request failed with HTTP 503 Service Unavailable" - )); - assert!(retry("Composio direct transport error: connection reset")); - assert!(!retry( - "Composio direct request failed with HTTP 401 Unauthorized" - )); - assert!(!retry( - "Composio proxy request failed with HTTP 404 Not Found" - )); - assert!(!retry( - "Composio direct request failed with HTTP 400 Bad Request" - )); -} - -/// An error payload is a failure even when the flag is absent or true. -#[test] -fn an_error_payload_is_never_a_success() { - let r = decode_direct_response(serde_json::json!({"error": "quota exceeded"})); - assert!(!r.successful, "missing flag + error must be a failure"); - assert_eq!(r.error.as_deref(), Some("quota exceeded")); - - let r = decode_direct_response(serde_json::json!({"successful": true, "error": " boom "})); - assert!(!r.successful, "flag=true + error must still be a failure"); - assert_eq!(r.error.as_deref(), Some("boom")); - - let r = decode_direct_response( - serde_json::json!({"successful": true, "error": " ", "data": {"x": 1}}), - ); - assert!(r.successful, "an empty error string is no error"); - assert!(r.error.is_none()); - assert_eq!(r.data["x"], 1); -} - -/// The client is built with finite timeouts; a build failure must not -/// silently degrade to an untimed client. -#[test] -fn client_builds_with_timeouts() { - let _ = ComposioClient::new(ComposioSyncConfig::default()); - assert!(CONNECT_TIMEOUT < REQUEST_TIMEOUT); -} - -#[test] -fn proxied_backend_envelope_decodes_provider_response() { - let response = decode_proxy_response(serde_json::json!({ - "success": true, - "data": { - "successful": true, - "data": {"messages": [{"messageId": "message-1"}]}, - "error": null - } - })) - .unwrap(); - - assert!(response.successful); - assert_eq!(response.data["messages"][0]["messageId"], "message-1"); -} - -#[test] -fn flat_proxy_response_remains_supported() { - let response = decode_proxy_response(serde_json::json!({ - "successful": true, - "data": {"items": [1]} - })) - .unwrap(); - - assert!(response.successful); - assert_eq!(response.data["items"], serde_json::json!([1])); -} - -/// The failure message now carries the response body, so a body that merely -/// mentions another status must not turn a permanent failure into a retry. -/// This is the hazard the needles were tightened against. -#[test] -fn a_surfaced_body_cannot_forge_a_retryable_status() { - let retry = |m: &str| retryable_transport_error(&anyhow::anyhow!("{m}")); - assert!(!retry( - "Composio direct request failed with HTTP 400 Bad Request: upstream said HTTP 503" - )); - assert!(!retry( - "Composio proxy request failed with HTTP 401 Unauthorized: retry after HTTP 429" - )); - // The real ones still classify. - assert!(retry( - "Composio direct request failed with HTTP 429 Too Many Requests: slow down" - )); -} - -/// Composio's structured error names the problem and how to fix it. This is the -/// payload from the report, verbatim. -#[test] -fn a_structured_error_body_reaches_the_message() { - let body = r#"{"error":{"message":"Connected account user ID does not match the provided user ID.","code":1812,"slug":"ActionExecute_ConnectedAccountEntityIdMismatch","status":400,"suggested_fix":"The connected_account_id you provided belongs to a different entity."}}"#; - let message = describe_failure("direct", reqwest::StatusCode::BAD_REQUEST, body); - - assert!( - message.starts_with("Composio direct request failed with HTTP 400"), - "the status clause must stay first and verbatim: {message}" - ); - assert!(message.contains("Connected account user ID does not match")); - assert!(message.contains("ActionExecute_ConnectedAccountEntityIdMismatch")); - assert!( - message.contains("belongs to a different entity"), - "the suggested fix is the part that turns a dead end into an action: {message}" - ); -} - -/// A bare `{"error": "..."}` string body is the other shape Composio returns. -#[test] -fn a_bare_error_string_body_reaches_the_message() { - let body = r#"{"error":"You have exceeded your credits limit.","tag":"NO_MORE_CREDITS"}"#; - let message = describe_failure("proxy", reqwest::StatusCode::PAYMENT_REQUIRED, body); - assert!(message.contains("exceeded your credits limit"), "{message}"); -} - -/// An unrecognised body is echoed but bounded, so an unexpected payload cannot -/// pour arbitrary content into the logs. -#[test] -fn an_unrecognised_body_is_truncated() { - let body = "x".repeat(5_000); - let message = describe_failure("direct", reqwest::StatusCode::BAD_GATEWAY, &body); - assert!( - message.contains('…'), - "expected an elision marker: {message}" - ); - assert!( - message.chars().count() < 600, - "the message grew to {} chars", - message.chars().count() - ); -} - -/// Truncation must cut on a char boundary — a multi-byte body must not panic -/// the error path. -#[test] -fn truncation_survives_multibyte_bodies() { - let body = "é".repeat(5_000); - let message = describe_failure("direct", reqwest::StatusCode::BAD_GATEWAY, &body); - assert!(message.contains('…'), "{message}"); -} - -/// No body, no change: the status line stands on its own as before. -#[test] -fn an_empty_body_leaves_the_status_line_alone() { - let message = describe_failure("direct", reqwest::StatusCode::NOT_FOUND, " "); - assert_eq!( - message, - "Composio direct request failed with HTTP 404 Not Found" - ); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/connect.rs b/crates/tinymemory-core/src/sync/pipelines/composio/connect.rs deleted file mode 100644 index 1ea09988..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/connect.rs +++ /dev/null @@ -1,364 +0,0 @@ -//! Composio v3 login/connect helpers. -//! -//! This module owns the reusable, host-agnostic pieces of the Composio -//! connection flow so a harness (or a server-side host) can drive an OAuth -//! login without re-deriving the wire contract: -//! -//! * a small per-integration **entity-id store** ([`EntityStore`]) that -//! remembers the `user_id` chosen for each toolkit across runs so re-runs -//! reuse the same Composio "entity" instead of orphaning connections; -//! * pure parsers for the connected-account **status** lifecycle; and -//! * thin async wrappers over the three v3 endpoints the connect flow needs. -//! -//! ## Verified v3 endpoints -//! -//! All confirmed against the Composio SDK source (the generated OpenAPI client -//! these SDKs wrap) — — and the v3 API -//! reference at : -//! -//! * `GET /api/v3/auth_configs?toolkit_slug={slug}` — list auth configs; -//! response `{ items: [ { id, toolkit: { slug } } ] }`. -//! (`ts/packages/core/src/models/AuthConfigs.ts`, `authConfigs.types.ts`.) -//! * `POST /api/v3/connected_accounts/link` — create a Composio Connect Link; -//! body `{ auth_config_id, user_id, callback_url? }`, response -//! `{ connected_account_id, redirect_url }`. -//! (`ConnectedAccounts.ts` `link()`, `connected_accounts.py` `link()`.) -//! * `GET /api/v3/connected_accounts/{nanoid}` — poll status; top-level -//! `status` in `INITIALIZING | INITIATED | ACTIVE | EXPIRED | FAILED | -//! REVOKED`. (`connected_accounts.py` `wait_for_connection`.) -//! -//! Authentication is the direct-mode `x-api-key` header, matching -//! [`super::client::ComposioClient`]. No secret is ever logged and error paths -//! discard raw response bodies (they can echo the key back verbatim). - -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use uuid::Uuid; - -/// Connection statuses that will never recover on their own — polling should -/// stop and fail. Mirrors the Composio SDK's `terminalErrorStates` -/// (`FAILED`, `EXPIRED`, `REVOKED`); `INACTIVE` is intentionally excluded -/// because it can transition back to `ACTIVE`. -const TERMINAL_STATUSES: &[&str] = &["FAILED", "EXPIRED", "REVOKED", "DELETED"]; - -/// Generate a fresh Composio entity id (`user_id`) for a new connection. -/// -/// The `tinycortex-` prefix keeps harness-created entities recognisable in the -/// Composio dashboard while the UUID guarantees uniqueness. -pub fn generate_entity_id() -> String { - format!("tinycortex-{}", Uuid::new_v4()) -} - -/// True when a connected-account status string means the account is live and -/// usable for tool execution. Case-insensitive. -pub fn status_is_active(status: &str) -> bool { - status.trim().eq_ignore_ascii_case("ACTIVE") -} - -/// True when a status string is a terminal failure that polling must give up -/// on. Case-insensitive. -pub fn status_is_terminal(status: &str) -> bool { - let status = status.trim(); - TERMINAL_STATUSES - .iter() - .any(|terminal| status.eq_ignore_ascii_case(terminal)) -} - -/// Pull the connected-account `status` out of a get-by-id response. -/// -/// Composio has shipped the status both at the top level and nested under -/// `state.val` / `connectionData.val`; probe the known shapes. -pub fn extract_status(record: &Value) -> Option { - [ - record.get("status"), - record.pointer("/state/val/status"), - record.pointer("/connectionData/val/status"), - record.pointer("/connection_data/val/status"), - ] - .into_iter() - .flatten() - .find_map(Value::as_str) - .map(str::trim) - .filter(|status| !status.is_empty()) - .map(str::to_owned) -} - -/// Extract the OAuth redirect URL from a create-link response. The v3 `/link` -/// endpoint returns a flat `redirect_url`; older shapes nested it under -/// `connectionData.val.redirectUrl`, so probe both. -pub fn extract_redirect_url(record: &Value) -> Option { - [ - record.get("redirect_url"), - record.get("redirectUrl"), - record.pointer("/connectionData/val/redirectUrl"), - record.pointer("/connection_data/val/redirect_url"), - ] - .into_iter() - .flatten() - .find_map(Value::as_str) - .map(str::trim) - .filter(|url| !url.is_empty()) - .map(str::to_owned) -} - -/// Extract the connected-account id from a create-link response. The v3 -/// `/link` endpoint returns `connected_account_id`; legacy `initiate` returned -/// a top-level `id`. -pub fn extract_account_id(record: &Value) -> Option { - ["connected_account_id", "connectedAccountId", "id", "nanoid"] - .iter() - .find_map(|key| record.get(key).and_then(Value::as_str)) - .map(str::trim) - .filter(|id| !id.is_empty()) - .map(str::to_owned) -} - -/// Resolve an auth-config id for `toolkit` from a `GET /auth_configs` response. -/// -/// Prefers an item whose `toolkit.slug` matches (case-insensitively); falls -/// back to the first listed config when the toolkit was already used as a -/// server-side filter and the slug field is shaped differently. -pub fn resolve_auth_config_id(list: &Value, toolkit: &str) -> Option { - let items = list - .pointer("/items") - .and_then(Value::as_array) - .or_else(|| list.get("data").and_then(Value::as_array)) - .or_else(|| list.as_array())?; - - let matches_toolkit = |item: &Value| { - [ - item.pointer("/toolkit/slug"), - item.pointer("/toolkit/name"), - item.get("toolkit"), - ] - .into_iter() - .flatten() - .find_map(Value::as_str) - .map(|slug| slug.trim().eq_ignore_ascii_case(toolkit)) - .unwrap_or(false) - }; - let auth_config_id = |item: &Value| { - ["id", "nanoid"] - .iter() - .find_map(|key| item.get(key).and_then(Value::as_str)) - .map(str::trim) - .filter(|id| !id.is_empty()) - .map(str::to_owned) - }; - - items - .iter() - .find(|item| matches_toolkit(item)) - .and_then(auth_config_id) - .or_else(|| items.iter().find_map(auth_config_id)) -} - -/// A newly-created Composio Connect Link. -#[derive(Debug, Clone)] -pub struct ConnectionLink { - /// The pending connected-account id to poll for `ACTIVE`. - pub connected_account_id: String, - /// The OAuth URL the user must open to complete login, when the scheme is - /// redirect-based. `None` for schemes that activate without a browser step. - pub redirect_url: Option, -} - -/// Persistent, per-toolkit map of the entity id (`user_id`) chosen for each -/// integration. -/// -/// Stored as a small JSON object on disk (e.g. `.composio-harness.json`) so a -/// re-run reuses the same Composio entity instead of creating a fresh — and -/// therefore orphaned — connection every time. Load is best-effort: a missing -/// or corrupt file yields an empty store rather than an error. -#[derive(Debug, Clone)] -pub struct EntityStore { - path: PathBuf, - entries: BTreeMap, -} - -#[derive(Default, Serialize, Deserialize)] -struct EntityStoreFile { - /// toolkit slug -> entity id (`user_id`). - #[serde(default)] - entities: BTreeMap, -} - -impl EntityStore { - /// Load the store from `path`, tolerating a missing or unreadable file. - pub fn load(path: impl Into) -> Self { - let path = path.into(); - let entries = std::fs::read_to_string(&path) - .ok() - .and_then(|raw| serde_json::from_str::(&raw).ok()) - .map(|file| file.entities) - .unwrap_or_default(); - Self { path, entries } - } - - /// The backing file path. - pub fn path(&self) -> &Path { - &self.path - } - - /// The entity id recorded for `toolkit`, if any. - pub fn get(&self, toolkit: &str) -> Option<&str> { - self.entries.get(toolkit).map(String::as_str) - } - - /// Record `entity_id` for `toolkit` in memory (call [`Self::save`] to - /// persist). - pub fn set(&mut self, toolkit: &str, entity_id: impl Into) { - self.entries.insert(toolkit.to_owned(), entity_id.into()); - } - - /// Resolve the entity id to use when connecting `toolkit`, persisting the - /// choice so future runs are stable. - /// - /// Precedence: a value already stored for this toolkit wins; otherwise an - /// explicit `override_id` (e.g. `COMPOSIO_ENTITY_ID`) is adopted; otherwise - /// a fresh id is generated. The resolved id is written back and saved. - pub fn entity_id_for( - &mut self, - toolkit: &str, - override_id: Option<&str>, - ) -> std::io::Result { - if let Some(existing) = self.get(toolkit) { - return Ok(existing.to_owned()); - } - let chosen = override_id - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) - .unwrap_or_else(generate_entity_id); - self.set(toolkit, chosen.clone()); - self.save()?; - Ok(chosen) - } - - /// Serialize the store to its backing file (pretty JSON). - pub fn save(&self) -> std::io::Result<()> { - let file = EntityStoreFile { - entities: self.entries.clone(), - }; - let json = serde_json::to_string_pretty(&file) - .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; - std::fs::write(&self.path, json) - } -} - -/// `GET /api/v3/auth_configs?toolkit_slug={toolkit}` — list auth configs, -/// optionally filtered to one toolkit. Returns the decoded JSON body so the -/// caller can resolve an id via [`resolve_auth_config_id`]. -pub async fn list_auth_configs( - http: &reqwest::Client, - base_url: &str, - api_key: &str, - toolkit: Option<&str>, -) -> anyhow::Result { - let mut request = http - .get(format!("{}/auth_configs", base_url.trim_end_matches('/'))) - .header("x-api-key", api_key); - if let Some(toolkit) = toolkit { - request = request.query(&[("toolkit_slug", toolkit)]); - } - let response = request - .send() - .await - .map_err(|error| anyhow::anyhow!("auth_configs request failed: {error}"))?; - let status = response.status(); - if !status.is_success() { - // Never echo the body; it can contain the key back verbatim. - let _ = response.bytes().await; - anyhow::bail!("auth_configs returned HTTP {status}"); - } - response - .json() - .await - .map_err(|error| anyhow::anyhow!("auth_configs decode failed: {error}")) -} - -/// `POST /api/v3/connected_accounts/link` — create a Composio Connect Link for -/// `auth_config_id` scoped to `user_id`. Returns the pending account id plus an -/// optional OAuth redirect URL. -pub async fn create_connection_link( - http: &reqwest::Client, - base_url: &str, - api_key: &str, - auth_config_id: &str, - user_id: &str, - callback_url: Option<&str>, -) -> anyhow::Result { - let mut body = serde_json::json!({ - "auth_config_id": auth_config_id, - "user_id": user_id, - }); - if let Some(callback_url) = callback_url - .map(str::trim) - .filter(|value| !value.is_empty()) - { - body["callback_url"] = serde_json::json!(callback_url); - } - let response = http - .post(format!( - "{}/connected_accounts/link", - base_url.trim_end_matches('/') - )) - .header("x-api-key", api_key) - .json(&body) - .send() - .await - .map_err(|error| anyhow::anyhow!("connected_accounts/link request failed: {error}"))?; - let status = response.status(); - if !status.is_success() { - let _ = response.bytes().await; - anyhow::bail!("connected_accounts/link returned HTTP {status}"); - } - let record: Value = response - .json() - .await - .map_err(|error| anyhow::anyhow!("connected_accounts/link decode failed: {error}"))?; - let connected_account_id = extract_account_id(&record).ok_or_else(|| { - anyhow::anyhow!("connected_accounts/link response missing a connected account id") - })?; - Ok(ConnectionLink { - connected_account_id, - redirect_url: extract_redirect_url(&record), - }) -} - -/// `GET /api/v3/connected_accounts/{account_id}` — fetch the current status of -/// a (possibly pending) connected account. Returns `None` if the response had -/// no recognisable status field. -pub async fn get_connection_status( - http: &reqwest::Client, - base_url: &str, - api_key: &str, - account_id: &str, -) -> anyhow::Result> { - let response = http - .get(format!( - "{}/connected_accounts/{account_id}", - base_url.trim_end_matches('/') - )) - .header("x-api-key", api_key) - .send() - .await - .map_err(|error| anyhow::anyhow!("connected_accounts/{{id}} request failed: {error}"))?; - let status = response.status(); - if !status.is_success() { - let _ = response.bytes().await; - anyhow::bail!("connected_accounts/{{id}} returned HTTP {status}"); - } - let record: Value = response - .json() - .await - .map_err(|error| anyhow::anyhow!("connected_accounts/{{id}} decode failed: {error}"))?; - Ok(extract_status(&record)) -} - -#[cfg(test)] -#[path = "connect_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/connect_tests.rs b/crates/tinymemory-core/src/sync/pipelines/composio/connect_tests.rs deleted file mode 100644 index 7e893bcd..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/connect_tests.rs +++ /dev/null @@ -1,305 +0,0 @@ -//! Unit tests for the pure Composio connect helpers: entity-id persistence, -//! status classification, and response-field extraction. No network I/O. - -use super::*; -use serde_json::json; -use wiremock::matchers::{body_partial_json, header, method, path as url_path, query_param}; -use wiremock::{Mock, MockServer, ResponseTemplate}; - -#[test] -fn generated_entity_id_is_prefixed_and_unique() { - let a = generate_entity_id(); - let b = generate_entity_id(); - assert!(a.starts_with("tinycortex-"), "unexpected id: {a}"); - assert_ne!(a, b, "two generated ids must differ"); -} - -#[test] -fn status_active_is_case_insensitive() { - assert!(status_is_active("ACTIVE")); - assert!(status_is_active("active")); - assert!(status_is_active(" Active ")); - assert!(!status_is_active("INITIATED")); - assert!(!status_is_active("FAILED")); -} - -#[test] -fn status_terminal_matches_failure_states_only() { - for terminal in ["FAILED", "expired", "Revoked", "DELETED"] { - assert!( - status_is_terminal(terminal), - "{terminal} should be terminal" - ); - } - for live in ["ACTIVE", "INITIATED", "INITIALIZING", "INACTIVE"] { - assert!(!status_is_terminal(live), "{live} should not be terminal"); - } -} - -#[test] -fn extract_status_probes_top_level_and_nested() { - assert_eq!( - extract_status(&json!({"status": "ACTIVE"})).as_deref(), - Some("ACTIVE") - ); - assert_eq!( - extract_status(&json!({"state": {"val": {"status": "INITIATED"}}})).as_deref(), - Some("INITIATED") - ); - assert_eq!( - extract_status(&json!({"connectionData": {"val": {"status": "EXPIRED"}}})).as_deref(), - Some("EXPIRED") - ); - assert_eq!(extract_status(&json!({"other": 1})), None); -} - -#[test] -fn extract_link_fields_from_v3_shape() { - let response = json!({ - "connected_account_id": "ca_abc123", - "redirect_url": "https://backend.composio.dev/oauth/start?token=xyz", - "link_token": "lt_123", - }); - assert_eq!(extract_account_id(&response).as_deref(), Some("ca_abc123")); - assert_eq!( - extract_redirect_url(&response).as_deref(), - Some("https://backend.composio.dev/oauth/start?token=xyz") - ); -} - -#[test] -fn extract_link_fields_tolerates_legacy_shape() { - let response = json!({ - "id": "ca_legacy", - "connectionData": {"val": {"status": "INITIATED", "redirectUrl": "https://x/y"}}, - }); - assert_eq!(extract_account_id(&response).as_deref(), Some("ca_legacy")); - assert_eq!( - extract_redirect_url(&response).as_deref(), - Some("https://x/y") - ); -} - -#[test] -fn resolve_auth_config_prefers_matching_toolkit() { - let list = json!({ - "items": [ - {"id": "ac_github", "toolkit": {"slug": "github"}}, - {"id": "ac_gmail", "toolkit": {"slug": "gmail"}}, - ] - }); - assert_eq!( - resolve_auth_config_id(&list, "gmail").as_deref(), - Some("ac_gmail") - ); - assert_eq!( - resolve_auth_config_id(&list, "github").as_deref(), - Some("ac_github") - ); -} - -#[test] -fn resolve_auth_config_falls_back_to_first_when_no_slug_match() { - // Server already filtered by toolkit_slug; items may not echo a slug shape - // we recognise, so fall back to the first listed config. - let list = json!({ "items": [ {"id": "ac_only"} ] }); - assert_eq!( - resolve_auth_config_id(&list, "gmail").as_deref(), - Some("ac_only") - ); - assert_eq!(resolve_auth_config_id(&json!({"items": []}), "gmail"), None); -} - -#[test] -fn entity_store_round_trips_and_reuses_ids() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join(".composio-harness.json"); - - let mut store = EntityStore::load(&path); - assert_eq!(store.get("gmail"), None); - - // First resolve for gmail with an explicit override adopts + persists it. - let gmail_id = store.entity_id_for("gmail", Some("my-entity")).unwrap(); - assert_eq!(gmail_id, "my-entity"); - - // A second toolkit with no override generates a fresh id. - let github_id = store.entity_id_for("github", None).unwrap(); - assert!(github_id.starts_with("tinycortex-")); - assert_ne!(github_id, gmail_id); - - // Reload from disk: both ids survived and are stable on re-resolve. - let mut reloaded = EntityStore::load(&path); - assert_eq!(reloaded.get("gmail"), Some("my-entity")); - assert_eq!(reloaded.get("github"), Some(github_id.as_str())); - // Stored value wins even if a different override is passed on re-run. - assert_eq!( - reloaded.entity_id_for("gmail", Some("different")).unwrap(), - "my-entity" - ); -} - -#[test] -fn entity_store_load_tolerates_missing_and_corrupt_files() { - let dir = tempfile::tempdir().unwrap(); - let missing = dir.path().join("nope.json"); - assert_eq!(EntityStore::load(&missing).get("gmail"), None); - - let corrupt = dir.path().join("corrupt.json"); - std::fs::write(&corrupt, "{ not json ").unwrap(); - assert_eq!(EntityStore::load(&corrupt).get("gmail"), None); -} - -#[tokio::test] -async fn list_auth_configs_sends_filter_and_decodes_success() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(url_path("/api/v3/auth_configs")) - .and(query_param("toolkit_slug", "gmail")) - .and(header("x-api-key", "secret")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [{"id": "ac_gmail", "toolkit": {"slug": "gmail"}}] - }))) - .mount(&server) - .await; - - let value = list_auth_configs( - &reqwest::Client::new(), - &format!("{}/api/v3/", server.uri()), - "secret", - Some("gmail"), - ) - .await - .unwrap(); - - assert_eq!( - resolve_auth_config_id(&value, "gmail").as_deref(), - Some("ac_gmail") - ); -} - -#[tokio::test] -async fn list_auth_configs_redacts_error_body_and_rejects_invalid_json() { - let failure = MockServer::start().await; - Mock::given(method("GET")) - .and(url_path("/auth_configs")) - .respond_with(ResponseTemplate::new(401).set_body_string("echoed-secret")) - .mount(&failure) - .await; - let error = list_auth_configs( - &reqwest::Client::new(), - &failure.uri(), - "echoed-secret", - None, - ) - .await - .unwrap_err() - .to_string(); - assert!(error.contains("HTTP 401")); - assert!(!error.contains("echoed-secret")); - - let malformed = MockServer::start().await; - Mock::given(method("GET")) - .and(url_path("/auth_configs")) - .respond_with(ResponseTemplate::new(200).set_body_string("not-json")) - .mount(&malformed) - .await; - assert!( - list_auth_configs(&reqwest::Client::new(), &malformed.uri(), "key", None) - .await - .unwrap_err() - .to_string() - .contains("decode failed") - ); -} - -#[tokio::test] -async fn create_connection_link_sends_trimmed_callback_and_extracts_fields() { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(url_path("/connected_accounts/link")) - .and(header("x-api-key", "secret")) - .and(body_partial_json(json!({ - "auth_config_id": "ac_1", - "user_id": "user_1", - "callback_url": "https://callback" - }))) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "connected_account_id": "ca_1", - "redirect_url": "https://oauth" - }))) - .mount(&server) - .await; - - let link = create_connection_link( - &reqwest::Client::new(), - &server.uri(), - "secret", - "ac_1", - "user_1", - Some(" https://callback "), - ) - .await - .unwrap(); - assert_eq!(link.connected_account_id, "ca_1"); - assert_eq!(link.redirect_url.as_deref(), Some("https://oauth")); -} - -#[tokio::test] -async fn create_connection_link_rejects_http_decode_and_missing_id_failures() { - for (status, body, expected) in [ - (500, "server error", "HTTP 500"), - (200, "not-json", "decode failed"), - (200, "{}", "missing a connected account id"), - ] { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(url_path("/connected_accounts/link")) - .respond_with(ResponseTemplate::new(status).set_body_string(body)) - .mount(&server) - .await; - let error = create_connection_link( - &reqwest::Client::new(), - &server.uri(), - "key", - "ac", - "user", - Some(" "), - ) - .await - .unwrap_err() - .to_string(); - assert!(error.contains(expected), "unexpected error: {error}"); - } -} - -#[tokio::test] -async fn get_connection_status_handles_success_and_safe_failures() { - let active = MockServer::start().await; - Mock::given(method("GET")) - .and(url_path("/connected_accounts/ca_1")) - .and(header("x-api-key", "key")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({"status": "ACTIVE"}))) - .mount(&active) - .await; - assert_eq!( - get_connection_status(&reqwest::Client::new(), &active.uri(), "key", "ca_1") - .await - .unwrap() - .as_deref(), - Some("ACTIVE") - ); - - for (status, body, expected) in [(403, "key", "HTTP 403"), (200, "invalid", "decode failed")] { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(url_path("/connected_accounts/ca_2")) - .respond_with(ResponseTemplate::new(status).set_body_string(body)) - .mount(&server) - .await; - let error = get_connection_status(&reqwest::Client::new(), &server.uri(), "key", "ca_2") - .await - .unwrap_err() - .to_string(); - assert!(error.contains(expected)); - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/gmail.rs b/crates/tinymemory-core/src/sync/pipelines/composio/gmail.rs deleted file mode 100644 index 21a4cb6d..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/gmail.rs +++ /dev/null @@ -1,408 +0,0 @@ -//! Incremental Gmail synchronization through Composio. - -use std::sync::Arc; - -use async_trait::async_trait; -use chrono::{DateTime, Utc}; -use serde_json::Value; - -use super::client::{ActionExecutor, ComposioClient}; -use super::orchestrator::{ - run_incremental_sync, IncrementalSource, PageFetch, SyncItem, SyncScope, -}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; -use tinymemory_sync::email_clean; -use tinymemory_sync::email_markdown::{self as email, EmailMessage, EmailThread}; - -const ACTION_FETCH_EMAILS: &str = "GMAIL_FETCH_EMAILS"; - -pub struct GmailSyncPipeline { - executor: Arc, - connection_id: String, - max_pages: usize, - page_size: usize, - query_override: Option, - /// Standing Gmail search filter (e.g. `label:brain`) ANDed onto every - /// fetch, *including* the incremental `after:` clause — unlike - /// [`Self::with_query`], which replaces the incremental clause outright - /// (backfill semantics). - filter: Option, -} - -impl GmailSyncPipeline { - /// Sync through a plain Composio client. - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self::with_executor(Arc::new(client), connection_id) - } - - /// Sync through a caller-supplied executor. - /// - /// The seam exists for host-side response reshaping: the Gmail envelope - /// rewrite (verbose MIME payload → one slim record per message, body - /// pre-rendered into `markdown`) lives in the host, above this crate, so - /// wrapping the executor is the only way it can reach the fetched page - /// before [`document`](SyncPipeline) turns it into a stored document. - pub fn with_executor( - executor: Arc, - connection_id: impl Into, - ) -> Self { - Self { - executor, - connection_id: connection_id.into(), - max_pages: 10, - // Gmail fetches full message payloads (`include_payload: true`), so a - // large page overflows Composio's tool-response size cap with HTTP - // 413. 25 full messages/request stays comfortably under it; callers - // needing more throughput can raise it via `with_limits`. - page_size: 25, - query_override: None, - filter: None, - } - } - - pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { - self.max_pages = max_pages.max(1); - self.page_size = page_size.max(1); - self - } - - pub fn with_query(mut self, query: impl Into) -> Self { - self.query_override = Some(query.into()); - self - } - - /// Set a standing Gmail search filter (e.g. `label:brain`). Every page - /// fetch ANDs it with the incremental clause (`after:` / - /// `sync_depth_days`), so background sync stays incremental while only - /// matching messages are ingested. Contrast [`Self::with_query`], which - /// *replaces* the incremental clause (backfill semantics). - pub fn with_filter(mut self, filter: impl Into) -> Self { - self.filter = Some(filter.into()); - self - } -} - -#[async_trait] -impl SyncPipeline for GmailSyncPipeline { - fn id(&self) -> &str { - "composio:gmail" - } - - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - - async fn init(&self, _config: &PipelineConfig, _context: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - - async fn tick( - &self, - _config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync( - self, - self.executor.as_ref(), - &self.connection_id, - _config, - context, - ) - .await - } -} - -#[async_trait] -impl IncrementalSource for GmailSyncPipeline { - fn toolkit(&self) -> &'static str { - "gmail" - } - - fn action(&self) -> &'static str { - ACTION_FETCH_EMAILS - } - - fn max_pages(&self) -> usize { - self.max_pages - } - fn stop_on_empty_pending(&self) -> bool { - true - } - - fn server_side_depth(&self) -> bool { - true - } - - /// Gmail pages are capped by `max_results`, and full message payloads make - /// a page's size depend on what is *in* the mail — a handful of large - /// attachments is enough for the provider to refuse 25 messages it accepted - /// yesterday. Naming the argument lets the orchestrator halve it and retry - /// rather than leaving the source stuck. - fn page_size_arg_key(&self) -> Option<&'static str> { - Some("max_results") - } - - fn arguments( - &self, - _scope: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - let mut arguments = serde_json::json!({ - "max_results": self.page_size, - "include_payload": true, - }); - if let Some(token) = page { - arguments["page_token"] = serde_json::json!(token); - } - // Gmail search ANDs space-separated clauses, so the standing filter - // (`label:brain`) composes with whichever incremental clause applies. - let mut clauses: Vec = Vec::new(); - if let Some(filter) = self.filter.as_deref() { - let filter = filter.trim(); - if !filter.is_empty() { - clauses.push(filter.to_string()); - } - } - if let Some(query) = self.query_override.as_deref() { - clauses.push(query.to_string()); - } else if let Some(cursor) = state.cursor.as_deref() { - clauses.push(format!( - "after:{}", - cursor_to_seconds(cursor).unwrap_or_default() - )); - } else if let Some(days) = config.sync_depth_days { - clauses.push(format!( - "after:{}", - (chrono::Utc::now() - chrono::Duration::days(days as i64)).timestamp() - )); - } - if !clauses.is_empty() { - arguments["query"] = Value::String(clauses.join(" ")); - } - arguments - } - - fn extract_page(&self, data: &Value, _page: Option<&str>) -> PageFetch { - PageFetch { - items: extract_messages(data), - next: extract_page_token(data), - } - } - - fn dedup_key(&self, item: &Value) -> Option { - item_id(item) - } - - fn sort_cursor(&self, item: &Value) -> Option { - item_cursor(item) - } - - async fn document( - &self, - _scope: &SyncScope, - connection_id: &str, - item: SyncItem, - _executor: &dyn ActionExecutor, - _state: &mut SyncState, - ) -> anyhow::Result { - let id = item_id(&item.raw).unwrap_or_else(|| item.dedup_key.clone()); - Ok(SkillDocument { - namespace_skill_id: "gmail".into(), - connection_id: connection_id.into(), - document_id: format!("gmail:{id}"), - title: message_title(&item.raw), - content: canonical_markdown(&item.raw, &id), - toolkit: "gmail".into(), - metadata: serde_json::json!({ - "source": "composio-provider-incremental", - "taint": "external_sync", - "message_id": id, - }), - }) - } -} - -fn extract_messages(data: &Value) -> Vec { - [ - "/data/messages", - "/messages", - "/data/data/messages", - "/data/items", - "/items", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_array)) - .cloned() - .unwrap_or_default() -} - -fn extract_page_token(data: &Value) -> Option { - [ - "/data/nextPageToken", - "/nextPageToken", - "/data/data/nextPageToken", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::trim) - .filter(|token| !token.is_empty()) - .map(str::to_owned) -} - -fn item_id(message: &Value) -> Option { - ["id", "messageId", "message_id"] - .iter() - .find_map(|key| message.get(key).and_then(Value::as_str)) - .map(str::trim) - .filter(|id| !id.is_empty()) - .map(str::to_owned) -} - -fn item_cursor(message: &Value) -> Option { - ["internalDate", "internal_date", "date"] - .iter() - .find_map(|key| message.get(key).and_then(Value::as_str)) - .map(str::trim) - .filter(|cursor| !cursor.is_empty()) - .map(str::to_owned) -} - -fn message_title(message: &Value) -> String { - ["subject", "title"] - .iter() - .find_map(|key| message.get(key).and_then(Value::as_str)) - .map(str::trim) - .filter(|title| !title.is_empty()) - .unwrap_or("Gmail message") - .to_owned() -} - -/// Render one Gmail message as canonical Markdown — the same shape the memory -/// tree ingests — rather than the provider's raw JSON. -/// -/// Storing `to_string_pretty(&item.raw)` puts a MIME tree, `Received:` headers -/// and base64 part bodies into the document: the literal words of the mail are -/// either absent or split mid-token by the chunker, so recall can never match -/// them. Routing through [`email::canonicalise`] reuses the canonicaliser the -/// tree already uses — headers as a small block, body through -/// `email_clean::clean_body` (reply chains and footer boilerplate stripped). -fn canonical_markdown(message: &Value, id: &str) -> String { - let thread = email_thread(message, id); - match email::thread_markdown(thread) { - Some(markdown) => markdown, - // The thread built here always holds exactly one message, so an empty - // thread (`None`) is unreachable in practice. Degrade to the bare body - // rather than dropping the message out of memory. - None => message_body(message), - } -} - -/// Adapt one provider message into the canonicaliser's input shape. A Gmail -/// sync item is a single message, so the thread wraps exactly one. -fn email_thread(message: &Value, id: &str) -> EmailThread { - let subject = message_title(message); - EmailThread { - provider: "gmail".into(), - thread_subject: subject.clone(), - messages: vec![EmailMessage { - from: message_sender(message), - to: message_recipients(message), - cc: Vec::new(), - subject, - sent_at: message_sent_at(message), - body: message_body(message), - source_ref: Some(format!("gmail:{id}")), - list_unsubscribe: None, - }], - } -} - -/// Body text for one message, best rendering first. -/// -/// `markdown` is what the Gmail response reshaper pins onto each message (HTML -/// stripped, URLs shortened, footers removed). `messageText` is the provider's -/// own plain-text rendering, used when the reshape did not run. `snippet` is a -/// last resort: truncated, but real prose — unlike the raw payload. -fn message_body(message: &Value) -> String { - ["markdown", "markdownFormatted", "messageText", "snippet"] - .iter() - .find_map(|key| nonempty_str(message, key)) - .unwrap_or_default() -} - -/// Sender header, rendered as `From:` and used by the canonicaliser as the -/// participant key. -fn message_sender(message: &Value) -> String { - ["from", "sender"] - .iter() - .find_map(|key| nonempty_str(message, key)) - .unwrap_or_else(|| "unknown".to_owned()) -} - -/// Recipients arrive as one comma-joined header string (some responses use an -/// array); split them so the canonicaliser can render a `To:` line. -fn message_recipients(message: &Value) -> Vec { - match message.get("to") { - Some(Value::String(header)) => header - .split(',') - .map(str::trim) - .filter(|address| !address.is_empty()) - .map(str::to_owned) - .collect(), - Some(Value::Array(values)) => values - .iter() - .filter_map(Value::as_str) - .map(str::trim) - .filter(|address| !address.is_empty()) - .map(str::to_owned) - .collect(), - _ => Vec::new(), - } -} - -/// Send time, preferring the canonicaliser's own `Value`-level date parser (it -/// already knows `date`, `internalDate`, and epoch-ms-as-string) and falling -/// back to the sync cursor. The epoch is the last resort because it is -/// *deterministic*: a message the provider dated with nothing must not rewrite -/// its own content — and so re-chunk and re-embed — on every sync. -fn message_sent_at(message: &Value) -> DateTime { - email_clean::parse_message_date(message) - .or_else(|| { - item_cursor(message) - .as_deref() - .and_then(cursor_to_seconds) - .and_then(|seconds| DateTime::from_timestamp(seconds, 0)) - }) - .unwrap_or_else(|| DateTime::from_timestamp(0, 0).expect("epoch is a valid timestamp")) -} - -/// Read `key` as a trimmed, non-empty string. Unlike a plain `get(..).as_str()` -/// chain over a candidate list, a present-but-blank field falls through to the -/// next candidate instead of ending the search. -fn nonempty_str(message: &Value, key: &str) -> Option { - message - .get(key) - .and_then(Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) -} - -fn cursor_to_seconds(cursor: &str) -> Option { - if let Ok(milliseconds) = cursor.trim().parse::() { - return Some(milliseconds / 1000); - } - chrono::DateTime::parse_from_rfc3339(cursor) - .ok() - .map(|date| date.timestamp()) -} - -#[cfg(test)] -#[path = "gmail_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/gmail_tests.rs b/crates/tinymemory-core/src/sync/pipelines/composio/gmail_tests.rs deleted file mode 100644 index e33ca480..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/gmail_tests.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! Tests for the Gmail message → canonical Markdown adapter and the -//! query-clause composition in [`GmailSyncPipeline::arguments`]. - -use std::sync::Arc; - -use serde_json::json; - -use super::GmailSyncPipeline; -use super::{canonical_markdown, message_body, message_recipients, message_sent_at}; -use crate::sync::pipelines::composio::client::{ActionExecutor, ExecuteResponse}; -use crate::sync::pipelines::composio::gmail::SyncState; -use crate::sync::pipelines::composio::orchestrator::{IncrementalSource, SyncScope}; -use crate::sync::pipelines::traits::PipelineConfig; - -/// Executor that must never run — `arguments` is pure argument-building. -struct NeverExecutor; - -#[async_trait::async_trait] -impl ActionExecutor for NeverExecutor { - async fn execute( - &self, - _action: &str, - _arguments: serde_json::Value, - _connection_id: Option<&str>, - ) -> anyhow::Result { - unreachable!("arguments() must not execute anything") - } -} - -fn query_of( - pipeline: &GmailSyncPipeline, - state: &SyncState, - config: &PipelineConfig, -) -> Option { - let args = pipeline.arguments(&SyncScope::flat(), config, state, None); - args.get("query") - .and_then(|q| q.as_str()) - .map(str::to_string) -} - -/// The standing filter ANDs with the incremental `after:` clause — -/// scoped sync stays incremental instead of re-querying the whole label. -#[test] -fn filter_composes_with_the_incremental_cursor_clause() { - let pipeline = GmailSyncPipeline::with_executor(Arc::new(NeverExecutor), "conn-1") - .with_filter("label:brain"); - let mut state = SyncState::new("gmail", "conn-1"); - state.cursor = Some("2026-05-02T09:15:00Z".into()); - - let query = query_of(&pipeline, &state, &PipelineConfig::default()).expect("query set"); - assert!(query.starts_with("label:brain after:"), "got: {query}"); -} - -/// Filter alone (no cursor, no depth cap): the query is exactly the filter. -#[test] -fn filter_alone_scopes_the_first_sync() { - let pipeline = GmailSyncPipeline::with_executor(Arc::new(NeverExecutor), "conn-1") - .with_filter("label:brain"); - let state = SyncState::new("gmail", "conn-1"); - - let query = query_of(&pipeline, &state, &PipelineConfig::default()).expect("query set"); - assert_eq!(query, "label:brain"); -} - -/// No filter, no cursor, no depth: no query argument at all (pre-existing -/// behaviour, must not regress to an empty-string query). -#[test] -fn no_clauses_means_no_query_argument() { - let pipeline = GmailSyncPipeline::with_executor(Arc::new(NeverExecutor), "conn-1"); - let state = SyncState::new("gmail", "conn-1"); - - assert_eq!( - query_of(&pipeline, &state, &PipelineConfig::default()), - None - ); -} - -/// `with_query` (backfill) still *replaces* the incremental clause, and a -/// standing filter composes in front of it. -#[test] -fn query_override_still_replaces_cursor_and_composes_with_filter() { - let pipeline = GmailSyncPipeline::with_executor(Arc::new(NeverExecutor), "conn-1") - .with_filter("label:brain") - .with_query("newer_than:3d"); - let mut state = SyncState::new("gmail", "conn-1"); - state.cursor = Some("2026-05-02T09:15:00Z".into()); - - let query = query_of(&pipeline, &state, &PipelineConfig::default()).expect("query set"); - assert_eq!( - query, "label:brain newer_than:3d", - "override wins over cursor" - ); -} - -/// One message in the shape the Gmail response reshaper emits: a slim envelope -/// whose body is pre-rendered into `markdown`. -fn slim_message() -> serde_json::Value { - json!({ - "id": "18f0abc", - "threadId": "18f0abc", - "subject": "Boulder visit", - "from": "Advising ", - "to": "me@example.com, second@example.com", - "date": "2026-05-02T09:15:00Z", - "labels": ["INBOX"], - "markdown": "The University of Colorado orientation is on May 20.\n\nOn Fri, 1 May 2026, someone wrote:\n> please ignore this quoted reply", - }) -} - -#[test] -fn canonical_markdown_renders_headers_and_cleaned_body() { - let content = canonical_markdown(&slim_message(), "18f0abc"); - - // The literal words of the mail — the thing recall has to match — are - // present as prose, and the headers are readable rather than a MIME tree. - assert!( - content.contains("The University of Colorado orientation is on May 20."), - "body text must survive canonicalisation: {content}" - ); - assert!( - content.contains("From: Advising "), - "{content}" - ); - assert!(content.contains("Subject: Boulder visit"), "{content}"); - assert!( - content.contains("To: me@example.com, second@example.com"), - "{content}" - ); - - // Canonicalisation is what strips the quoted reply chain. - assert!( - !content.contains("please ignore this quoted reply"), - "reply chain must be stripped by clean_body: {content}" - ); - - // Nothing JSON-shaped is left: this is the regression the fix exists for. - assert!( - !content.contains("\"markdown\""), - "raw JSON must not be stored: {content}" - ); - assert!( - !content.contains("threadId"), - "envelope keys must not be stored: {content}" - ); -} - -#[test] -fn body_falls_back_to_message_text_when_the_reshape_did_not_run() { - // No `markdown` field — the provider's own plain text is used instead. - let raw = json!({ - "id": "18f0def", - "subject": "Direct", - "messageText": "Plain provider text about Colorado.", - }); - assert_eq!(message_body(&raw), "Plain provider text about Colorado."); - assert!(canonical_markdown(&raw, "18f0def").contains("Plain provider text about Colorado.")); -} - -#[test] -fn body_skips_a_present_but_blank_field() { - // A blank `markdown` must not shadow a usable `messageText`: the candidate - // list falls through on emptiness, not just on absence. - let raw = json!({ "markdown": " ", "messageText": "real body" }); - assert_eq!(message_body(&raw), "real body"); -} - -#[test] -fn recipients_split_from_either_a_header_string_or_an_array() { - let joined = json!({ "to": "a@x.com, b@y.com" }); - assert_eq!(message_recipients(&joined), vec!["a@x.com", "b@y.com"]); - - let array = json!({ "to": ["a@x.com", " b@y.com "] }); - assert_eq!(message_recipients(&array), vec!["a@x.com", "b@y.com"]); - - assert!(message_recipients(&json!({})).is_empty()); -} - -#[test] -fn sent_at_reads_epoch_millis_and_is_deterministic_when_undated() { - // Gmail's `internalDate` is epoch millis as a string. - let dated = json!({ "internalDate": "1777712100000" }); - assert_eq!(message_sent_at(&dated).timestamp(), 1_777_712_100); - - // Undated messages must resolve to the same value every sync, otherwise the - // rendered `Date:` header changes and the document re-chunks forever. - let undated = json!({ "subject": "no date anywhere" }); - assert_eq!(message_sent_at(&undated), message_sent_at(&undated)); - assert_eq!(message_sent_at(&undated).timestamp(), 0); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/mod.rs b/crates/tinymemory-core/src/sync/pipelines/composio/mod.rs deleted file mode 100644 index 6c8645b6..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Composio sync, engine-free: HTTP client, connection lifecycle, the -//! incremental-sync orchestrator, and one pipeline per toolkit. - -pub mod client; -pub mod connect; -pub mod gmail; -pub mod orchestrator; -pub(crate) mod page_size; -pub mod providers; - -pub use client::{ActionExecutor, ComposioClient, ExecuteError, ExecuteResponse}; -pub use connect::{ - create_connection_link, generate_entity_id, get_connection_status, list_auth_configs, - resolve_auth_config_id, status_is_active, status_is_terminal, ConnectionLink, EntityStore, -}; -pub use gmail::GmailSyncPipeline; -pub use orchestrator::{run_incremental_sync, IncrementalSource, PageFetch, SyncItem, SyncScope}; -pub use providers::{ - ClickUpSyncPipeline, GitHubSyncPipeline, GoogleCalendarSyncPipeline, GoogleDocsSyncPipeline, - GoogleDriveSyncPipeline, GoogleSheetsSyncPipeline, LinearSyncPipeline, NotionSyncPipeline, - OutlookSyncPipeline, SlackSearchBackfillPipeline, SlackSyncPipeline, TodoistSyncPipeline, -}; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator.rs b/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator.rs deleted file mode 100644 index f036e31b..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator.rs +++ /dev/null @@ -1,538 +0,0 @@ -//! Shared bounded incremental synchronization control flow. - -use async_trait::async_trait; -use serde_json::Value; - -use super::client::ActionExecutor; -use super::page_size::{apply_page_size, is_payload_too_large, shrink_page_size}; -use crate::sync::composio::providers::sync_state::{PersistedSyncState, SyncState}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncEvent, SyncOutcome, SyncRunError, SyncStage, -}; - -#[derive(Debug)] -pub struct PageFetch { - pub items: Vec, - pub next: Option, -} - -#[derive(Debug)] -pub struct SyncItem { - pub dedup_key: String, - pub sort_cursor: Option, - pub raw: Value, -} - -#[derive(Clone, Debug, Default)] -pub struct SyncScope { - pub id: String, - pub label: String, - pub metadata: Value, -} - -impl SyncScope { - pub fn flat() -> Self { - Self::default() - } - - pub fn named(id: impl Into, label: impl Into) -> Self { - Self { - id: id.into(), - label: label.into(), - metadata: Value::Null, - } - } - - pub fn with_metadata(mut self, metadata: Value) -> Self { - self.metadata = metadata; - self - } -} - -#[async_trait] -pub trait IncrementalSource: Send + Sync { - fn toolkit(&self) -> &'static str; - fn action(&self) -> &'static str; - fn max_pages(&self) -> usize { - 10 - } - fn per_scope_cursors(&self) -> bool { - false - } - fn tolerate_scope_errors(&self) -> bool { - false - } - fn retain_dedup_keys(&self) -> bool { - true - } - fn stop_on_empty_pending(&self) -> bool { - false - } - fn server_side_depth(&self) -> bool { - false - } - fn depth_floor(&self, config: &PipelineConfig, state: &SyncState) -> Option { - if state.cursor.is_some() { - return None; - } - config.sync_depth_days.map(|days| { - (chrono::Utc::now() - chrono::Duration::days(days as i64)) - .format("%Y-%m-%dT%H:%M:%SZ") - .to_string() - }) - } - fn advance_scope_cursor(&self, _state: &mut SyncState, _scope: &SyncScope, _cursor: &str) {} - async fn scopes( - &self, - _executor: &dyn ActionExecutor, - _connection_id: &str, - _state: &mut SyncState, - ) -> anyhow::Result> { - Ok(vec![SyncScope::flat()]) - } - /// Name of the argument that caps how many items one page requests - /// (`max_results` for Gmail), when the action has one. - /// - /// Returning `Some` opts the source into the too-large-page retry: a page - /// the provider refuses purely for size is re-requested with the cap - /// halved, instead of failing the whole run. A source whose action has no - /// such knob returns `None` and keeps the previous behaviour. - fn page_size_arg_key(&self) -> Option<&'static str> { - None - } - fn arguments( - &self, - scope: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value; - fn extract_page(&self, data: &Value, page: Option<&str>) -> PageFetch; - fn dedup_key(&self, item: &Value) -> Option; - fn sort_cursor(&self, item: &Value) -> Option; - async fn document( - &self, - scope: &SyncScope, - connection_id: &str, - item: SyncItem, - executor: &dyn ActionExecutor, - state: &mut SyncState, - ) -> anyhow::Result; -} - -pub async fn run_incremental_sync( - source: &dyn IncrementalSource, - executor: &dyn ActionExecutor, - connection_id: &str, - config: &PipelineConfig, - context: &SyncContext, -) -> anyhow::Result { - let toolkit = source.toolkit(); - emit(context, toolkit, connection_id, SyncStage::Fetching, None).await; - tracing::debug!(toolkit, connection_id, "[sync:orchestrator] sync starting"); - - let mut state = SyncState::load(context.state.as_ref(), toolkit, connection_id).await?; - if state.budget_exhausted() { - tracing::debug!( - toolkit, - connection_id, - "[sync:orchestrator] daily budget exhausted" - ); - return Ok(SyncOutcome { - note: Some("daily request budget exhausted".into()), - ..SyncOutcome::default() - }); - } - - let result = match source.scopes(executor, connection_id, &mut state).await { - Ok(scopes) => { - run_pages( - source, - executor, - connection_id, - config, - context, - &mut state, - &scopes, - ) - .await - } - Err(error) => Err(error), - }; - state.last_sync_at_ms = Some(now_ms()); - if let Err(error) = state.save(context.state.as_ref()).await { - emit( - context, - toolkit, - connection_id, - SyncStage::Failed, - Some("sync state persistence failed".into()), - ) - .await; - return Err(error); - } - - match result { - Ok(outcome) => { - emit( - context, - toolkit, - connection_id, - SyncStage::Stored, - Some(format!("{} records", outcome.records_ingested)), - ) - .await; - emit(context, toolkit, connection_id, SyncStage::Completed, None).await; - tracing::debug!( - toolkit, - connection_id, - records = outcome.records_ingested, - more_pending = outcome.more_pending, - "[sync:orchestrator] sync completed" - ); - Ok(outcome) - } - Err(error) => { - tracing::warn!(toolkit, connection_id, %error, "[sync:orchestrator] sync failed"); - emit( - context, - toolkit, - connection_id, - SyncStage::Failed, - Some(error.to_string()), - ) - .await; - Err(SyncRunError::new( - error.to_string(), - state.run_requests, - state.run_provider_cost_usd, - ) - .into()) - } - } -} - -async fn run_pages( - source: &dyn IncrementalSource, - executor: &dyn ActionExecutor, - connection_id: &str, - config: &PipelineConfig, - context: &SyncContext, - state: &mut SyncState, - scopes: &[SyncScope], -) -> anyhow::Result { - let mut newest_cursor = state.cursor.clone(); - let mut ingested = 0u32; - // Estimated tokens of stored content this run, for `max_tokens_per_sync`. - let mut tokens_ingested: u64 = 0; - let mut more_pending = false; - let depth_floor = (!source.server_side_depth()) - .then(|| source.depth_floor(config, state)) - .flatten(); - - // Once a page proves too large for the provider, every later page of this - // run asks for the smaller size straight away rather than paying a rejected - // round-trip to rediscover the same limit. - let mut page_size_override: Option = None; - - 'scopes: for scope in scopes { - let mut page_token = None; - let mut scope_newest_cursor: Option = None; - let mut scope_failed = false; - 'pages: for page_index in 0..source.max_pages().max(1) { - if state.budget_exhausted() { - more_pending = true; - break 'scopes; - } - let mut arguments = source.arguments(scope, config, state, page_token.as_deref()); - apply_page_size( - &mut arguments, - source.page_size_arg_key(), - page_size_override, - ); - // The size a shrink most recently *tried*. Promoted to the run's - // sticky override only once the provider accepts a page at it — - // before that it names a size that may itself be refused. - let mut attempted_page_size: Option = None; - let response = loop { - let response = match executor - .execute(source.action(), arguments.clone(), Some(connection_id)) - .await - { - Ok(response) => response, - Err(error) if source.tolerate_scope_errors() => { - if let Some(execute_error) = - error.downcast_ref::() - { - state.record_requests(execute_error.attempts); - } - tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] scope fetch failed; continuing"); - scope_failed = true; - break 'pages; - } - Err(error) => { - if let Some(execute_error) = - error.downcast_ref::() - { - state.record_requests(execute_error.attempts); - } - return Err(error); - } - }; - // A completed provider round-trip is billable even when its - // envelope reports failure. Transport failures return before - // this point. - state.record_action(response.attempts, response.cost_usd); - if response.successful { - // Sticky for the rest of the run, and set HERE rather than - // at the shrink: with several halvings (25 → 12 → 6) an - // assignment per attempt leaves the last *rejected* size in - // the override on every step but the final one, and is - // correct at the end only because that step happens to be - // the accepted one. Recording the accepted size makes the - // intent independent of the retry order. - if let Some(accepted) = attempted_page_size { - page_size_override = Some(accepted); - } - break response; - } - // A page refused purely for its size is the one provider - // failure a *smaller request* can fix, so shrink and retry - // instead of failing the run. Without this a single oversized - // page stops the source dead until someone notices: on one live - // workspace Gmail sync sat broken for nine days that way. - if is_payload_too_large(response.error.as_deref()) { - if state.budget_exhausted() { - more_pending = true; - break 'scopes; - } - if let Some(reduced) = - shrink_page_size(&mut arguments, source.page_size_arg_key()) - { - attempted_page_size = Some(reduced); - tracing::warn!( - toolkit = source.toolkit(), - connection_id, - scope = %scope.label, - reduced_page_size = reduced, - "[sync:orchestrator] provider refused the page as too large; retrying with a smaller page" - ); - continue; - } - } - let error = anyhow::anyhow!( - "{} provider failure: {}", - source.toolkit(), - response - .error - .unwrap_or_else(|| "unknown provider error".into()) - ); - if source.tolerate_scope_errors() { - tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] provider rejected scope; continuing"); - scope_failed = true; - break 'pages; - } - return Err(error); - }; - - let fetched = source.extract_page(&response.data, page_token.as_deref()); - let mut reached_cursor_boundary = false; - let mut saw_unsynced_item = false; - for raw in fetched.items { - if config.max_items.is_some_and(|limit| ingested >= limit) { - more_pending = true; - break 'scopes; - } - if state.budget_exhausted() { - more_pending = true; - break 'scopes; - } - // Per-source spend caps (#18): checked with the same "stop, - // leave the rest pending" contract as `max_items`, so a - // capped run resumes from its cursor next tick. - if config - .max_cost_per_sync_usd - .is_some_and(|cap| state.run_provider_cost_usd >= cap) - { - more_pending = true; - break 'scopes; - } - if config - .max_tokens_per_sync - .is_some_and(|cap| tokens_ingested >= cap) - { - more_pending = true; - break 'scopes; - } - let Some(dedup_key) = source.dedup_key(&raw) else { - continue; - }; - if state.is_synced(&dedup_key) { - continue; - } - saw_unsynced_item = true; - let sort_cursor = source.sort_cursor(&raw); - if sort_cursor - .as_deref() - .zip(depth_floor.as_deref()) - .is_some_and(|(item_cursor, floor)| item_cursor < floor) - { - reached_cursor_boundary = true; - break; - } - if !source.per_scope_cursors() - && sort_cursor - .as_deref() - .zip(state.cursor.as_deref()) - .is_some_and(|(item_cursor, persisted_cursor)| { - item_cursor <= persisted_cursor - }) - { - tracing::debug!( - toolkit = source.toolkit(), - connection_id, - scope = %scope.label, - "[sync:orchestrator] reached persisted cursor boundary" - ); - reached_cursor_boundary = true; - break; - } - let document = match source - .document( - scope, - connection_id, - SyncItem { - dedup_key: dedup_key.clone(), - sort_cursor: sort_cursor.clone(), - raw, - }, - executor, - state, - ) - .await - { - Ok(document) => document, - Err(error) if source.tolerate_scope_errors() => { - tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] scope document conversion failed; continuing"); - scope_failed = true; - break; - } - Err(error) => return Err(error), - }; - // Same rough estimate the tree's budgeting uses (~4 chars per - // token); a cap, not an invoice. - tokens_ingested = - tokens_ingested.saturating_add((document.content.len() / 4) as u64); - if let Err(error) = context.documents.store(document).await { - // Scope tolerance exists for per-item flakiness. A corrupt - // store is not that: every later item fails identically in - // every scope, so tolerating it here re-buys the - // openhuman#5820 flood one scope at a time. Corruption - // always aborts the run. - if source.tolerate_scope_errors() - && !crate::corruption::is_sqlite_corrupt(&error) - { - tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] scope document store failed; continuing"); - scope_failed = true; - break; - } - return Err(error); - } - if source.retain_dedup_keys() { - state.mark_synced(dedup_key); - } - if let Some(cursor) = sort_cursor { - let target = if source.per_scope_cursors() { - &mut scope_newest_cursor - } else { - &mut newest_cursor - }; - if target - .as_deref() - .is_none_or(|current| cursor.as_str() > current) - { - *target = Some(cursor); - } - } - ingested = ingested.saturating_add(1); - if config.max_items.is_some_and(|limit| ingested >= limit) { - more_pending = true; - break 'scopes; - } - } - - page_token = fetched.next; - if source.stop_on_empty_pending() && !saw_unsynced_item { - tracing::debug!( - toolkit = source.toolkit(), - connection_id, - scope = %scope.label, - "[sync:orchestrator] stopping after all-deduplicated page" - ); - break; - } - if reached_cursor_boundary { - break; - } - if page_token.is_none() { - break; - } - if page_index + 1 == source.max_pages().max(1) { - more_pending = true; - } - } - if source.per_scope_cursors() && !scope_failed && !more_pending { - if let Some(cursor) = scope_newest_cursor.as_deref() { - source.advance_scope_cursor(state, scope, cursor); - state.save(context.state.as_ref()).await?; - } - } - } - - if !source.per_scope_cursors() && !more_pending { - if let Some(cursor) = newest_cursor { - state.advance_cursor(cursor); - } - } - Ok(SyncOutcome { - records_ingested: ingested, - more_pending, - actions_called: state.run_requests, - provider_cost_usd: state.run_provider_cost_usd, - note: None, - // Stamped by the runner from the sink's counter; the orchestrator only - // sees store()'s Ok/Err and the tolerated failures return Ok. - tree_ingest_failures: 0, - }) -} - -async fn emit( - context: &SyncContext, - toolkit: &str, - connection_id: &str, - stage: SyncStage, - message: Option, -) { - let _ = context - .events - .emit(SyncEvent { - source_id: format!("composio:{toolkit}:{connection_id}"), - toolkit: toolkit.into(), - connection_id: Some(connection_id.into()), - stage, - message, - }) - .await; -} - -fn now_ms() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64 -} - -#[cfg(test)] -#[path = "orchestrator_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator_tests.rs b/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator_tests.rs deleted file mode 100644 index ba3f46af..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator_tests.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! Tests for the too-large-page retry. - -use std::sync::{Arc, Mutex}; - -use serde_json::json; - -use super::*; -use crate::sync::composio::providers::sync_state::SyncStateStore; -use crate::sync::pipelines::composio::client::ExecuteResponse; -use crate::sync::pipelines::traits::{SkillDocSink, SkillDocument, SyncEvent, SyncEventSink}; - -/// Executor that mimics a provider with a response-size ceiling: it refuses any -/// page asking for more than `accepts` items and records every size it was -/// asked for. -struct SizeLimitedExecutor { - accepts: u64, - requested: Mutex>, -} - -impl SizeLimitedExecutor { - fn new(accepts: u64) -> Self { - Self { - accepts, - requested: Mutex::new(Vec::new()), - } - } - - fn requested_sizes(&self) -> Vec { - self.requested.lock().unwrap().clone() - } -} - -#[async_trait] -impl ActionExecutor for SizeLimitedExecutor { - async fn execute( - &self, - _action: &str, - arguments: Value, - _connection_id: Option<&str>, - ) -> anyhow::Result { - let requested = arguments - .get("max_results") - .and_then(Value::as_u64) - .unwrap_or(0); - self.requested.lock().unwrap().push(requested); - - let mut response: ExecuteResponse = serde_json::from_value(json!({})).unwrap(); - if requested > self.accepts { - response.successful = false; - response.error = Some( - "413 {\"error\":{\"message\":\"The tool response payload is too large.\",\ - \"code\":1613,\"slug\":\"Upstream_PayloadTooLarge\"}}" - .to_string(), - ); - return Ok(response); - } - response.successful = true; - response.data = json!({ - "messages": [{ "id": format!("m{requested}"), "date": "1700000000000" }], - }); - Ok(response) - } -} - -/// Minimal paged source: one page, one item, page size declared as -/// `max_results` unless `declare_page_size` is off. -struct StubSource { - declare_page_size: bool, - initial_page_size: u64, -} - -#[async_trait] -impl IncrementalSource for StubSource { - fn toolkit(&self) -> &'static str { - "stub" - } - fn action(&self) -> &'static str { - "STUB_FETCH" - } - fn max_pages(&self) -> usize { - 1 - } - fn page_size_arg_key(&self) -> Option<&'static str> { - self.declare_page_size.then_some("max_results") - } - fn arguments( - &self, - _scope: &SyncScope, - _config: &PipelineConfig, - _state: &SyncState, - _page: Option<&str>, - ) -> Value { - json!({ "max_results": self.initial_page_size }) - } - fn extract_page(&self, data: &Value, _page: Option<&str>) -> PageFetch { - PageFetch { - items: data - .get("messages") - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(), - next: None, - } - } - fn dedup_key(&self, item: &Value) -> Option { - item.get("id").and_then(Value::as_str).map(str::to_string) - } - fn sort_cursor(&self, item: &Value) -> Option { - item.get("date").and_then(Value::as_str).map(str::to_string) - } - async fn document( - &self, - _scope: &SyncScope, - connection_id: &str, - item: SyncItem, - _executor: &dyn ActionExecutor, - _state: &mut SyncState, - ) -> anyhow::Result { - Ok(SkillDocument { - namespace_skill_id: "stub".into(), - connection_id: connection_id.into(), - document_id: item.dedup_key, - title: "stub".into(), - content: "stub".into(), - toolkit: "stub".into(), - metadata: Value::Null, - }) - } -} - -#[derive(Default)] -struct NoopHost(Mutex>); - -#[async_trait] -impl SkillDocSink for NoopHost { - async fn store(&self, _: SkillDocument) -> anyhow::Result<()> { - Ok(()) - } - async fn delete(&self, _: &str, _: &str) -> anyhow::Result<()> { - Ok(()) - } -} - -#[async_trait] -impl SyncEventSink for NoopHost { - async fn emit(&self, _: SyncEvent) -> anyhow::Result<()> { - Ok(()) - } -} - -#[async_trait] -impl SyncStateStore for NoopHost { - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - Ok(self - .0 - .lock() - .unwrap() - .get(&format!("{namespace}:{key}")) - .cloned()) - } - async fn set(&self, namespace: &str, key: &str, value: &Value) -> anyhow::Result<()> { - self.0 - .lock() - .unwrap() - .insert(format!("{namespace}:{key}"), value.clone()); - Ok(()) - } -} - -fn context() -> SyncContext { - let host = Arc::new(NoopHost::default()); - SyncContext { - events: host.clone(), - documents: host.clone(), - state: host, - } -} - -#[tokio::test] -async fn an_oversized_page_is_halved_until_the_provider_accepts_it() { - // The provider takes 6 at a time; the source asks for 25. - let executor = SizeLimitedExecutor::new(6); - let source = StubSource { - declare_page_size: true, - initial_page_size: 25, - }; - - let outcome = run_incremental_sync( - &source, - &executor, - "conn-1", - &PipelineConfig::default(), - &context(), - ) - .await - .expect("a too-large page must not fail the run"); - - assert_eq!( - outcome.records_ingested, 1, - "the page is ingested after the retry" - ); - assert_eq!( - executor.requested_sizes(), - vec![25, 12, 6], - "each rejection halves the request until it fits" - ); -} - -#[tokio::test] -async fn a_source_without_a_page_size_argument_still_fails_fast() { - // No `page_size_arg_key` — there is nothing to shrink, so the old - // behaviour (surface the provider failure) must be preserved rather than - // looping on a request that can never change. - let executor = SizeLimitedExecutor::new(6); - let source = StubSource { - declare_page_size: false, - initial_page_size: 25, - }; - - let error = run_incremental_sync( - &source, - &executor, - "conn-1", - &PipelineConfig::default(), - &context(), - ) - .await - .expect_err("an unshrinkable too-large page is still a failure"); - - assert!(error.to_string().contains("provider failure"), "{error}"); - assert_eq!( - executor.requested_sizes(), - vec![25], - "no pointless retry of an identical request" - ); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/page_size.rs b/crates/tinymemory-core/src/sync/pipelines/composio/page_size.rs deleted file mode 100644 index d970493b..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/page_size.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! Page-size retry for a provider that refuses a page for its size. -//! -//! A page rejected purely because the response is too big is the one provider -//! failure a *smaller request* can fix. The orchestrator halves the page-size -//! argument and retries rather than failing the source, so a single oversized -//! page cannot stop a sync dead — on one live workspace a Gmail sync sat broken -//! for nine days that way. - -use serde_json::Value; - -/// Smallest page a shrink will ask for. One item is the point past which a -/// too-large response is about that single item, not the batch size. -pub(super) const MIN_PAGE_SIZE: u64 = 1; - -/// Whether the provider refused a page for its *size* rather than for anything -/// about the request's content — the only failure a smaller page can fix. -/// -/// Matched on the error text because that is all the envelope carries: Composio -/// reports it as HTTP 413 with a `Upstream_PayloadTooLarge` slug, and other -/// backends phrase it as "payload too large" / "response too large". -pub(super) fn is_payload_too_large(error: Option<&str>) -> bool { - error.is_some_and(|error| { - let lower = error.to_ascii_lowercase(); - lower.contains("payloadtoolarge") - || lower.contains("payload_too_large") - || mentions_status_413(&lower) - || (lower.contains("too large") - && (lower.contains("payload") || lower.contains("response"))) - }) -} - -/// Whether `text` names HTTP 413 as a status code. -/// -/// The digits have to stand alone. An unanchored `contains("413")` also matches -/// a message id, an amount, or a timestamp that merely contains those three -/// digits, and every such match costs a shrink-and-retry cycle before the real -/// error is finally surfaced — on a failure that a smaller page was never going -/// to fix. -fn mentions_status_413(lower: &str) -> bool { - lower.match_indices("413").any(|(at, _)| { - let before_is_digit = lower[..at] - .chars() - .next_back() - .is_some_and(|c| c.is_ascii_digit()); - let after_is_digit = lower[at + 3..] - .chars() - .next() - .is_some_and(|c| c.is_ascii_digit()); - !before_is_digit && !after_is_digit - }) -} - -/// Pin the page-size argument to `size`, if the source declared one. -pub(super) fn apply_page_size(arguments: &mut Value, key: Option<&str>, size: Option) { - if let (Some(key), Some(size)) = (key, size) { - if let Some(slot) = arguments.get_mut(key) { - *slot = Value::from(size); - } - } -} - -/// Halve the page-size argument in place, returning the new value. -/// -/// `None` means retrying is pointless — the source declares no page-size -/// argument, this request does not carry it, or it is already at the floor. -pub(super) fn shrink_page_size(arguments: &mut Value, key: Option<&str>) -> Option { - let key = key?; - let current = arguments.get(key)?.as_u64()?; - if current <= MIN_PAGE_SIZE { - return None; - } - let reduced = (current / 2).max(MIN_PAGE_SIZE); - arguments[key] = Value::from(reduced); - Some(reduced) -} - -#[cfg(test)] -#[path = "page_size_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/page_size_tests.rs b/crates/tinymemory-core/src/sync/pipelines/composio/page_size_tests.rs deleted file mode 100644 index 6328c7b0..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/page_size_tests.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Tests for the page-size retry helpers. - -use serde_json::json; - -use super::*; - -#[test] -fn payload_too_large_is_told_apart_from_other_provider_errors() { - assert!(is_payload_too_large(Some( - "413 {\"slug\":\"Upstream_PayloadTooLarge\"}" - ))); - assert!(is_payload_too_large(Some("Response too large for tool"))); - assert!(!is_payload_too_large(Some("rate limit exceeded"))); - assert!(!is_payload_too_large(Some("invalid grant"))); - assert!(!is_payload_too_large(None)); -} - -#[test] -fn shrinking_stops_at_the_floor() { - let mut arguments = json!({ "max_results": 3 }); - assert_eq!( - shrink_page_size(&mut arguments, Some("max_results")), - Some(1) - ); - assert_eq!(arguments["max_results"], json!(1)); - // At one item per page there is nothing left to halve. - assert_eq!(shrink_page_size(&mut arguments, Some("max_results")), None); - // A source that declares no key, or a request that lacks it, cannot shrink. - assert_eq!(shrink_page_size(&mut arguments, None), None); - assert_eq!( - shrink_page_size(&mut json!({ "other": 10 }), Some("max_results")), - None - ); -} - -/// The digits have to stand alone. Every false positive here costs a -/// shrink-and-retry cycle on a failure a smaller page was never going to fix, -/// and delays the real error reaching the caller. -#[test] -fn a_number_that_merely_contains_413_is_not_a_status_code() { - assert!(is_payload_too_large(Some("HTTP 413 Payload Too Large"))); - assert!(is_payload_too_large(Some("upstream returned 413."))); - assert!(is_payload_too_large(Some("(413)"))); - - assert!(!is_payload_too_large(Some("message id 4130 not found"))); - assert!(!is_payload_too_large(Some("amount 1413 exceeds the cap"))); - assert!(!is_payload_too_large(Some("thread 94137 is archived"))); - assert!(!is_payload_too_large(Some( - "at 1782891413 the token expired" - ))); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/clickup.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/clickup.rs deleted file mode 100644 index d16176ec..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/clickup.rs +++ /dev/null @@ -1,196 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_USER: &str = "CLICKUP_GET_AUTHORIZED_USER"; -const ACTION_WORKSPACES: &str = "CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES"; -const ACTION_TASKS: &str = "CLICKUP_GET_FILTERED_TEAM_TASKS"; - -pub struct ClickUpSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl ClickUpSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 20, - page_size: 50, - } - } -} - -#[async_trait] -impl SyncPipeline for ClickUpSyncPipeline { - fn id(&self) -> &str { - "composio:clickup" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for ClickUpSyncPipeline { - fn toolkit(&self) -> &'static str { - "clickup" - } - fn action(&self) -> &'static str { - ACTION_TASKS - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn depth_floor(&self, config: &PipelineConfig, state: &SyncState) -> Option { - if state.cursor.is_some() { - return None; - } - config.sync_depth_days.map(|days| { - (chrono::Utc::now() - chrono::Duration::days(days as i64)) - .timestamp_millis() - .to_string() - }) - } - async fn scopes( - &self, - executor: &dyn ActionExecutor, - connection_id: &str, - state: &mut SyncState, - ) -> anyhow::Result> { - let user_response = checked_execute( - executor, - ACTION_USER, - serde_json::json!({}), - connection_id, - state, - ) - .await?; - let user_id = ["/user/id", "/data/user/id", "/id", "/data/id"] - .iter() - .find_map(|path| user_response.data.pointer(path)) - .and_then(value_string) - .ok_or_else(|| anyhow::anyhow!("{ACTION_USER} returned no user id"))?; - if state.budget_exhausted() { - return Ok(Vec::new()); - } - let workspace_response = checked_execute( - executor, - ACTION_WORKSPACES, - serde_json::json!({}), - connection_id, - state, - ) - .await?; - let workspaces = first_array( - &workspace_response.data, - &["/teams", "/data/teams", "/workspaces", "/data/workspaces"], - ); - Ok(workspaces - .into_iter() - .filter_map(|workspace| pick_str(&workspace, &["id", "team_id", "workspace_id"])) - .map(|id| { - SyncScope::named(id.clone(), format!("workspace:{id}")) - .with_metadata(serde_json::json!({"user_id": user_id})) - }) - .collect()) - } - fn arguments( - &self, - scope: &SyncScope, - _: &PipelineConfig, - _: &SyncState, - page: Option<&str>, - ) -> Value { - serde_json::json!({"team_id": scope.id, "assignees": [scope.metadata.get("user_id").and_then(Value::as_str).unwrap_or_default()], "order_by": "updated", "reverse": true, "page": page.and_then(|value| value.parse::().ok()).unwrap_or(0), "page_size": self.page_size, "subtasks": true}) - } - fn extract_page(&self, data: &Value, page: Option<&str>) -> PageFetch { - let items = first_array( - data, - &[ - "/data/tasks", - "/tasks", - "/data/data/tasks", - "/data/results", - "/results", - "/data/items", - "/items", - ], - ); - let page_number = page - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); - let next = (items.len() == self.page_size).then(|| (page_number + 1).to_string()); - PageFetch { items, next } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "task_id", "data.task_id"])?; - Some(match self.sort_cursor(item) { - Some(updated) => format!("{id}@{updated}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &[ - "date_updated", - "data.date_updated", - "updated_at", - "data.updated_at", - "dateUpdated", - "data.dateUpdated", - ], - ) - } - async fn document( - &self, - scope: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "task_id", "data.task_id"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str(&item.raw, &["name", "data.name", "title", "data.title"]) - .unwrap_or_else(|| format!("ClickUp task {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - let mut result = document("clickup", connection_id, &id, title, content, item.raw); - result.metadata["workspace_id"] = Value::String(scope.id.clone()); - Ok(result) - } -} - -fn value_string(value: &Value) -> Option { - match value { - Value::String(value) => Some(value.clone()), - Value::Number(value) => Some(value.to_string()), - _ => None, - } - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/common.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/common.rs deleted file mode 100644 index a76fa519..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/common.rs +++ /dev/null @@ -1,110 +0,0 @@ -use serde_json::Value; - -use crate::sync::pipelines::traits::SkillDocument; - -/// Walk a JSON document by dotted path and return the first non-empty scalar. -/// -/// # Not interchangeable with [`normalize::helpers::pick_str`] -/// -/// A second `pick_str` lives in [`normalize::helpers`], and the two differ. -/// This one resolves paths with [`Value::pointer`] (so a numeric segment -/// indexes into an array) and **coerces `Number` to its string form**; that -/// one walks with [`Value::get`] (objects only) and returns `None` for any -/// non-string leaf. Swapping one for the other changes what normalisers emit -/// for numeric fields. Keep them separate. -/// -/// [`normalize::helpers`]: tinymemory_sync::helpers -/// [`normalize::helpers::pick_str`]: tinymemory_sync::helpers::pick_str -pub fn pick_str(value: &Value, paths: &[&str]) -> Option { - paths.iter().find_map(|path| { - let pointer = format!("/{}", path.replace('.', "/")); - value - .pointer(&pointer) - .and_then(|value| match value { - Value::String(value) => Some(value.clone()), - Value::Number(value) => Some(value.to_string()), - _ => None, - }) - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) - }) -} - -pub fn first_array(data: &Value, pointers: &[&str]) -> Vec { - pointers - .iter() - .find_map(|pointer| data.pointer(pointer).and_then(Value::as_array)) - .cloned() - .unwrap_or_default() -} - -/// Reads a Google-style `nextPageToken` from the common Composio response -/// envelopes (single- and double-`data`-wrapped), trimming and dropping empty -/// tokens. Shared by the Google provider pipelines to avoid drift. -pub fn next_page_token(data: &Value) -> Option { - [ - "/data/nextPageToken", - "/nextPageToken", - "/data/data/nextPageToken", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::trim) - .filter(|token| !token.is_empty()) - .map(str::to_owned) -} - -pub fn document( - toolkit: &str, - connection_id: &str, - id: &str, - title: String, - content: String, - raw: Value, -) -> SkillDocument { - SkillDocument { - namespace_skill_id: toolkit.into(), - connection_id: connection_id.into(), - document_id: format!("{toolkit}:{id}"), - title, - content, - toolkit: toolkit.into(), - metadata: serde_json::json!({ - "source": "composio-provider-incremental", - "taint": "external_sync", - "provider_id": id, - "raw": raw, - }), - } -} - -pub async fn checked_execute( - executor: &dyn super::super::client::ActionExecutor, - action: &str, - arguments: Value, - connection_id: &str, - state: &mut crate::sync::composio::providers::sync_state::SyncState, -) -> anyhow::Result { - let response = match executor - .execute(action, arguments, Some(connection_id)) - .await - { - Ok(response) => response, - Err(error) => { - if let Some(error) = error.downcast_ref::() { - state.record_requests(error.attempts); - } - return Err(error); - } - }; - state.record_action(response.attempts, response.cost_usd); - anyhow::ensure!( - response.successful, - "{action} provider failure: {}", - response - .error - .as_deref() - .unwrap_or("unknown provider error") - ); - Ok(response) -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/github.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/github.rs deleted file mode 100644 index 032935ae..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/github.rs +++ /dev/null @@ -1,178 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_USER: &str = "GITHUB_GET_THE_AUTHENTICATED_USER"; -const ACTION_SEARCH: &str = "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS"; - -pub struct GitHubSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl GitHubSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 20, - page_size: 50, - } - } -} - -#[async_trait] -impl SyncPipeline for GitHubSyncPipeline { - fn id(&self) -> &str { - "composio:github" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for GitHubSyncPipeline { - fn toolkit(&self) -> &'static str { - "github" - } - fn action(&self) -> &'static str { - ACTION_SEARCH - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn server_side_depth(&self) -> bool { - true - } - async fn scopes( - &self, - executor: &dyn ActionExecutor, - connection_id: &str, - state: &mut SyncState, - ) -> anyhow::Result> { - let response = checked_execute( - executor, - ACTION_USER, - serde_json::json!({}), - connection_id, - state, - ) - .await?; - let login = pick_str(&response.data, &["login", "data.login"]) - .ok_or_else(|| anyhow::anyhow!("{ACTION_USER} returned no login"))?; - Ok(vec![SyncScope::named( - login.clone(), - format!("involves:{login}"), - )]) - } - fn arguments( - &self, - scope: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - let mut query = format!("involves:{}", scope.id); - if let Some(cursor) = state.cursor.as_deref() { - query.push_str(&format!(" updated:>{cursor}")); - } else if let Some(days) = config.sync_depth_days { - let floor = chrono::Utc::now() - chrono::Duration::days(days as i64); - query.push_str(&format!(" updated:>{}", floor.format("%Y-%m-%dT%H:%M:%SZ"))); - } - serde_json::json!({ "q": query, "sort": "updated", "order": "desc", "per_page": self.page_size, "page": page.and_then(|value| value.parse::().ok()).unwrap_or(1) }) - } - fn extract_page(&self, data: &Value, page: Option<&str>) -> PageFetch { - let items = first_array( - data, - &[ - "/data/items", - "/items", - "/data/data/items", - "/data/results", - "/results", - ], - ); - let page_number = page - .and_then(|value| value.parse::().ok()) - .unwrap_or(1); - let next = (items.len() == self.page_size).then(|| (page_number + 1).to_string()); - PageFetch { items, next } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = issue_id(item)?; - Some(match self.sort_cursor(item) { - Some(updated) => format!("{id}@{updated}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &[ - "updated_at", - "data.updated_at", - "updatedAt", - "data.updatedAt", - ], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = issue_id(&item.raw).unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str(&item.raw, &["title", "data.title"]) - .unwrap_or_else(|| format!("GitHub issue {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - Ok(document( - "github", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} - -fn issue_id(item: &Value) -> Option { - pick_str(item, &["id", "data.id"]).or_else(|| { - let url = pick_str(item, &["html_url", "data.html_url", "url", "data.url"])?; - let parts: Vec<_> = url.trim_end_matches('/').split('/').collect(); - (parts.len() >= 7).then(|| { - format!( - "{}/{}#{}", - parts[parts.len() - 4], - parts[parts.len() - 3], - parts[parts.len() - 1] - ) - }) - }) -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_calendar.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_calendar.rs deleted file mode 100644 index 6e0a1f3e..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_calendar.rs +++ /dev/null @@ -1,173 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{document, first_array, next_page_token, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_EVENTS_LIST: &str = "GOOGLECALENDAR_EVENTS_LIST"; - -/// Incremental Google Calendar synchronization through Composio. -/// -/// Events are self-contained records (stable id + `updated` timestamp), so this -/// follows the document-shaped pattern (`LinearSyncPipeline`) rather than the -/// message-shaped one: a single list action, client-visible `updated` cursor, -/// content taken directly from the event payload with no secondary fetch. -pub struct GoogleCalendarSyncPipeline { - client: ComposioClient, - connection_id: String, - calendar_id: String, - max_pages: usize, - page_size: usize, -} - -impl GoogleCalendarSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - calendar_id: "primary".into(), - max_pages: 10, - page_size: 50, - } - } - - pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { - self.max_pages = max_pages.max(1); - // Google Calendar caps `maxResults` at 2500; stay well under it. - self.page_size = page_size.clamp(1, 2500); - self - } -} - -#[async_trait] -impl SyncPipeline for GoogleCalendarSyncPipeline { - fn id(&self) -> &str { - "composio:googlecalendar" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for GoogleCalendarSyncPipeline { - fn toolkit(&self) -> &'static str { - "googlecalendar" - } - fn action(&self) -> &'static str { - ACTION_EVENTS_LIST - } - fn max_pages(&self) -> usize { - self.max_pages - } - // NB: `stop_on_empty_pending` is left at its default (false). The cursor only - // advances on a *complete* sync, so a run capped by `max_pages`/budget leaves - // it unadvanced; stopping early on an all-deduplicated first page would then - // permanently skip the still-unsynced tail. The persisted-cursor boundary - // already halts incremental runs at the right point. - fn server_side_depth(&self) -> bool { - true - } - fn arguments( - &self, - _: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - // `single_events` expands recurring series into concrete instances so - // each carries a stable id; `order_by: "updated"` sorts ascending by - // modification time (oldest change first), matching the `updated` cursor. - let mut args = serde_json::json!({ - "calendar_id": self.calendar_id, - "max_results": self.page_size, - "single_events": true, - "order_by": "updated", - }); - if let Some(page) = page { - args["page_token"] = serde_json::json!(page); - } - // The cursor is a modification time (the item `updated` field), so it - // belongs on `updated_min` (last-modified lower bound) — NOT `time_min`, - // which filters by event *start* time and would drop recently-edited - // past events. `time_min` is only the start-time horizon for the first, - // cursorless backfill; once a cursor exists, `updated_min` fully bounds - // the incremental window. - if let Some(cursor) = state.cursor.as_deref() { - args["updated_min"] = serde_json::json!(cursor); - } else if let Some(days) = config.sync_depth_days { - args["time_min"] = serde_json::json!((chrono::Utc::now() - - chrono::Duration::days(days as i64)) - .to_rfc3339()); - } - args - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/items", - "/items", - "/data/data/items", - "/data/events", - "/events", - ], - ), - next: next_page_token(data), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "iCalUID", "data.iCalUID"])?; - Some(match self.sort_cursor(item) { - Some(updated) => format!("{id}@{updated}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str(item, &["updated", "data.updated"]) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "iCalUID", "data.iCalUID"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str( - &item.raw, - &["summary", "data.summary", "title", "data.title"], - ) - .unwrap_or_else(|| format!("Calendar event {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - Ok(document( - "googlecalendar", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_docs.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_docs.rs deleted file mode 100644 index e1809744..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_docs.rs +++ /dev/null @@ -1,216 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_SEARCH: &str = "GOOGLEDOCS_SEARCH_DOCUMENTS"; -const ACTION_PLAINTEXT: &str = "GOOGLEDOCS_GET_DOCUMENT_PLAINTEXT"; - -/// Incremental Google Docs synchronization through Composio. -/// -/// Two-step, document-shaped (like `NotionSyncPipeline`): `GOOGLEDOCS_SEARCH_DOCUMENTS` -/// enumerates accessible documents, then `GOOGLEDOCS_GET_DOCUMENT_PLAINTEXT` fetches the -/// body for each item inside [`IncrementalSource::document`]. -pub struct GoogleDocsSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl GoogleDocsSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - // NOTE: SEARCH_DOCUMENTS' page-token arg name is not pinned by the - // curated catalog, so we do a single-page-per-tick fetch (no page - // token emitted) rather than guessing a pagination scheme. Capped at - // 1 page: since `arguments()` never advances the token, a >1 cap - // would re-fire the identical page-1 request and burn budget slots - // for silently-deduplicated items. - max_pages: 1, - page_size: 25, - } - } -} - -#[async_trait] -impl SyncPipeline for GoogleDocsSyncPipeline { - fn id(&self) -> &str { - "composio:googledocs" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for GoogleDocsSyncPipeline { - fn toolkit(&self) -> &'static str { - "googledocs" - } - fn action(&self) -> &'static str { - ACTION_SEARCH - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn arguments( - &self, - _: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - _page: Option<&str>, - ) -> Value { - // `GOOGLEDOCS_SEARCH_DOCUMENTS` fronts Drive's `files.list`, so it takes - // the same server-side controls Drive does. Order deterministically by - // modification time and bound the window with a `q` clause, so each - // tick fetches what changed since the cursor rather than the same - // first batch forever. Without this the action returned the identical - // page every tick and documents past `max_results` were unreachable. - let mut args = serde_json::json!({ - "query": "", - "max_results": self.page_size, - "order_by": "modifiedTime desc", - }); - // Prefer the last-synced cursor, else the configured horizon. The - // cursor is validated as RFC 3339 before it is interpolated into `q`, - // so a malformed persisted value can never inject into the query — on - // a bad value the depth filter is simply omitted (full scan). - let floor = state - .cursor - .as_deref() - .filter(|cursor| chrono::DateTime::parse_from_rfc3339(cursor).is_ok()) - .map(str::to_owned) - .or_else(|| { - config.sync_depth_days.map(|days| { - (chrono::Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339() - }) - }); - if let Some(floor) = floor { - args["q"] = serde_json::json!(format!("modifiedTime > '{floor}'")); - } - args - } - fn server_side_depth(&self) -> bool { - // The `q` floor above bounds depth on the server, so the orchestrator - // must not additionally treat the cursor as a client-side stop. - true - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/documents", - "/documents", - "/data/files", - "/files", - "/data/results", - "/results", - "/data/items", - "/items", - ], - ), - // Bounded fetch: no page token consumed (see `max_pages`). The - // pointers are read defensively should Composio surface one. - next: [ - "/data/nextPageToken", - "/nextPageToken", - "/data/next_page_token", - "/next_page_token", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::trim) - .filter(|token| !token.is_empty()) - .map(str::to_owned), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "documentId", "data.documentId"])?; - Some(match self.sort_cursor(item) { - Some(modified) => format!("{id}@{modified}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &[ - "modifiedTime", - "data.modifiedTime", - "modified_time", - "updatedTime", - ], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - executor: &dyn ActionExecutor, - state: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str( - &item.raw, - &["id", "data.id", "documentId", "data.documentId"], - ) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str(&item.raw, &["title", "data.title", "name", "data.name"]) - .unwrap_or_else(|| format!("Google Doc {id}")); - // NOTE: GET_DOCUMENT_PLAINTEXT identifies the doc by an id argument; - // Composio commonly keys this as "id" (or "document_id"). We send "id". - let response = checked_execute( - executor, - ACTION_PLAINTEXT, - serde_json::json!({"id": id}), - connection_id, - state, - ) - .await?; - let content = [ - "/data/text", - "/text", - "/data/plaintext", - "/plaintext", - "/data/content", - "/content", - "/data/response_data/text", - ] - .iter() - .find_map(|path| response.data.pointer(path).and_then(Value::as_str)) - .filter(|value| !value.trim().is_empty()) - .map(str::to_owned) - .unwrap_or(serde_json::to_string_pretty(&item.raw)?); - Ok(document( - "googledocs", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_drive.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_drive.rs deleted file mode 100644 index d619aa2b..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_drive.rs +++ /dev/null @@ -1,179 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{document, first_array, next_page_token, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -// Composio deprecated `GOOGLEDRIVE_LIST_FILES` (2026-03-28) in favour of -// `GOOGLEDRIVE_FIND_FILE`, which is the current `files.list`-backed listing -// action (same paging/ordering/`q` filter surface). -const ACTION_FIND_FILE: &str = "GOOGLEDRIVE_FIND_FILE"; - -/// Incremental Google Drive synchronization through Composio. -/// -/// File-shaped: each Drive file is a record with a stable id and a -/// `modifiedTime`. This indexes file *metadata* only — it never downloads -/// binary bodies (which may be arbitrarily large and are not memory-shaped); -/// the document content is the file's structured metadata. -pub struct GoogleDriveSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl GoogleDriveSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 10, - page_size: 50, - } - } - - pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { - self.max_pages = max_pages.max(1); - // Google Drive caps `pageSize` at 1000. - self.page_size = page_size.clamp(1, 1000); - self - } -} - -#[async_trait] -impl SyncPipeline for GoogleDriveSyncPipeline { - fn id(&self) -> &str { - "composio:googledrive" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for GoogleDriveSyncPipeline { - fn toolkit(&self) -> &'static str { - "googledrive" - } - fn action(&self) -> &'static str { - ACTION_FIND_FILE - } - fn max_pages(&self) -> usize { - self.max_pages - } - // NB: `stop_on_empty_pending` stays at its default (false) — see the note on - // the Calendar pipeline. The cursor only advances on a complete sync, so a - // capped run must not stop early on an all-deduplicated first page or it - // would permanently skip the unsynced tail. - fn server_side_depth(&self) -> bool { - true - } - fn arguments( - &self, - _: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - let mut args = serde_json::json!({ - "page_size": self.page_size, - "order_by": "modifiedTime desc", - // Guarantee the fields the cursor/title/dedup depend on come back, - // regardless of the action's default projection. - "fields": "files(id,name,mimeType,modifiedTime),nextPageToken", - }); - if let Some(page) = page { - args["page_token"] = serde_json::json!(page); - } - // Depth window via a Drive `q` clause on modification time. Prefer the - // last-synced cursor, else the configured horizon. The cursor is - // validated as an RFC3339 timestamp before being interpolated into the - // query so a malformed persisted value can never inject into the `q` - // clause — on a bad value we simply omit the depth filter (full scan). - let floor = state - .cursor - .as_deref() - .filter(|cursor| chrono::DateTime::parse_from_rfc3339(cursor).is_ok()) - .map(str::to_owned) - .or_else(|| { - config.sync_depth_days.map(|days| { - (chrono::Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339() - }) - }); - if let Some(floor) = floor { - // `GOOGLEDRIVE_FIND_FILE` names the Drive query parameter `q` (the - // native `files.list` name), not `query` — an unrecognised key would - // be ignored and defeat server-side depth bounding. - args["q"] = serde_json::json!(format!("modifiedTime > '{floor}'")); - } - args - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/files", - "/files", - "/data/data/files", - "/data/items", - "/items", - ], - ), - next: next_page_token(data), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "fileId", "data.fileId"])?; - Some(match self.sort_cursor(item) { - Some(modified) => format!("{id}@{modified}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &["modifiedTime", "data.modifiedTime", "modified_time"], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "fileId", "data.fileId"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str(&item.raw, &["name", "data.name", "title", "data.title"]) - .unwrap_or_else(|| format!("Drive file {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - Ok(document( - "googledrive", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_sheets.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_sheets.rs deleted file mode 100644 index fbc26657..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_sheets.rs +++ /dev/null @@ -1,186 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_SEARCH: &str = "GOOGLESHEETS_SEARCH_SPREADSHEETS"; -const ACTION_INFO: &str = "GOOGLESHEETS_GET_SPREADSHEET_INFO"; - -/// Incremental Google Sheets synchronization through Composio. -/// -/// Two-step, document-shaped (like `NotionSyncPipeline`): `GOOGLESHEETS_SEARCH_SPREADSHEETS` -/// enumerates accessible spreadsheets, then `GOOGLESHEETS_GET_SPREADSHEET_INFO` fetches the -/// spreadsheet metadata for each item inside [`IncrementalSource::document`]. -pub struct GoogleSheetsSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl GoogleSheetsSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - // NOTE: SEARCH_SPREADSHEETS' page-token arg name is not pinned by the - // curated catalog, so we do a single-page-per-tick fetch (no page - // token emitted) rather than guessing a pagination scheme. Capped at - // 1 page: since `arguments()` never advances the token, a >1 cap - // would re-fire the identical page-1 request and burn budget slots - // for silently-deduplicated items. - max_pages: 1, - page_size: 25, - } - } -} - -#[async_trait] -impl SyncPipeline for GoogleSheetsSyncPipeline { - fn id(&self) -> &str { - "composio:googlesheets" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for GoogleSheetsSyncPipeline { - fn toolkit(&self) -> &'static str { - "googlesheets" - } - fn action(&self) -> &'static str { - ACTION_SEARCH - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn arguments( - &self, - _: &SyncScope, - _: &PipelineConfig, - _: &SyncState, - _page: Option<&str>, - ) -> Value { - // NOTE: an empty/broad `query` enumerates every accessible spreadsheet; - // `max_results` bounds the batch. Both mirror the underlying Drive - // search parameters. No page token is emitted (see `max_pages`). - serde_json::json!({"query": "", "max_results": self.page_size}) - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/spreadsheets", - "/spreadsheets", - "/data/files", - "/files", - "/data/results", - "/results", - "/data/items", - "/items", - ], - ), - // Bounded fetch: no page token consumed (see `max_pages`). The - // pointers are read defensively should Composio surface one. - next: [ - "/data/nextPageToken", - "/nextPageToken", - "/data/next_page_token", - "/next_page_token", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::trim) - .filter(|token| !token.is_empty()) - .map(str::to_owned), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str( - item, - &["id", "data.id", "spreadsheetId", "data.spreadsheetId"], - )?; - Some(match self.sort_cursor(item) { - Some(modified) => format!("{id}@{modified}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &["modifiedTime", "data.modifiedTime", "modified_time"], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - executor: &dyn ActionExecutor, - state: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str( - &item.raw, - &["id", "data.id", "spreadsheetId", "data.spreadsheetId"], - ) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str( - &item.raw, - &[ - "title", - "data.title", - "properties.title", - "data.properties.title", - "name", - ], - ) - .unwrap_or_else(|| format!("Google Sheet {id}")); - // NOTE: GET_SPREADSHEET_INFO identifies the spreadsheet by a - // "spreadsheet_id" argument (Google's canonical parameter name). - let response = checked_execute( - executor, - ACTION_INFO, - serde_json::json!({"spreadsheet_id": id}), - connection_id, - state, - ) - .await?; - // `response.data` is the already-unwrapped payload; the pointers catch - // any additional Composio wrapping, else we serialize the payload root. - let info = ["/data", "/data/data"] - .iter() - .find_map(|path| response.data.pointer(path)) - .unwrap_or(&response.data); - let content = serde_json::to_string_pretty(info)?; - Ok(document( - "googlesheets", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/linear.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/linear.rs deleted file mode 100644 index bfa08f7c..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/linear.rs +++ /dev/null @@ -1,193 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_USERS: &str = "LINEAR_LIST_LINEAR_USERS"; -const ACTION_ISSUES: &str = "LINEAR_LIST_LINEAR_ISSUES"; - -pub struct LinearSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl LinearSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 20, - page_size: 50, - } - } -} - -#[async_trait] -impl SyncPipeline for LinearSyncPipeline { - fn id(&self) -> &str { - "composio:linear" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for LinearSyncPipeline { - fn toolkit(&self) -> &'static str { - "linear" - } - fn action(&self) -> &'static str { - ACTION_ISSUES - } - fn max_pages(&self) -> usize { - self.max_pages - } - async fn scopes( - &self, - executor: &dyn ActionExecutor, - connection_id: &str, - state: &mut SyncState, - ) -> anyhow::Result> { - let response = checked_execute( - executor, - ACTION_USERS, - serde_json::json!({"isMe": true}), - connection_id, - state, - ) - .await?; - let users = first_array( - &response.data, - &[ - "/data/nodes", - "/nodes", - "/data/data/nodes", - "/data/users/nodes", - ], - ); - let viewer = users.first().unwrap_or(&response.data); - let id = pick_str(viewer, &["id", "data.id"]) - .ok_or_else(|| anyhow::anyhow!("{ACTION_USERS} returned no viewer id"))?; - Ok(vec![SyncScope::named(id, "assignee:me")]) - } - fn arguments( - &self, - scope: &SyncScope, - _: &PipelineConfig, - _: &SyncState, - page: Option<&str>, - ) -> Value { - let mut args = serde_json::json!({"assigneeId": scope.id, "first": self.page_size, "orderBy": "updatedAt"}); - if let Some(page) = page { - args["after"] = serde_json::json!(page); - } - args - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - let items = first_array( - data, - &[ - "/data/nodes", - "/nodes", - "/data/data/nodes", - "/data/issues/nodes", - "/data/results", - "/results", - "/data/items", - "/items", - ], - ); - let page_info = [ - "/data/pageInfo", - "/pageInfo", - "/data/data/pageInfo", - "/data/issues/pageInfo", - ] - .iter() - .find_map(|path| data.pointer(path)); - let next = page_info - .filter(|info| { - info.get("hasNextPage") - .and_then(Value::as_bool) - .unwrap_or(false) - }) - .and_then(|info| info.get("endCursor").and_then(Value::as_str)) - .map(str::to_owned); - PageFetch { items, next } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "identifier", "data.identifier"])?; - Some(match self.sort_cursor(item) { - Some(updated) => format!("{id}@{updated}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &[ - "updatedAt", - "data.updatedAt", - "updated_at", - "data.updated_at", - ], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str( - &item.raw, - &["id", "data.id", "identifier", "data.identifier"], - ) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str( - &item.raw, - &[ - "title", - "data.title", - "name", - "data.name", - "identifier", - "data.identifier", - ], - ) - .unwrap_or_else(|| format!("Linear issue {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - Ok(document( - "linear", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/mod.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/mod.rs deleted file mode 100644 index 120cf15c..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/mod.rs +++ /dev/null @@ -1,31 +0,0 @@ -//! One pipeline per Composio toolkit. Normalisation lives in -//! `tinymemory-sync`; these drive fetch, budget, and the write path. - -mod clickup; -mod common; -mod github; -mod google_calendar; -mod google_docs; -mod google_drive; -mod google_sheets; -mod linear; -mod notion; -mod outlook; -mod slack; -mod slack_parse; -mod todoist; - -#[cfg(test)] -mod provider_tests; - -pub use clickup::ClickUpSyncPipeline; -pub use github::GitHubSyncPipeline; -pub use google_calendar::GoogleCalendarSyncPipeline; -pub use google_docs::GoogleDocsSyncPipeline; -pub use google_drive::GoogleDriveSyncPipeline; -pub use google_sheets::GoogleSheetsSyncPipeline; -pub use linear::LinearSyncPipeline; -pub use notion::NotionSyncPipeline; -pub use outlook::OutlookSyncPipeline; -pub use slack::{SlackSearchBackfillPipeline, SlackSyncPipeline}; -pub use todoist::TodoistSyncPipeline; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/notion.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/notion.rs deleted file mode 100644 index e693c130..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/notion.rs +++ /dev/null @@ -1,203 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_FETCH: &str = "NOTION_FETCH_DATA"; -const ACTION_MARKDOWN: &str = "NOTION_GET_PAGE_MARKDOWN"; - -pub struct NotionSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl NotionSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 20, - page_size: 25, - } - } -} - -#[async_trait] -impl SyncPipeline for NotionSyncPipeline { - fn id(&self) -> &str { - "composio:notion" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for NotionSyncPipeline { - fn toolkit(&self) -> &'static str { - "notion" - } - fn action(&self) -> &'static str { - ACTION_FETCH - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn arguments( - &self, - _: &SyncScope, - _: &PipelineConfig, - _: &SyncState, - page: Option<&str>, - ) -> Value { - // `fetch_type` is required by Composio's `NOTION_FETCH_DATA`; omitting - // it fails the action's input-schema validation with `Invalid request - // data provided - Following fields are missing: {'fetch_type'}`, so - // every periodic Notion sync tick errors out. - // - // Unconditionally `"pages"`: this path hardcodes `filter: {value: - // "page"}`, so no other value is valid here. The agent-tool path infers - // the value instead (OpenHuman's `ensure_notion_fetch_type`) because - // there the filter is caller-supplied; that inference is deliberately - // not duplicated into this pipeline. - let mut args = serde_json::json!({"fetch_type": "pages", "page_size": self.page_size, "filter": {"value": "page", "property": "object"}, "sort": {"direction": "descending", "timestamp": "last_edited_time"}}); - if let Some(page) = page { - args["start_cursor"] = serde_json::json!(page); - } - args - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/results", - "/results", - "/data/data/results", - "/data/items", - "/items", - ], - ), - next: [ - "/data/next_cursor", - "/next_cursor", - "/data/data/next_cursor", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::to_owned), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "pageId", "data.pageId"])?; - Some(match self.sort_cursor(item) { - Some(edited) => format!("{id}@{edited}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &[ - "last_edited_time", - "data.last_edited_time", - "lastEditedTime", - "data.lastEditedTime", - ], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - executor: &dyn ActionExecutor, - state: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "pageId", "data.pageId"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = notion_title(&item.raw).unwrap_or_else(|| format!("Notion page {id}")); - let response = checked_execute( - executor, - ACTION_MARKDOWN, - serde_json::json!({"page_id": id}), - connection_id, - state, - ) - .await?; - let content = [ - "/markdown", - "/data/markdown", - "/data/response_data/markdown", - "/response_data/markdown", - "/data/content", - "/content", - "/text", - "/data/text", - ] - .iter() - .find_map(|path| response.data.pointer(path).and_then(Value::as_str)) - .filter(|value| !value.trim().is_empty()) - .map(str::to_owned) - .unwrap_or(serde_json::to_string_pretty(&item.raw)?); - Ok(document( - "notion", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} - -fn notion_title(page: &Value) -> Option { - let properties = page - .get("properties") - .or_else(|| page.pointer("/data/properties")); - properties - .and_then(Value::as_object) - .and_then(|props| { - props.values().find_map(|property| { - (property.get("type").and_then(Value::as_str) == Some("title")) - .then(|| { - property - .get("title") - .and_then(Value::as_array) - .map(|parts| { - parts - .iter() - .filter_map(|part| { - part.get("plain_text").and_then(Value::as_str) - }) - .collect::>() - .join("") - }) - }) - .flatten() - .filter(|title| !title.is_empty()) - }) - }) - .or_else(|| pick_str(page, &["title", "data.title", "name", "data.name"])) -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/outlook.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/outlook.rs deleted file mode 100644 index bfe0780b..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/outlook.rs +++ /dev/null @@ -1,205 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_LIST_MESSAGES: &str = "OUTLOOK_LIST_MESSAGES"; - -/// Incremental Microsoft Outlook mail synchronization through Composio. -/// -/// Outlook messages carry a stable `id` and a `receivedDateTime` timestamp, so -/// this follows the message-shaped pattern (`GmailSyncPipeline`): a single list -/// action ordered newest-first, a client-visible `receivedDateTime` cursor, and -/// content taken directly from the message payload with no secondary fetch. -pub struct OutlookSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl OutlookSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 10, - page_size: 25, - } - } - - pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { - self.max_pages = max_pages.max(1); - self.page_size = page_size.max(1); - self - } -} - -#[async_trait] -impl SyncPipeline for OutlookSyncPipeline { - fn id(&self) -> &str { - "composio:outlook" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for OutlookSyncPipeline { - fn toolkit(&self) -> &'static str { - "outlook" - } - fn action(&self) -> &'static str { - ACTION_LIST_MESSAGES - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn stop_on_empty_pending(&self) -> bool { - true - } - fn server_side_depth(&self) -> bool { - true - } - fn arguments( - &self, - _: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - // Microsoft Graph list-messages params passed through Composio: `top` - // bounds the page size, `orderby` sorts newest-first by receive time. - let mut args = serde_json::json!({ - "top": self.page_size, - "orderby": "receivedDateTime desc", - }); - if let Some(page) = page { - // Graph paginates via a `$skiptoken`; `extract_page` has already - // reduced the `@odata.nextLink` URL to the bare token. The exact - // Composio arg name for feeding it back is not fully certain — we - // send `skip_token` (the Graph-native name), so a mislabel here - // surfaces as a single-page fetch, not silent data loss. - args["skip_token"] = serde_json::json!(page); - } - // Depth window: prefer the last-synced cursor over the configured - // horizon (same precedence as the Gmail/Calendar pipelines). Graph - // filters server-side via `$filter` on `receivedDateTime`. - if let Some(cursor) = state.cursor.as_deref() { - args["filter"] = serde_json::json!(format!("receivedDateTime ge {cursor}")); - } else if let Some(days) = config.sync_depth_days { - let horizon = (chrono::Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339(); - args["filter"] = serde_json::json!(format!("receivedDateTime ge {horizon}")); - } - args - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/value", - "/value", - "/data/messages", - "/messages", - "/data/data/value", - "/data/items", - "/items", - ], - ), - next: [ - "/data/@odata.nextLink", - "/@odata.nextLink", - "/data/nextPageToken", - "/nextPageToken", - "/data/skip_token", - "/skip_token", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::trim) - .filter(|token| !token.is_empty()) - .map(normalize_skip_token), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "messageId", "data.messageId"])?; - Some(match self.sort_cursor(item) { - Some(received) => format!("{id}@{received}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - // Only `receivedDateTime` — the same field the `$filter` depth window - // keys on. A `lastModifiedDateTime` fallback would store a cursor in a - // different field than the filter compares, so on the next sync the - // `receivedDateTime ge ` window could skip valid messages. - pick_str( - item, - &[ - "receivedDateTime", - "data.receivedDateTime", - "received_date_time", - ], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "messageId", "data.messageId"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str(&item.raw, &["subject", "data.subject", "title"]) - .unwrap_or_else(|| format!("Outlook message {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - Ok(document( - "outlook", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} - -/// Reduce a Graph paging token to the bare `$skiptoken` value. -/// -/// Graph returns `@odata.nextLink` as a full URL -/// (`https://graph.microsoft.com/v1.0/me/messages?$skiptoken=ABC...`). Feeding -/// that whole URL back as the paging arg would not resume pagination, so when -/// the token looks like a URL we extract just the `skiptoken` query value; -/// otherwise (Composio may already surface the bare token) we pass it through. -fn normalize_skip_token(token: &str) -> String { - let lower = token.to_ascii_lowercase(); - if let Some(pos) = lower.find("skiptoken=") { - let value = &token[pos + "skiptoken=".len()..]; - let end = value.find('&').unwrap_or(value.len()); - return value[..end].to_string(); - } - token.to_string() -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/provider_tests.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/provider_tests.rs deleted file mode 100644 index 073817bf..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/provider_tests.rs +++ /dev/null @@ -1,1028 +0,0 @@ -//! Deterministic contract tests for the document-oriented Composio providers. - -use std::collections::{HashMap, VecDeque}; -use std::sync::{Arc, Mutex}; - -use async_trait::async_trait; -use serde_json::{json, Value}; - -use super::{ - ClickUpSyncPipeline, GitHubSyncPipeline, GoogleCalendarSyncPipeline, GoogleDocsSyncPipeline, - GoogleDriveSyncPipeline, GoogleSheetsSyncPipeline, LinearSyncPipeline, NotionSyncPipeline, - OutlookSyncPipeline, SlackSearchBackfillPipeline, SlackSyncPipeline, TodoistSyncPipeline, -}; -use crate::sync::composio::providers::sync_state::{PersistedSyncState, SyncState, SyncStateStore}; -use crate::sync::pipelines::composio::{ - ActionExecutor, ComposioClient, ExecuteResponse, IncrementalSource, SyncItem, SyncScope, -}; -use crate::sync::pipelines::traits::{ - ComposioSyncConfig, PipelineConfig, SkillDocSink, SkillDocument, SyncContext, SyncEvent, - SyncEventSink, SyncPipeline, SyncPipelineKind, -}; - -#[derive(Debug)] -struct StubExecutor { - response: anyhow::Result, - calls: Mutex)>>, -} - -impl StubExecutor { - fn succeeds(data: Value) -> Self { - Self { - response: Ok(ExecuteResponse { - data, - successful: true, - error: None, - cost_usd: 0.25, - markdown_formatted: None, - attempts: 2, - }), - calls: Mutex::new(Vec::new()), - } - } - - fn provider_failure(message: &'static str) -> Self { - Self { - response: Ok(ExecuteResponse { - data: Value::Null, - successful: false, - error: Some(message.into()), - cost_usd: 0.0, - markdown_formatted: None, - attempts: 1, - }), - calls: Mutex::new(Vec::new()), - } - } -} - -#[async_trait] -impl ActionExecutor for StubExecutor { - async fn execute( - &self, - action: &str, - arguments: Value, - connection_id: Option<&str>, - ) -> anyhow::Result { - self.calls.lock().expect("calls lock").push(( - action.into(), - arguments, - connection_id.map(str::to_owned), - )); - match &self.response { - Ok(response) => Ok(response.clone()), - Err(message) => anyhow::bail!(*message), - } - } -} - -#[derive(Debug)] -struct QueueExecutor { - responses: Mutex>, - calls: Mutex)>>, -} - -impl QueueExecutor { - fn new(data: impl IntoIterator) -> Self { - Self { - responses: Mutex::new( - data.into_iter() - .map(|data| ExecuteResponse { - data, - successful: true, - error: None, - cost_usd: 0.0, - markdown_formatted: None, - attempts: 1, - }) - .collect(), - ), - calls: Mutex::new(Vec::new()), - } - } - - fn from_responses(responses: impl IntoIterator) -> Self { - Self { - responses: Mutex::new(responses.into_iter().collect()), - calls: Mutex::new(Vec::new()), - } - } -} - -#[async_trait] -impl ActionExecutor for QueueExecutor { - async fn execute( - &self, - action: &str, - arguments: Value, - connection_id: Option<&str>, - ) -> anyhow::Result { - self.calls.lock().expect("calls lock").push(( - action.into(), - arguments, - connection_id.map(str::to_owned), - )); - self.responses - .lock() - .expect("responses lock") - .pop_front() - .ok_or_else(|| anyhow::anyhow!("no queued response for {action}")) - } -} - -#[derive(Default)] -struct NoopSyncHost(Mutex>); - -#[async_trait] -impl SkillDocSink for NoopSyncHost { - async fn store(&self, _: SkillDocument) -> anyhow::Result<()> { - Ok(()) - } - - async fn delete(&self, _: &str, _: &str) -> anyhow::Result<()> { - Ok(()) - } -} - -#[async_trait] -impl SyncEventSink for NoopSyncHost { - async fn emit(&self, _: SyncEvent) -> anyhow::Result<()> { - Ok(()) - } -} - -#[async_trait] -impl SyncStateStore for NoopSyncHost { - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - Ok(self - .0 - .lock() - .expect("state lock") - .get(&format!("{namespace}:{key}")) - .cloned()) - } - - async fn set(&self, namespace: &str, key: &str, value: &Value) -> anyhow::Result<()> { - self.0 - .lock() - .expect("state lock") - .insert(format!("{namespace}:{key}"), value.clone()); - Ok(()) - } -} - -fn sync_context(host: Arc) -> SyncContext { - SyncContext { - events: host.clone(), - documents: host.clone(), - state: host, - } -} - -fn client() -> ComposioClient { - ComposioClient::new(ComposioSyncConfig::default()) -} - -fn item(raw: Value, dedup_key: &str) -> SyncItem { - SyncItem { - dedup_key: dedup_key.into(), - sort_cursor: None, - raw, - } -} - -#[test] -fn providers_publish_stable_action_and_paging_contracts() { - let calendar = GoogleCalendarSyncPipeline::new(client(), "calendar").with_limits(0, 9_999); - let docs = GoogleDocsSyncPipeline::new(client(), "docs"); - let drive = GoogleDriveSyncPipeline::new(client(), "drive").with_limits(0, 9_999); - let sheets = GoogleSheetsSyncPipeline::new(client(), "sheets"); - let outlook = OutlookSyncPipeline::new(client(), "outlook").with_limits(0, 0); - let todoist = TodoistSyncPipeline::new(client(), "todoist").with_limits(0, 500); - - let cases: [(&dyn IncrementalSource, &str, &str, usize, bool, bool); 6] = [ - ( - &calendar, - "googlecalendar", - "GOOGLECALENDAR_EVENTS_LIST", - 1, - false, - true, - ), - ( - &docs, - "googledocs", - "GOOGLEDOCS_SEARCH_DOCUMENTS", - 1, - false, - true, - ), - ( - &drive, - "googledrive", - "GOOGLEDRIVE_FIND_FILE", - 1, - false, - true, - ), - ( - &sheets, - "googlesheets", - "GOOGLESHEETS_SEARCH_SPREADSHEETS", - 1, - false, - false, - ), - (&outlook, "outlook", "OUTLOOK_LIST_MESSAGES", 1, true, true), - (&todoist, "todoist", "TODOIST_GET_ALL_TASKS", 1, true, false), - ]; - for (provider, toolkit, action, pages, stop_on_empty, server_depth) in cases { - assert_eq!(provider.toolkit(), toolkit); - assert_eq!(provider.action(), action); - assert_eq!(provider.max_pages(), pages); - assert_eq!(provider.stop_on_empty_pending(), stop_on_empty); - assert_eq!(provider.server_side_depth(), server_depth); - } - - let pipelines: [(&dyn SyncPipeline, &str); 6] = [ - (&calendar, "composio:googlecalendar"), - (&docs, "composio:googledocs"), - (&drive, "composio:googledrive"), - (&sheets, "composio:googlesheets"), - (&outlook, "composio:outlook"), - (&todoist, "composio:todoist"), - ]; - for (pipeline, id) in pipelines { - assert_eq!(pipeline.id(), id); - assert_eq!(pipeline.kind(), SyncPipelineKind::Composio); - } -} - -#[test] -fn calendar_uses_cursor_page_and_normalizes_event_shapes() { - let provider = GoogleCalendarSyncPipeline::new(client(), "connection").with_limits(3, 0); - let mut state = SyncState::new("googlecalendar", "connection"); - state.cursor = Some("2026-01-02T03:04:05Z".into()); - let args = provider.arguments( - &SyncScope::flat(), - &PipelineConfig::default(), - &state, - Some(" next "), - ); - assert_eq!(args["calendar_id"], "primary"); - assert_eq!(args["max_results"], 1); - assert_eq!(args["page_token"], " next "); - assert_eq!(args["updated_min"], "2026-01-02T03:04:05Z"); - assert!(args.get("time_min").is_none()); - - let page = provider.extract_page( - &json!({"data":{"events":[{"id":"event"}],"nextPageToken":" token "}}), - None, - ); - assert_eq!(page.items, vec![json!({"id":"event"})]); - assert_eq!(page.next.as_deref(), Some("token")); - let event = json!({"iCalUID":"ical", "updated":"2026-01-03T00:00:00Z"}); - assert_eq!( - provider.dedup_key(&event).as_deref(), - Some("ical@2026-01-03T00:00:00Z") - ); - assert_eq!( - provider.sort_cursor(&event).as_deref(), - Some("2026-01-03T00:00:00Z") - ); - assert_eq!(provider.dedup_key(&json!({})), None); -} - -#[tokio::test] -async fn calendar_document_preserves_external_metadata_and_fallbacks() { - let provider = GoogleCalendarSyncPipeline::new(client(), "unused"); - let mut state = SyncState::new("googlecalendar", "connection"); - let executor = StubExecutor::succeeds(Value::Null); - let document = provider - .document( - &SyncScope::flat(), - "connection", - item(json!({"iCalUID": 42, "summary":"Planning"}), "fallback"), - &executor, - &mut state, - ) - .await - .expect("calendar document"); - assert_eq!(document.document_id, "googlecalendar:42"); - assert_eq!(document.title, "Planning"); - assert_eq!(document.metadata["provider_id"], "42"); - assert_eq!(document.metadata["taint"], "external_sync"); - assert!(document.content.contains("Planning")); -} - -#[test] -fn docs_validate_cursor_and_accept_wrapped_search_results() { - let provider = GoogleDocsSyncPipeline::new(client(), "connection"); - let mut state = SyncState::new("googledocs", "connection"); - state.cursor = Some("2026-04-05T06:07:08Z".into()); - let args = provider.arguments( - &SyncScope::flat(), - &PipelineConfig::default(), - &state, - Some("ignored"), - ); - assert_eq!(args["q"], "modifiedTime > '2026-04-05T06:07:08Z'"); - assert_eq!(args["max_results"], 25); - assert!(args.get("page_token").is_none()); - state.cursor = Some("' or trashed = false".into()); - assert!(provider - .arguments(&SyncScope::flat(), &PipelineConfig::default(), &state, None) - .get("q") - .is_none()); - - let page = provider.extract_page( - &json!({"data":{"documents":[{"documentId":"doc"}]},"next_page_token":" p2 "}), - None, - ); - assert_eq!(page.items.len(), 1); - assert_eq!(page.next.as_deref(), Some("p2")); - let doc = json!({"data":{"documentId":"doc","modifiedTime":"2026-05-01T00:00:00Z"}}); - assert_eq!( - provider.dedup_key(&doc).as_deref(), - Some("doc@2026-05-01T00:00:00Z") - ); -} - -#[tokio::test] -async fn docs_fetch_plaintext_and_propagate_provider_failures() { - let provider = GoogleDocsSyncPipeline::new(client(), "unused"); - let executor = StubExecutor::succeeds(json!({"data":{"plaintext":"body text"}})); - let mut state = SyncState::new("googledocs", "connection"); - let document = provider - .document( - &SyncScope::flat(), - "connection", - item(json!({"documentId":"doc-1","name":"Roadmap"}), "key"), - &executor, - &mut state, - ) - .await - .expect("docs document"); - assert_eq!(document.title, "Roadmap"); - assert_eq!(document.content, "body text"); - assert_eq!(state.run_requests, 2); - assert_eq!(state.run_provider_cost_usd, 0.25); - assert_eq!( - executor.calls.lock().expect("calls lock")[0], - ( - "GOOGLEDOCS_GET_DOCUMENT_PLAINTEXT".into(), - json!({"id":"doc-1"}), - Some("connection".into()), - ) - ); - - let failing = StubExecutor::provider_failure("permission denied"); - let error = provider - .document( - &SyncScope::flat(), - "connection", - item(json!({"id":"doc-2"}), "key"), - &failing, - &mut state, - ) - .await - .expect_err("provider failure must propagate"); - assert!(error.to_string().contains("permission denied")); - - let empty = StubExecutor::succeeds(json!({"text":" "})); - let fallback = provider - .document( - &SyncScope::flat(), - "connection", - item(json!({"id":"doc-3","name":"Empty body"}), "key"), - &empty, - &mut state, - ) - .await - .expect("empty plaintext falls back to search metadata"); - assert!(fallback.content.contains("Empty body")); -} - -#[test] -fn drive_clamps_limits_validates_cursor_and_paginates() { - let provider = GoogleDriveSyncPipeline::new(client(), "connection").with_limits(2, 2_000); - let mut state = SyncState::new("googledrive", "connection"); - state.cursor = Some("2026-06-01T12:00:00+00:00".into()); - let args = provider.arguments( - &SyncScope::flat(), - &PipelineConfig::default(), - &state, - Some("page-2"), - ); - assert_eq!(provider.max_pages(), 2); - assert_eq!(args["page_size"], 1_000); - assert_eq!(args["page_token"], "page-2"); - assert_eq!(args["q"], "modifiedTime > '2026-06-01T12:00:00+00:00'"); - assert!(args["fields"] - .as_str() - .expect("fields") - .contains("modifiedTime")); - let page = provider.extract_page( - &json!({"data":{"data":{"files":[{"fileId":7}],"nextPageToken":"p3"}}}), - None, - ); - assert_eq!(page.items, vec![json!({"fileId":7})]); - assert_eq!(page.next.as_deref(), Some("p3")); - let file = json!({"fileId":7,"modified_time":"cursor"}); - assert_eq!(provider.dedup_key(&file).as_deref(), Some("7@cursor")); -} - -#[tokio::test] -async fn drive_document_serializes_metadata_without_fetching_body() { - let provider = GoogleDriveSyncPipeline::new(client(), "unused"); - let executor = StubExecutor::provider_failure("must not execute"); - let mut state = SyncState::new("googledrive", "connection"); - let document = provider - .document( - &SyncScope::flat(), - "connection", - item( - json!({"fileId":"file-1","name":"Budget.pdf","mimeType":"application/pdf"}), - "key", - ), - &executor, - &mut state, - ) - .await - .expect("drive document"); - assert_eq!(document.document_id, "googledrive:file-1"); - assert_eq!(document.title, "Budget.pdf"); - assert!(document.content.contains("application/pdf")); - assert!(executor.calls.lock().expect("calls lock").is_empty()); -} - -#[test] -fn sheets_use_bounded_search_and_normalize_spreadsheet_shapes() { - let provider = GoogleSheetsSyncPipeline::new(client(), "connection"); - let args = provider.arguments( - &SyncScope::flat(), - &PipelineConfig::default(), - &SyncState::new("googlesheets", "connection"), - Some("ignored"), - ); - assert_eq!(args, json!({"query":"","max_results":25})); - let page = provider.extract_page( - &json!({"data":{"files":[{"spreadsheetId":"sheet"}],"next_page_token":" next "}}), - None, - ); - assert_eq!(page.items.len(), 1); - assert_eq!(page.next.as_deref(), Some("next")); - let sheet = json!({"spreadsheetId":"sheet","modified_time":"cursor"}); - assert_eq!(provider.dedup_key(&sheet).as_deref(), Some("sheet@cursor")); -} - -#[tokio::test] -async fn sheets_fetch_info_with_canonical_argument_and_accounting() { - let provider = GoogleSheetsSyncPipeline::new(client(), "unused"); - let executor = StubExecutor::succeeds(json!({"data":{"properties":{"locale":"en_US"}}})); - let mut state = SyncState::new("googlesheets", "connection"); - let document = provider - .document( - &SyncScope::flat(), - "connection", - item( - json!({"spreadsheetId":"sheet-1","properties":{"title":"Forecast"}}), - "key", - ), - &executor, - &mut state, - ) - .await - .expect("sheets document"); - assert_eq!(document.title, "Forecast"); - assert!(document.content.contains("en_US")); - assert_eq!(state.run_requests, 2); - assert_eq!( - executor.calls.lock().expect("calls lock")[0].1, - json!({"spreadsheet_id":"sheet-1"}) - ); -} - -#[test] -fn outlook_filters_by_cursor_and_extracts_graph_skiptoken() { - let provider = OutlookSyncPipeline::new(client(), "connection").with_limits(4, 0); - let mut state = SyncState::new("outlook", "connection"); - state.cursor = Some("2026-07-01T01:02:03Z".into()); - let args = provider.arguments( - &SyncScope::flat(), - &PipelineConfig::default(), - &state, - Some("already-bare"), - ); - assert_eq!(args["top"], 1); - assert_eq!(args["skip_token"], "already-bare"); - assert_eq!(args["filter"], "receivedDateTime ge 2026-07-01T01:02:03Z"); - let page = provider.extract_page( - &json!({ - "value":[{"messageId":"mail"}], - "@odata.nextLink":"https://graph.example/messages?foo=1&$skiptoken=A%2BB&top=25" - }), - None, - ); - assert_eq!(page.items.len(), 1); - assert_eq!(page.next.as_deref(), Some("A%2BB")); - let mail = - json!({"messageId":"mail","received_date_time":"cursor","lastModifiedDateTime":"wrong"}); - assert_eq!(provider.dedup_key(&mail).as_deref(), Some("mail@cursor")); - assert_eq!( - provider.sort_cursor(&json!({"lastModifiedDateTime":"wrong"})), - None - ); -} - -#[tokio::test] -async fn outlook_document_uses_subject_and_raw_message_body() { - let provider = OutlookSyncPipeline::new(client(), "unused"); - let executor = StubExecutor::provider_failure("must not execute"); - let mut state = SyncState::new("outlook", "connection"); - let document = provider - .document( - &SyncScope::flat(), - "connection", - item( - json!({"id":"mail-1","subject":"Hello","body":{"content":"World"}}), - "key", - ), - &executor, - &mut state, - ) - .await - .expect("outlook document"); - assert_eq!(document.title, "Hello"); - assert!(document.content.contains("World")); - assert!(executor.calls.lock().expect("calls lock").is_empty()); -} - -#[test] -fn todoist_handles_bare_and_wrapped_arrays_and_fingerprints_edits() { - let provider = TodoistSyncPipeline::new(client(), "connection"); - assert_eq!( - provider.arguments( - &SyncScope::flat(), - &PipelineConfig::default(), - &SyncState::new("todoist", "connection"), - Some("ignored"), - ), - json!({}) - ); - assert_eq!( - provider - .extract_page(&json!([{"id":"1"}]), None) - .items - .len(), - 1 - ); - assert_eq!( - provider - .extract_page(&json!({"data":{"tasks":[{"id":"2"}]}}), None) - .items - .len(), - 1 - ); - let first = json!({"id":"task","content":"write","nested":{"b":2,"a":1}}); - let reordered = json!({"nested":{"a":1,"b":2},"content":"write","id":"task"}); - let edited = json!({"id":"task","content":"ship","nested":{"a":1,"b":2}}); - assert_eq!(provider.dedup_key(&first), provider.dedup_key(&reordered)); - assert_ne!(provider.dedup_key(&first), provider.dedup_key(&edited)); - assert_eq!(provider.sort_cursor(&first), None); -} - -#[tokio::test] -async fn todoist_document_combines_task_text_and_description() { - let provider = TodoistSyncPipeline::new(client(), "unused"); - let executor = StubExecutor::provider_failure("must not execute"); - let mut state = SyncState::new("todoist", "connection"); - let document = provider - .document( - &SyncScope::flat(), - "connection", - item( - json!({"task_id":9,"content":"Write tests","description":"Cover failures"}), - "key", - ), - &executor, - &mut state, - ) - .await - .expect("todoist document"); - assert_eq!(document.document_id, "todoist:9"); - assert_eq!(document.content, "Write tests\n\nCover failures"); - assert!(executor.calls.lock().expect("calls lock").is_empty()); - - let fallback = provider - .document( - &SyncScope::flat(), - "connection", - item(json!({"id":"task-raw","priority":4}), "key"), - &executor, - &mut state, - ) - .await - .expect("task without content uses raw payload"); - assert_eq!(fallback.title, "Todoist task task-raw"); - assert!(fallback.content.contains("priority")); -} - -#[tokio::test] -async fn scoped_work_providers_normalize_directory_pages_and_documents() { - let clickup = ClickUpSyncPipeline::new(client(), "connection"); - assert_eq!(clickup.id(), "composio:clickup"); - assert_eq!(clickup.kind(), SyncPipelineKind::Composio); - let click_executor = QueueExecutor::new([ - json!({"user":{"id":42}}), - json!({"teams":[{"id":"team-1"},{"name":"missing id"}]}), - ]); - let mut click_state = SyncState::new("clickup", "connection"); - let click_scopes = clickup - .scopes(&click_executor, "connection", &mut click_state) - .await - .expect("clickup scopes"); - assert_eq!(click_scopes.len(), 1); - assert_eq!(click_scopes[0].label, "workspace:team-1"); - assert_eq!(click_scopes[0].metadata["user_id"], "42"); - assert_eq!(click_state.run_requests, 2); - let click_args = clickup.arguments( - &click_scopes[0], - &PipelineConfig::default(), - &click_state, - Some("3"), - ); - assert_eq!(click_args["team_id"], "team-1"); - assert_eq!(click_args["assignees"], json!(["42"])); - assert_eq!(click_args["page"], 3); - assert_eq!(clickup.max_pages(), 20); - let click_tasks = vec![json!({"id":"task"}); 50]; - let click_page = clickup.extract_page(&json!({"tasks":click_tasks}), Some("3")); - assert_eq!(click_page.next.as_deref(), Some("4")); - let click_raw = json!({"task_id":"task-1","name":"Ship","dateUpdated":123}); - assert_eq!(clickup.dedup_key(&click_raw).as_deref(), Some("task-1@123")); - let click_document = clickup - .document( - &click_scopes[0], - "connection", - item(click_raw, "key"), - &click_executor, - &mut click_state, - ) - .await - .expect("clickup document"); - assert_eq!(click_document.title, "Ship"); - assert_eq!(click_document.metadata["workspace_id"], "team-1"); - - let github = GitHubSyncPipeline::new(client(), "connection"); - assert_eq!(github.id(), "composio:github"); - let github_executor = QueueExecutor::new([json!({"data":{"login":"alice"}})]); - let mut github_state = SyncState::new("github", "connection"); - let github_scopes = github - .scopes(&github_executor, "connection", &mut github_state) - .await - .expect("github scopes"); - assert_eq!(github_scopes[0].label, "involves:alice"); - github_state.cursor = Some("2026-01-02T00:00:00Z".into()); - let github_args = github.arguments( - &github_scopes[0], - &PipelineConfig::default(), - &github_state, - Some("2"), - ); - assert_eq!( - github_args["q"], - "involves:alice updated:>2026-01-02T00:00:00Z" - ); - assert_eq!(github_args["page"], 2); - assert!(github.server_side_depth()); - let github_items = vec![json!({"id":"issue"}); 50]; - assert_eq!( - github - .extract_page(&json!({"data":{"items":github_items}}), None) - .next - .as_deref(), - Some("2") - ); - let github_raw = json!({ - "html_url":"https://github.com/acme/widget/issues/7", - "title":"Bug", - "updatedAt":"cursor" - }); - assert_eq!( - github.dedup_key(&github_raw).as_deref(), - Some("acme/widget#7@cursor") - ); - let github_document = github - .document( - &github_scopes[0], - "connection", - item(github_raw, "key"), - &github_executor, - &mut github_state, - ) - .await - .expect("github document"); - assert_eq!(github_document.document_id, "github:acme/widget#7"); - - let linear = LinearSyncPipeline::new(client(), "connection"); - assert_eq!(linear.id(), "composio:linear"); - let linear_executor = QueueExecutor::new([json!({"data":{"nodes":[{"id":"user-1"}]}})]); - let mut linear_state = SyncState::new("linear", "connection"); - let linear_scopes = linear - .scopes(&linear_executor, "connection", &mut linear_state) - .await - .expect("linear scopes"); - assert_eq!(linear_scopes[0].id, "user-1"); - let linear_args = linear.arguments( - &linear_scopes[0], - &PipelineConfig::default(), - &linear_state, - Some("cursor-2"), - ); - assert_eq!(linear_args["assigneeId"], "user-1"); - assert_eq!(linear_args["after"], "cursor-2"); - let linear_page = linear.extract_page( - &json!({"data":{"issues":{"nodes":[{"identifier":"ENG-7"}],"pageInfo":{"hasNextPage":true,"endCursor":"next"}}}}), - None, - ); - assert_eq!(linear_page.items.len(), 1); - assert_eq!(linear_page.next.as_deref(), Some("next")); - let linear_raw = json!({"identifier":"ENG-7","title":"Fix it","updated_at":"cursor"}); - assert_eq!( - linear.dedup_key(&linear_raw).as_deref(), - Some("ENG-7@cursor") - ); - let linear_document = linear - .document( - &linear_scopes[0], - "connection", - item(linear_raw, "key"), - &linear_executor, - &mut linear_state, - ) - .await - .expect("linear document"); - assert_eq!(linear_document.title, "Fix it"); -} - -#[tokio::test] -async fn notion_and_slack_cover_secondary_fetch_and_per_scope_cursor_contracts() { - let notion = NotionSyncPipeline::new(client(), "connection"); - assert_eq!(notion.id(), "composio:notion"); - assert_eq!(notion.max_pages(), 20); - let notion_args = notion.arguments( - &SyncScope::flat(), - &PipelineConfig::default(), - &SyncState::new("notion", "connection"), - Some("page-2"), - ); - assert_eq!(notion_args["page_size"], 25); - assert_eq!(notion_args["start_cursor"], "page-2"); - // Composio rejects `NOTION_FETCH_DATA` without `fetch_type` ("Following - // fields are missing"), which broke every periodic Notion sync tick. - assert_eq!(notion_args["fetch_type"], "pages"); - let notion_page = notion.extract_page( - &json!({"data":{"results":[{"pageId":"page-1"}],"next_cursor":"next"}}), - None, - ); - assert_eq!(notion_page.items.len(), 1); - assert_eq!(notion_page.next.as_deref(), Some("next")); - let notion_raw = json!({ - "pageId":"page-1", - "lastEditedTime":"cursor", - "properties":{"Name":{"type":"title","title":[{"plain_text":"Road"},{"plain_text":"map"}]}} - }); - assert_eq!( - notion.dedup_key(¬ion_raw).as_deref(), - Some("page-1@cursor") - ); - let notion_executor = QueueExecutor::new([json!({"response_data":{"markdown":"# Body"}})]); - let mut notion_state = SyncState::new("notion", "connection"); - let notion_document = notion - .document( - &SyncScope::flat(), - "connection", - item(notion_raw, "key"), - ¬ion_executor, - &mut notion_state, - ) - .await - .expect("notion document"); - assert_eq!(notion_document.title, "Roadmap"); - assert_eq!(notion_document.content, "# Body"); - assert_eq!(notion_state.run_requests, 1); - - let slack = SlackSyncPipeline::new(client(), "connection"); - assert_eq!(slack.id(), "composio:slack"); - assert!(slack.per_scope_cursors()); - assert!(slack.server_side_depth()); - assert!(slack.tolerate_scope_errors()); - assert!(slack.retain_dedup_keys()); - let slack_executor = QueueExecutor::new([ - json!({"members":[ - {"id":"U1","profile":{"display_name":"Alice"}}, - {"id":"U2","real_name":"Bob"}, - {"name":"missing id"} - ]}), - json!({"channels":[ - {"id":"C1","name":"general","is_private":false}, - {"id":"C2","name":"secret","is_private":true} - ]}), - ]); - let mut slack_state = SyncState::new("slack", "connection"); - let slack_scopes = slack - .scopes(&slack_executor, "connection", &mut slack_state) - .await - .expect("slack scopes"); - assert_eq!(slack_scopes.len(), 2); - assert_eq!(slack_scopes[0].label, "#general"); - assert_eq!(slack_scopes[1].label, "private:secret"); - assert_eq!(slack_scopes[0].metadata["users"]["U1"], "Alice"); - slack.advance_scope_cursor(&mut slack_state, &slack_scopes[0], "1700000000.000001"); - let slack_args = slack.arguments( - &slack_scopes[0], - &PipelineConfig::default(), - &slack_state, - Some("page-2"), - ); - assert_eq!(slack_args["channel"], "C1"); - assert_eq!(slack_args["oldest"], "1700000000.000001"); - assert_eq!(slack_args["cursor"], "page-2"); - let slack_page = slack.extract_page( - &json!({"messages":[{"ts":"1700000001.000002","text":"Hi"}],"response_metadata":{"next_cursor":" next "}}), - None, - ); - assert_eq!(slack_page.items.len(), 1); - assert_eq!(slack_page.next.as_deref(), Some("next")); - let slack_raw = json!({"ts":"1700000001.000002","text":"Hi <@U2>","user":"U1"}); - assert_eq!( - slack.dedup_key(&slack_raw).as_deref(), - Some("1700000001.000002") - ); - assert_eq!(slack.dedup_key(&json!({"ts":"bad","text":"Hi"})), None); - let slack_document = slack - .document( - &slack_scopes[0], - "connection", - item(slack_raw, "key"), - &slack_executor, - &mut slack_state, - ) - .await - .expect("slack document"); - assert_eq!(slack_document.title, "Slack #general from Alice"); - assert_eq!(slack_document.content, "[1700000001.000002] Alice: Hi @Bob"); - assert_eq!(slack_document.metadata["channel_id"], "C1"); -} - -#[tokio::test] -async fn scoped_provider_failures_and_content_fallbacks_are_explicit() { - let successful = |data| ExecuteResponse { - data, - successful: true, - error: None, - cost_usd: 0.0, - markdown_formatted: None, - attempts: 1, - }; - let rejected = ExecuteResponse { - data: Value::Null, - successful: false, - error: Some("directory denied".into()), - cost_usd: 0.0, - markdown_formatted: None, - attempts: 2, - }; - - let clickup = ClickUpSyncPipeline::new(client(), "connection"); - let missing_click_user = QueueExecutor::new([json!({"user":{}})]); - let mut click_state = SyncState::new("clickup", "connection"); - let error = clickup - .scopes(&missing_click_user, "connection", &mut click_state) - .await - .expect_err("clickup user id is required"); - assert!(error.to_string().contains("returned no user id")); - - let github = GitHubSyncPipeline::new(client(), "connection"); - let missing_login = QueueExecutor::new([json!({"data":{}})]); - let mut github_state = SyncState::new("github", "connection"); - let error = github - .scopes(&missing_login, "connection", &mut github_state) - .await - .expect_err("github login is required"); - assert!(error.to_string().contains("returned no login")); - assert_eq!( - github.dedup_key(&json!({"html_url":"https://github.com/too-short"})), - None - ); - - let linear = LinearSyncPipeline::new(client(), "connection"); - let missing_viewer = QueueExecutor::new([json!({"nodes":[]})]); - let mut linear_state = SyncState::new("linear", "connection"); - let error = linear - .scopes(&missing_viewer, "connection", &mut linear_state) - .await - .expect_err("linear viewer id is required"); - assert!(error.to_string().contains("returned no viewer id")); - assert_eq!( - linear - .extract_page( - &json!({"nodes":[],"pageInfo":{"hasNextPage":false,"endCursor":"ignored"}}), - None, - ) - .next, - None - ); - - let slack_executor = QueueExecutor::from_responses([ - successful(json!({ - "members":[{"id":"U1","name":"alice"}], - "response_metadata":{"next_cursor":"users-2"} - })), - rejected, - successful(json!({ - "channels":[{"id":"C1","name":"one"}], - "response_metadata":{"next_cursor":"channels-2"} - })), - successful(json!({"channels":[{"id":"C2"}]})), - ]); - let slack = SlackSyncPipeline::new(client(), "connection"); - let mut slack_state = SyncState::new("slack", "connection"); - let scopes = slack - .scopes(&slack_executor, "connection", &mut slack_state) - .await - .expect("slack directory tolerates rejected second user page"); - assert_eq!(scopes.len(), 2); - assert_eq!(scopes[1].label, "#C2"); - assert_eq!(slack_state.run_requests, 5); - { - let calls = slack_executor.calls.lock().expect("calls lock"); - assert_eq!(calls[1].0, "SLACK_LIST_ALL_USERS"); - assert_eq!(calls[1].1["cursor"], "users-2"); - assert_eq!(calls[3].1["cursor"], "channels-2"); - } - - let notion = NotionSyncPipeline::new(client(), "connection"); - let empty_markdown = QueueExecutor::new([json!({"markdown":" "})]); - let mut notion_state = SyncState::new("notion", "connection"); - let fallback = notion - .document( - &SyncScope::flat(), - "connection", - item(json!({"id":"page-2","name":"Fallback title"}), "key"), - &empty_markdown, - &mut notion_state, - ) - .await - .expect("blank markdown falls back to page JSON"); - assert_eq!(fallback.title, "Fallback title"); - assert!(fallback.content.contains("page-2")); -} - -#[tokio::test] -async fn pipeline_initialization_is_noop_and_backfill_honors_exhausted_budget() { - let host = Arc::new(NoopSyncHost::default()); - let context = sync_context(host.clone()); - let config = PipelineConfig::default(); - let calendar = GoogleCalendarSyncPipeline::new(client(), "connection"); - let docs = GoogleDocsSyncPipeline::new(client(), "connection"); - let drive = GoogleDriveSyncPipeline::new(client(), "connection"); - let sheets = GoogleSheetsSyncPipeline::new(client(), "connection"); - let outlook = OutlookSyncPipeline::new(client(), "connection"); - let todoist = TodoistSyncPipeline::new(client(), "connection"); - let clickup = ClickUpSyncPipeline::new(client(), "connection"); - let github = GitHubSyncPipeline::new(client(), "connection"); - let linear = LinearSyncPipeline::new(client(), "connection"); - let notion = NotionSyncPipeline::new(client(), "connection"); - let slack = SlackSyncPipeline::new(client(), "connection"); - let backfill = SlackSearchBackfillPipeline::new(client(), "connection", 0); - let pipelines: [&dyn SyncPipeline; 12] = [ - &calendar, &docs, &drive, &sheets, &outlook, &todoist, &clickup, &github, &linear, ¬ion, - &slack, &backfill, - ]; - for pipeline in pipelines { - pipeline - .init(&config, &context) - .await - .expect("provider initialization"); - } - assert_eq!(backfill.id(), "composio:slack:search-backfill"); - assert_eq!(backfill.kind(), SyncPipelineKind::Composio); - - let mut state = SyncState::new("slack", "connection"); - state.daily_budget.limit = 0; - state - .save(host.as_ref()) - .await - .expect("seed exhausted state"); - let outcome = backfill - .tick(&config, &context) - .await - .expect("exhausted backfill exits without provider I/O"); - assert_eq!(outcome.records_ingested, 0); - assert_eq!( - outcome.note.as_deref(), - Some("slack search-backfill skipped: daily budget exhausted") - ); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack.rs deleted file mode 100644 index 2b032fb5..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack.rs +++ /dev/null @@ -1,468 +0,0 @@ -use std::collections::HashMap; - -use async_trait::async_trait; -use chrono::Utc; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use super::slack_parse::{ - decode_cursors, next_cursor, parse_ts, replace_mentions, search_matches, search_total_pages, -}; -use crate::sync::composio::providers::sync_state::{PersistedSyncState, SyncState}; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_CHANNELS: &str = "SLACK_LIST_CONVERSATIONS"; -const ACTION_HISTORY: &str = "SLACK_FETCH_CONVERSATION_HISTORY"; -const ACTION_SEARCH: &str = "SLACK_SEARCH_MESSAGES"; - -pub struct SlackSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, - backfill_days: i64, -} - -pub struct SlackSearchBackfillPipeline { - client: ComposioClient, - connection_id: String, - backfill_days: i64, - max_pages: u32, -} - -impl SlackSearchBackfillPipeline { - pub fn new( - client: ComposioClient, - connection_id: impl Into, - backfill_days: i64, - ) -> Self { - Self { - client, - connection_id: connection_id.into(), - backfill_days: backfill_days.max(1), - max_pages: 50, - } - } -} - -#[async_trait] -impl SyncPipeline for SlackSearchBackfillPipeline { - fn id(&self) -> &str { - "composio:slack:search-backfill" - } - - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - - async fn tick(&self, _: &PipelineConfig, context: &SyncContext) -> anyhow::Result { - let mut state = - SyncState::load(context.state.as_ref(), "slack", &self.connection_id).await?; - if state.budget_exhausted() { - return Ok(SyncOutcome { - note: Some("slack search-backfill skipped: daily budget exhausted".into()), - ..SyncOutcome::default() - }); - } - - // Run the body, then save the state on BOTH paths. `checked_execute` - // records billable requests and provider cost into `state` before it - // returns an error; propagating that error before the save would lose - // the accounting and leave the daily budget unadvanced, so a backfill - // that fails repeatedly could keep calling the search action unbudgeted. - // Same contract `run_incremental_sync` keeps. - let result = self.run_backfill(&mut state, context).await; - state.last_sync_at_ms = Some(Utc::now().timestamp_millis() as u64); - state.save(context.state.as_ref()).await?; - result - } -} - -impl SlackSearchBackfillPipeline { - async fn run_backfill( - &self, - state: &mut SyncState, - context: &SyncContext, - ) -> anyhow::Result { - let directory = SlackSyncPipeline::new(self.client.clone(), self.connection_id.clone()); - let scopes = directory - .scopes(&self.client, &self.connection_id, state) - .await?; - let channels: HashMap<_, _> = scopes - .into_iter() - .map(|scope| (scope.id.clone(), scope)) - .collect(); - let users = channels - .values() - .find_map(|scope| scope.metadata.get("users").and_then(Value::as_object)); - let after = (Utc::now() - chrono::Duration::days(self.backfill_days)) - .format("%Y-%m-%d") - .to_string(); - let mut page = 1u32; - let mut total_pages = 1u32; - let mut stored = 0u32; - - loop { - if state.budget_exhausted() || page > self.max_pages { - break; - } - let response = checked_execute( - &self.client, - ACTION_SEARCH, - serde_json::json!({ - "query": format!("after:{after}"), - "count": 100, - "sort": "timestamp", - "sort_dir": "asc", - "page": page, - }), - &self.connection_id, - state, - ) - .await?; - if page == 1 { - total_pages = search_total_pages(&response.data).min(self.max_pages); - } - let matches = search_matches(&response.data); - let fetched = matches.len(); - for raw in matches { - let Some(ts) = pick_str(&raw, &["ts"]) else { - continue; - }; - if parse_ts(&ts).is_none() { - continue; - } - let Some(text) = pick_str(&raw, &["text"]).filter(|text| !text.trim().is_empty()) - else { - continue; - }; - let Some(channel_id) = pick_str(&raw, &["channel.id", "channel_id"]) else { - continue; - }; - let Some(scope) = channels.get(&channel_id) else { - tracing::warn!(channel_id, "[sync:slack-search] unknown channel skipped"); - continue; - }; - let author_id = - pick_str(&raw, &["user", "bot_id"]).unwrap_or_else(|| "unknown".into()); - let author = users - .and_then(|users| users.get(&author_id)) - .and_then(Value::as_str) - .unwrap_or(&author_id); - let text = replace_mentions(&text, users); - let mut doc = document( - "slack", - &self.connection_id, - &format!("{channel_id}:{ts}"), - format!("Slack {} from {author}", scope.label), - format!("[{ts}] {author}: {text}"), - raw, - ); - doc.metadata["channel_id"] = Value::String(channel_id); - doc.metadata["channel_label"] = Value::String(scope.label.clone()); - context.documents.store(doc).await?; - stored = stored.saturating_add(1); - } - if fetched == 0 || page >= total_pages { - break; - } - page = page.saturating_add(1); - } - - Ok(SyncOutcome { - records_ingested: stored, - more_pending: page < total_pages, - actions_called: state.run_requests, - provider_cost_usd: state.run_provider_cost_usd, - note: Some(format!( - "slack search-backfill: pages={page} records={stored}" - )), - tree_ingest_failures: 0, - }) - } -} - -impl SlackSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 20, - page_size: 200, - backfill_days: 30, - } - } -} - -#[async_trait] -impl SyncPipeline for SlackSyncPipeline { - fn id(&self) -> &str { - "composio:slack" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for SlackSyncPipeline { - fn toolkit(&self) -> &'static str { - "slack" - } - fn action(&self) -> &'static str { - ACTION_HISTORY - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn per_scope_cursors(&self) -> bool { - true - } - fn server_side_depth(&self) -> bool { - true - } - fn tolerate_scope_errors(&self) -> bool { - true - } - fn retain_dedup_keys(&self) -> bool { - true - } - - fn advance_scope_cursor(&self, state: &mut SyncState, scope: &SyncScope, cursor: &str) { - let mut cursors = decode_cursors(state.cursor.as_deref()); - cursors.insert(scope.id.clone(), cursor.into()); - state.cursor = serde_json::to_string(&cursors).ok(); - } - - async fn scopes( - &self, - executor: &dyn ActionExecutor, - connection_id: &str, - state: &mut SyncState, - ) -> anyhow::Result> { - let users = fetch_users(executor, connection_id, state).await; - let mut cursor: Option = None; - let mut channels = Vec::new(); - for _ in 0..20 { - if state.budget_exhausted() { - break; - } - let mut args = serde_json::json!({"limit": 200, "types": "public_channel,private_channel", "exclude_archived": true}); - if let Some(cursor) = cursor.as_deref() { - args["cursor"] = Value::String(cursor.into()); - } - let response = - checked_execute(executor, ACTION_CHANNELS, args, connection_id, state).await?; - channels.extend(first_array( - &response.data, - &["/data/channels", "/channels", "/data/data/channels"], - )); - cursor = next_cursor(&response.data); - if cursor.is_none() { - break; - } - } - Ok(channels - .into_iter() - .filter_map(|channel| { - let id = pick_str(&channel, &["id", "data.id"])?; - let name = pick_str(&channel, &["name", "data.name"]).unwrap_or_else(|| id.clone()); - let private = channel - .get("is_private") - .and_then(Value::as_bool) - .unwrap_or(false); - let label = if private { - format!("private:{name}") - } else { - format!("#{name}") - }; - Some( - SyncScope::named(id, label).with_metadata(serde_json::json!({ - "channel": channel, - "users": users, - })), - ) - }) - .collect()) - } - - fn arguments( - &self, - scope: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - let cursors = decode_cursors(state.cursor.as_deref()); - let oldest = cursors.get(&scope.id).cloned().unwrap_or_else(|| { - format!( - "{}.000000", - (Utc::now() - - chrono::Duration::days( - config - .sync_depth_days - .map(i64::from) - .unwrap_or(self.backfill_days) - )) - .timestamp() - ) - }); - let mut args = serde_json::json!({"channel": scope.id, "oldest": oldest, "inclusive": false, "limit": self.page_size}); - if let Some(page) = page { - args["cursor"] = Value::String(page.into()); - } - args - } - - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &["/data/messages", "/messages", "/data/data/messages"], - ), - next: next_cursor(data), - } - } - - fn dedup_key(&self, item: &Value) -> Option { - let ts = pick_str(item, &["ts", "data.ts"])?; - parse_ts(&ts)?; - let text = pick_str(item, &["text", "data.text"])?; - (!text.trim().is_empty()).then_some(ts) - } - - fn sort_cursor(&self, item: &Value) -> Option { - pick_str(item, &["ts", "data.ts"]) - } - - async fn document( - &self, - scope: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let ts = pick_str(&item.raw, &["ts", "data.ts"]).unwrap_or(item.dedup_key); - let raw_text = pick_str(&item.raw, &["text", "data.text"]).unwrap_or_default(); - let author_id = pick_str( - &item.raw, - &["user", "data.user", "username", "data.username"], - ) - .unwrap_or_else(|| "unknown".into()); - let users = scope.metadata.get("users").and_then(Value::as_object); - let author = users - .and_then(|users| users.get(&author_id)) - .and_then(Value::as_str) - .unwrap_or(&author_id) - .to_owned(); - let text = replace_mentions(&raw_text, users); - let title = format!("Slack {} from {}", scope.label, author); - let content = format!("[{ts}] {author}: {text}"); - let mut result = document( - "slack", - connection_id, - &format!("{}:{ts}", scope.id), - title, - content, - item.raw, - ); - result.metadata["channel_id"] = Value::String(scope.id.clone()); - result.metadata["channel_label"] = Value::String(scope.label.clone()); - Ok(result) - } -} - -async fn fetch_users( - executor: &dyn ActionExecutor, - connection_id: &str, - state: &mut SyncState, -) -> HashMap { - let mut users = HashMap::new(); - let mut cursor: Option = None; - for page in 0..20 { - if state.budget_exhausted() { - break; - } - let mut arguments = serde_json::json!({"limit": 200}); - if let Some(cursor) = cursor.as_deref() { - arguments["cursor"] = Value::String(cursor.into()); - } - let response = match executor - .execute("SLACK_LIST_ALL_USERS", arguments, Some(connection_id)) - .await - { - Ok(response) => response, - Err(error) => { - if let Some(error) = error.downcast_ref::() { - state.record_requests(error.attempts); - } - tracing::warn!(page, %error, "[sync:slack] user directory fetch failed; using collected users"); - break; - } - }; - state.record_action(response.attempts, response.cost_usd); - if !response.successful { - tracing::warn!( - page, - error = response.error.as_deref().unwrap_or("provider failure"), - "[sync:slack] user directory rejected; using collected users" - ); - break; - } - let members = first_array( - &response.data, - &[ - "/data/members", - "/members", - "/data/users", - "/users", - "/data/data/members", - ], - ); - for member in members { - let Some(id) = pick_str(&member, &["id"]) else { - continue; - }; - if let Some(name) = [ - "profile.display_name", - "profile.real_name", - "real_name", - "name", - "profile.display_name_normalized", - "profile.real_name_normalized", - ] - .iter() - .find_map(|path| pick_str(&member, &[*path])) - { - users.insert(id, name); - } - } - cursor = next_cursor(&response.data); - if cursor.is_none() { - break; - } - } - users -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack_parse.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack_parse.rs deleted file mode 100644 index 8efb1e9a..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack_parse.rs +++ /dev/null @@ -1,98 +0,0 @@ -//! Slack response cursor, mention, and timestamp parsing. - -use std::collections::BTreeMap; -use std::sync::OnceLock; - -use serde_json::Value; - -use super::common::first_array; - -/// Return the cached matcher for Slack `<@USERID>` mentions. -pub(super) fn mention_regex() -> &'static regex::Regex { - static REGEX: OnceLock = OnceLock::new(); - REGEX.get_or_init(|| regex::Regex::new(r"<@(U[A-Z0-9]+)>").expect("Slack mention regex")) -} - -/// Replace Slack mention tokens with resolved display names, falling back to -/// the raw user id when the optional user map has no match. -pub(super) fn replace_mentions( - text: &str, - users: Option<&serde_json::Map>, -) -> String { - mention_regex() - .replace_all(text, |captures: ®ex::Captures<'_>| { - let id = &captures[1]; - let resolved = users - .and_then(|users| users.get(id)) - .and_then(Value::as_str) - .unwrap_or(id); - format!("@{resolved}") - }) - .into_owned() -} - -/// Read the first non-blank next cursor across supported response envelopes. -pub(super) fn next_cursor(data: &Value) -> Option { - [ - "/data/response_metadata/next_cursor", - "/response_metadata/next_cursor", - "/data/next_cursor", - "/next_cursor", - "/data/data/response_metadata/next_cursor", - ] - .iter() - .find_map(|path| { - data.pointer(path) - .and_then(Value::as_str) - .map(str::trim) - .filter(|cursor| !cursor.is_empty()) - }) - .map(str::to_owned) -} - -/// Extract Slack search matches across legacy and nested response envelopes. -pub(super) fn search_matches(data: &Value) -> Vec { - first_array( - data, - &[ - "/data/messages/matches", - "/messages/matches", - "/data/data/messages/matches", - "/messages", - ], - ) -} - -/// Extract the search page count, defaulting to one when paging is absent. -pub(super) fn search_total_pages(data: &Value) -> u32 { - [ - "/data/messages/paging/pages", - "/messages/paging/pages", - "/data/data/messages/paging/pages", - "/pages", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_u64)) - .unwrap_or(1) as u32 -} - -/// Decode persisted per-scope cursors, returning an empty map for absent or -/// malformed JSON so synchronization can restart safely. -pub(super) fn decode_cursors(raw: Option<&str>) -> BTreeMap { - raw.and_then(|raw| serde_json::from_str(raw).ok()) - .unwrap_or_default() -} - -/// Parse Slack's `seconds.fraction` timestamp into numeric components. -/// Missing fractions become zero; malformed numeric components return `None`. -pub(super) fn parse_ts(ts: &str) -> Option<(i64, u64)> { - let mut parts = ts.splitn(2, '.'); - Some(( - parts.next()?.parse().ok()?, - parts.next().unwrap_or("0").parse().ok()?, - )) -} - -#[cfg(test)] -#[path = "slack_parse_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack_parse_tests.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack_parse_tests.rs deleted file mode 100644 index be3224a8..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack_parse_tests.rs +++ /dev/null @@ -1,63 +0,0 @@ -//! Tests for tolerant Slack cursor, mention, and timestamp parsing. - -use serde_json::json; - -use super::*; - -#[test] -fn mentions_resolve_known_users_and_keep_unknown_ids() { - let users = serde_json::Map::from_iter([ - ("U123".to_string(), json!("Ada")), - ("U999".to_string(), json!(17)), - ]); - assert_eq!( - replace_mentions("hi <@U123>, ask <@U456> and <@U999>", Some(&users)), - "hi @Ada, ask @U456 and @U999" - ); - assert_eq!(replace_mentions("plain", None), "plain"); -} - -#[test] -fn cursor_parser_skips_blank_values_and_understands_nested_envelopes() { - let data = json!({ - "data": { - "response_metadata": { "next_cursor": " " }, - "next_cursor": " page-2 " - } - }); - assert_eq!(next_cursor(&data).as_deref(), Some("page-2")); - assert_eq!(next_cursor(&json!({"next_cursor": 2})), None); -} - -#[test] -fn malformed_cursor_json_restarts_with_an_empty_map() { - assert!(decode_cursors(None).is_empty()); - assert!(decode_cursors(Some("not json")).is_empty()); - assert!(decode_cursors(Some("[1,2]")).is_empty()); - assert_eq!( - decode_cursors(Some(r#"{"C1":"100.2","C2":"200.0"}"#)) - .get("C2") - .map(String::as_str), - Some("200.0") - ); -} - -#[test] -fn timestamp_parser_rejects_malformed_numeric_components() { - assert_eq!(parse_ts("1714003200.000100"), Some((1_714_003_200, 100))); - assert_eq!(parse_ts("1714003200"), Some((1_714_003_200, 0))); - for malformed in ["", "abc.1", "12.abc", "12.", ".1"] { - assert_eq!(parse_ts(malformed), None, "accepted {malformed:?}"); - } -} - -#[test] -fn search_helpers_default_safely_for_malformed_payloads() { - assert!(search_matches(&json!(null)).is_empty()); - assert!(search_matches(&json!({"messages": "wrong"})).is_empty()); - assert_eq!(search_total_pages(&json!({"pages": "many"})), 1); - assert_eq!( - search_total_pages(&json!({"data":{"data":{"messages":{"paging":{"pages":7}}}}})), - 7 - ); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/todoist.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/todoist.rs deleted file mode 100644 index cbbedc60..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/todoist.rs +++ /dev/null @@ -1,226 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_GET_ALL_TASKS: &str = "TODOIST_GET_ALL_TASKS"; - -/// Incremental Todoist synchronization through Composio. -/// -/// Todoist tasks are self-contained records (stable id + `created_at` -/// timestamp), so this follows the document-shaped pattern -/// (`LinearSyncPipeline`) rather than the message-shaped one: a single list -/// action, content taken directly from the task payload with no secondary -/// fetch. Todoist's active-tasks endpoint returns a plain array and is not -/// paginated, so there is no server-side incremental filter, and a task carries -/// no modification timestamp. Incremental behavior is therefore driven by the -/// orchestrator's client-side dedup (`synced_ids`) keyed on a payload -/// fingerprint (see [`dedup_key`](Self::dedup_key)) — an unchanged task is -/// skipped, while any edit re-ingests. -pub struct TodoistSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, -} - -impl TodoistSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 1, - } - } - - pub fn with_limits(mut self, max_pages: usize, _page_size: usize) -> Self { - self.max_pages = max_pages.max(1); - // Todoist active-tasks is unpaginated; the sibling `page_size` argument - // is accepted for signature parity but has no effect. - self - } -} - -#[async_trait] -impl SyncPipeline for TodoistSyncPipeline { - fn id(&self) -> &str { - "composio:todoist" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for TodoistSyncPipeline { - fn toolkit(&self) -> &'static str { - "todoist" - } - fn action(&self) -> &'static str { - ACTION_GET_ALL_TASKS - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn stop_on_empty_pending(&self) -> bool { - true - } - fn server_side_depth(&self) -> bool { - false - } - fn arguments( - &self, - _: &SyncScope, - _: &PipelineConfig, - _: &SyncState, - _page: Option<&str>, - ) -> Value { - // Todoist "get all active tasks" needs no required arguments and ignores - // pagination; do not invent a page token. - serde_json::json!({}) - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - // Todoist's active-tasks response is sometimes the bare task array - // (already unwrapped from the Composio `data` envelope by the client) - // and sometimes wrapped under `tasks`/`items`. Handle the top-level - // array first, then the wrapped shapes. - let items = data.as_array().cloned().unwrap_or_else(|| { - first_array( - data, - &[ - "/data/tasks", - "/tasks", - "/data/items", - "/items", - "/data/data", - ], - ) - }); - PageFetch { - items, - // Todoist active tasks are returned as a single unpaginated array. - next: None, - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "task_id", "data.task_id"])?; - // Todoist tasks have no modification timestamp, so `created_at` (which is - // immutable) would never change and edited tasks would never re-ingest. - // Key on a fingerprint of the task payload instead: any change to - // content/due/project yields a new key and re-ingests, while an - // unchanged task keeps its key and is deduped. - Some(format!("{id}@{}", payload_fingerprint(item))) - } - fn sort_cursor(&self, _item: &Value) -> Option { - // Todoist active tasks have no modification timestamp and the endpoint - // is unpaginated, so there is no meaningful sort cursor. Returning None - // is deliberate: the orchestrator's cursor-boundary short-circuit keys - // on `sort_cursor`, and using the immutable `created_at` would halt the - // scan (and skip re-ingest) for an edited task created before the - // persisted cursor. Freshness is handled entirely by `dedup_key`. - None - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "task_id", "data.task_id"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str( - &item.raw, - &["content", "data.content", "title", "data.title"], - ) - .unwrap_or_else(|| format!("Todoist task {id}")); - // A Todoist task's meaningful text is its `content` (title line) plus an - // optional `description`; store that as the document body so retrieval - // embeds the task text, not JSON syntax. Fall back to the raw payload - // only when the task carries no content field. - let content = match pick_str(&item.raw, &["content", "data.content"]) { - Some(text) => match pick_str(&item.raw, &["description", "data.description"]) { - Some(desc) if !desc.trim().is_empty() => format!("{text}\n\n{desc}"), - _ => text, - }, - None => serde_json::to_string_pretty(&item.raw)?, - }; - Ok(document( - "todoist", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} - -/// Stable content fingerprint of a task payload, used as the freshness half of -/// the dedup key. Computed as FNV-1a over a canonical serialization (object keys -/// sorted recursively). The key is **persisted** in `SyncState`, so the hash -/// must be stable across Rust toolchains and independent of `serde_json` map -/// ordering — `DefaultHasher` guarantees neither, and an unstable value would -/// silently re-ingest every task on a toolchain bump. -fn payload_fingerprint(item: &Value) -> u64 { - let mut canonical = String::new(); - write_canonical(item, &mut canonical); - // FNV-1a 64-bit — a fixed, specified algorithm. - let mut hash: u64 = 0xcbf2_9ce4_8422_2325; - for byte in canonical.as_bytes() { - hash ^= u64::from(*byte); - hash = hash.wrapping_mul(0x0000_0100_0000_01b3); - } - hash -} - -/// Serialize `value` with object keys sorted recursively so the byte stream is -/// canonical regardless of map insertion order. -fn write_canonical(value: &Value, out: &mut String) { - match value { - Value::Object(map) => { - out.push('{'); - let mut keys: Vec<&String> = map.keys().collect(); - keys.sort_unstable(); - for (index, key) in keys.iter().enumerate() { - if index > 0 { - out.push(','); - } - out.push_str(&serde_json::to_string(key).unwrap_or_default()); - out.push(':'); - write_canonical(&map[*key], out); - } - out.push('}'); - } - Value::Array(items) => { - out.push('['); - for (index, item) in items.iter().enumerate() { - if index > 0 { - out.push(','); - } - write_canonical(item, out); - } - out.push(']'); - } - other => out.push_str(&other.to_string()), - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/dispatcher.rs b/crates/tinymemory-core/src/sync/pipelines/dispatcher.rs deleted file mode 100644 index 4e40554d..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/dispatcher.rs +++ /dev/null @@ -1,123 +0,0 @@ -//! Pipeline registry and fault-isolated synchronization dispatcher. - -use std::collections::BTreeMap; -use std::sync::Arc; - -use serde::{Deserialize, Serialize}; - -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind}; - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SyncRunResult { - pub pipeline_id: String, - pub kind: SyncPipelineKind, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub outcome: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -#[derive(Default)] -pub struct SyncDispatcher { - pipelines: BTreeMap>, -} - -impl SyncDispatcher { - pub fn new() -> Self { - Self::default() - } - - pub fn register(&mut self, pipeline: Arc) -> anyhow::Result<()> { - let id = pipeline.id().trim(); - anyhow::ensure!(!id.is_empty(), "sync pipeline id must not be empty"); - anyhow::ensure!( - !self.pipelines.contains_key(id), - "sync pipeline already registered: {id}" - ); - tracing::debug!( - pipeline_id = id, - kind = pipeline.kind().as_str(), - "[memory_sync:dispatcher] registering pipeline" - ); - self.pipelines.insert(id.to_owned(), pipeline); - Ok(()) - } - - pub fn ids(&self) -> Vec<&str> { - self.pipelines.keys().map(String::as_str).collect() - } - - pub async fn init_all( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> Vec { - let mut results = Vec::with_capacity(self.pipelines.len()); - for (id, pipeline) in &self.pipelines { - tracing::debug!( - pipeline_id = id, - "[memory_sync:dispatcher] initializing pipeline" - ); - let result = pipeline.init(config, context).await; - results.push(SyncRunResult { - pipeline_id: id.clone(), - kind: pipeline.kind(), - outcome: result.as_ref().ok().map(|_| SyncOutcome::default()), - error: result.err().map(|error| error.to_string()), - }); - } - results - } - - pub async fn tick( - &self, - pipeline_id: &str, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - let pipeline = self - .pipelines - .get(pipeline_id) - .ok_or_else(|| anyhow::anyhow!("unknown sync pipeline: {pipeline_id}"))?; - tracing::debug!( - pipeline_id, - "[memory_sync:dispatcher] pipeline tick starting" - ); - let outcome = pipeline.tick(config, context).await; - match &outcome { - Ok(outcome) => tracing::debug!( - pipeline_id, - records = outcome.records_ingested, - more_pending = outcome.more_pending, - "[memory_sync:dispatcher] pipeline tick completed" - ), - Err(error) => { - tracing::warn!(pipeline_id, %error, "[memory_sync:dispatcher] pipeline tick failed") - } - } - outcome - } - - pub async fn tick_all( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> Vec { - let mut results = Vec::with_capacity(self.pipelines.len()); - for (id, pipeline) in &self.pipelines { - let result = pipeline.tick(config, context).await; - results.push(SyncRunResult { - pipeline_id: id.clone(), - kind: pipeline.kind(), - outcome: result.as_ref().ok().cloned(), - error: result.err().map(|error| error.to_string()), - }); - } - results - } -} - -#[cfg(test)] -#[path = "dispatcher_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/dispatcher_tests.rs b/crates/tinymemory-core/src/sync/pipelines/dispatcher_tests.rs deleted file mode 100644 index fe85502b..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/dispatcher_tests.rs +++ /dev/null @@ -1,207 +0,0 @@ -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; - -use async_trait::async_trait; - -use super::*; -use crate::sync::composio::providers::sync_state::SyncStateStore; -use crate::sync::pipelines::traits::{SkillDocSink, SkillDocument, SyncEvent, SyncEventSink}; - -struct FakePipeline { - id: &'static str, - fail: bool, - init_fail: bool, -} - -#[async_trait] -impl SyncPipeline for FakePipeline { - fn id(&self) -> &str { - self.id - } - - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Workspace - } - - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - if self.init_fail { - anyhow::bail!("expected init failure") - } - Ok(()) - } - - async fn tick(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result { - if self.fail { - anyhow::bail!("expected failure") - } - Ok(SyncOutcome { - records_ingested: 3, - more_pending: false, - actions_called: 0, - provider_cost_usd: 0.0, - note: None, - tree_ingest_failures: 0, - }) - } -} - -#[derive(Default)] -struct NoopHost(Mutex>); - -#[async_trait] -impl SkillDocSink for NoopHost { - async fn store(&self, _: SkillDocument) -> anyhow::Result<()> { - Ok(()) - } - - async fn delete(&self, _: &str, _: &str) -> anyhow::Result<()> { - Ok(()) - } -} - -#[async_trait] -impl SyncEventSink for NoopHost { - async fn emit(&self, _: SyncEvent) -> anyhow::Result<()> { - Ok(()) - } -} - -#[async_trait] -impl SyncStateStore for NoopHost { - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - Ok(self - .0 - .lock() - .unwrap() - .get(&format!("{namespace}:{key}")) - .cloned()) - } - - async fn set( - &self, - namespace: &str, - key: &str, - value: &serde_json::Value, - ) -> anyhow::Result<()> { - self.0 - .lock() - .unwrap() - .insert(format!("{namespace}:{key}"), value.clone()); - Ok(()) - } -} - -fn context() -> SyncContext { - let host = Arc::new(NoopHost::default()); - SyncContext { - events: host.clone(), - documents: host.clone(), - state: host, - } -} - -#[tokio::test] -async fn tick_all_is_deterministic_and_isolates_failures() { - let mut dispatcher = SyncDispatcher::new(); - dispatcher - .register(Arc::new(FakePipeline { - id: "z-fail", - fail: true, - init_fail: false, - })) - .unwrap(); - dispatcher - .register(Arc::new(FakePipeline { - id: "a-ok", - fail: false, - init_fail: false, - })) - .unwrap(); - assert_eq!(dispatcher.ids(), vec!["a-ok", "z-fail"]); - assert!(dispatcher - .register(Arc::new(FakePipeline { - id: "a-ok", - fail: false, - init_fail: false, - })) - .is_err()); - let results = dispatcher - .tick_all(&PipelineConfig::default(), &context()) - .await; - assert_eq!(results.len(), 2); - assert_eq!(results[0].outcome.as_ref().unwrap().records_ingested, 3); - assert!(results[1] - .error - .as_deref() - .unwrap() - .contains("expected failure")); -} - -#[tokio::test] -async fn register_rejects_blank_ids_and_tick_reports_unknown_pipeline() { - let mut dispatcher = SyncDispatcher::new(); - assert!(dispatcher - .register(Arc::new(FakePipeline { - id: " ", - fail: false, - init_fail: false, - })) - .is_err()); - let error = dispatcher - .tick("missing", &PipelineConfig::default(), &context()) - .await - .unwrap_err(); - assert!(error.to_string().contains("unknown sync pipeline")); -} - -#[tokio::test] -async fn init_all_and_individual_tick_preserve_success_and_failure_details() { - let mut dispatcher = SyncDispatcher::new(); - dispatcher - .register(Arc::new(FakePipeline { - id: "a-init-fails", - fail: false, - init_fail: true, - })) - .unwrap(); - dispatcher - .register(Arc::new(FakePipeline { - id: "b-ok", - fail: false, - init_fail: false, - })) - .unwrap(); - dispatcher - .register(Arc::new(FakePipeline { - id: "c-tick-fails", - fail: true, - init_fail: false, - })) - .unwrap(); - let config = PipelineConfig::default(); - let context = context(); - - let initialized = dispatcher.init_all(&config, &context).await; - assert!(initialized[0] - .error - .as_deref() - .unwrap() - .contains("expected init failure")); - assert!(initialized[1].outcome.is_some()); - assert_eq!( - dispatcher - .tick("b-ok", &config, &context) - .await - .unwrap() - .records_ingested, - 3 - ); - assert!(dispatcher - .tick("c-tick-fails", &config, &context) - .await - .is_err()); - - let encoded = serde_json::to_value(&initialized[0]).unwrap(); - assert_eq!(encoded["pipeline_id"], "a-init-fails"); - assert!(encoded.get("outcome").is_none()); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/host.rs b/crates/tinymemory-core/src/sync/pipelines/host.rs deleted file mode 100644 index ef7c9690..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/host.rs +++ /dev/null @@ -1,605 +0,0 @@ -//! The host side of the engine-free pipelines (#18 §B1): the sink adapter -//! over [`MemoryClient`](crate::store::MemoryClient), the Composio settings -//! mapping, and the runners the -//! rest of `core/src/sync/` calls. -//! -//! This is the piece §B5's acceptance rests on: a pipeline sees three -//! capabilities — events, documents, state — and every one resolves through -//! [`MemoryClient`](crate::store::MemoryClient), so whatever driver the host -//! bound serves the sync. - -use std::collections::HashMap; -use std::sync::{Arc, Mutex, OnceLock}; - -use async_trait::async_trait; -use tokio::sync::OwnedMutexGuard; - -use crate::store::MemoryClientRef; -use crate::sync::composio::providers::sync_state::SyncStateStore; -use crate::sync::pipelines::composio::{ - ClickUpSyncPipeline, ComposioClient, GitHubSyncPipeline, GmailSyncPipeline, LinearSyncPipeline, - NotionSyncPipeline, SlackSearchBackfillPipeline, SlackSyncPipeline, -}; -use crate::sync::pipelines::dispatcher::SyncDispatcher; -use crate::sync::pipelines::traits::{ - ComposioMode, ComposioSyncConfig, PipelineConfig, SecretString, SkillDocSink, SkillDocument, - SyncContext, SyncEvent, SyncEventSink, SyncOutcome, SyncPipeline, SyncRunError, -}; -use crate::Config; - -/// A failed pipeline run, with whatever usage it burned before failing. -#[derive(Debug)] -pub struct PipelineFailure { - pub message: String, - pub actions_called: u32, - pub provider_cost_usd: f64, -} - -impl std::fmt::Display for PipelineFailure { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.message) - } -} - -impl std::error::Error for PipelineFailure {} - -impl PipelineFailure { - pub fn without_usage(message: impl Into) -> Self { - Self { - message: message.into(), - actions_called: 0, - provider_cost_usd: 0.0, - } - } -} - -/// Adapter giving the pipelines their three capabilities over the bound -/// memory client. The engine's `HostSyncAdapter` remains for the engine's own -/// pipelines; this one exists so a Composio sync never needs the engine. -pub struct PipelineHost { - memory: MemoryClientRef, - config: Option>, - /// Items whose skill-store write committed but whose (non-corrupt) tree - /// ingest failed during this adapter's run. Read back into - /// [`SyncOutcome::tree_ingest_failures`] by the runners, because the - /// orchestrator only sees `store()`'s `Ok`/`Err` and the tolerated - /// failures deliberately return `Ok` (openhuman#5820). - tree_ingest_failures: std::sync::atomic::AtomicU32, -} - -impl PipelineHost { - /// An adapter that also feeds the memory tree after each stored document - /// (parity with the engine adapter's #5473 behaviour). - pub fn new(memory: MemoryClientRef, config: Arc) -> Self { - Self { - memory, - config: Some(config), - tree_ingest_failures: std::sync::atomic::AtomicU32::new(0), - } - } - - /// An adapter with no host config: documents are stored, tree ingest is - /// skipped. This is the shape a non-TinyCortex host uses. - pub fn without_tree_ingest(memory: MemoryClientRef) -> Self { - Self { - memory, - config: None, - tree_ingest_failures: std::sync::atomic::AtomicU32::new(0), - } - } - - /// The pipeline context over this adapter. - pub fn context(self: &Arc) -> SyncContext { - SyncContext { - events: self.clone(), - documents: self.clone(), - state: self.clone(), - } - } - - /// Tolerated (non-corrupt) tree-ingest failures recorded so far. - pub fn tree_ingest_failures(&self) -> u32 { - self.tree_ingest_failures - .load(std::sync::atomic::Ordering::Relaxed) - } -} - -#[async_trait] -impl SkillDocSink for PipelineHost { - async fn store(&self, document: SkillDocument) -> anyhow::Result<()> { - tracing::debug!( - toolkit = %document.toolkit, - connection_id = %document.connection_id, - document_id = %document.document_id, - "[memory_sync] storing synchronized document" - ); - self.memory - .store_skill_sync( - &document.namespace_skill_id, - &document.connection_id, - &document.title, - &document.content, - Some("tinycortex-sync".into()), - Some(document.metadata.clone()), - Some("medium".into()), - None, - None, - Some(document.document_id.clone()), - ) - .await - .map_err(anyhow::Error::msg)?; - - // #5473: additively reconnect the synced item to the memory tree — a - // best-effort secondary index; the skill store above is the source of - // truth and has committed. An ordinary failure here must NOT abort the - // sync (one poisonous item would stall the connection and re-buy the - // page on every retry), but it is COUNTED so the run's verdict can say - // "fetched, not tree-ingested" instead of success. Corruption is the - // exception (openhuman#5820): a malformed `chunks.db` fails every - // later item identically — 747 warns in 34 minutes in the incident — - // so it escalates through the shared recovery and aborts the run. - // The config-less adapter skips tree ingest entirely. - if let Some(config) = self.config.as_deref() { - if let Err(error) = ingest_into_tree(config, &document).await { - let rendered = format!("{error:#}"); - crate::corruption::escalate_or_count( - "composio tree ingest", - config, - error, - &self.tree_ingest_failures, - )?; - tracing::warn!( - error = %rendered, - document_id = %document.document_id, - "[memory_sync] tree ingest failed; skill store remains authoritative" - ); - } - } - Ok(()) - } - - async fn delete(&self, namespace_skill_id: &str, document_id: &str) -> anyhow::Result<()> { - let namespace = format!("skill-{}", namespace_skill_id.trim()); - tracing::debug!( - namespace, - document_id, - "[memory_sync] deleting synchronized document" - ); - self.memory - .delete_document(&namespace, document_id) - .await - .map(|_| ()) - .map_err(anyhow::Error::msg) - } -} - -/// Mirror of the engine adapter's tree reconnect (`engine::sync`'s -/// `ingest_document_into_memory_tree`): route the stored document through -/// core's ingest funnel under the same addressing scheme. -/// -/// # The scheme is the contract, not an implementation detail -/// -/// The tree scope a chunk seals under is `path_scope`, falling back to -/// `source_id`. Retrieval selects source trees by that scope and classifies -/// them by their **platform prefix** — `gmail:` is email, `slack:` is chat. -/// So the scope has to be `"{toolkit}:{connection_id}"`: one tree per -/// connection, named by a prefix retrieval knows. -/// -/// Passing no `path_scope` is not a smaller version of that. It makes each -/// item's own `source_id` the scope, which is a *tree per document*, named by -/// a prefix that matches no platform — the items are stored and then -/// unreachable, which is the #5473 defect this reconnect exists to fix. The -/// `source_id LIKE` prefix the memory-source status and diff snapshots query -/// by is keyed on this same scheme — see `sources::status::source_id_prefix`. -/// -/// Tags are a deliberate superset of the engine adapter's: it tags the toolkit -/// alone, this also tags `composio_sync`. Tags feed scoring and filtering, not -/// addressing, so the extra one costs nothing and marks the ingest path. -async fn ingest_into_tree(config: &Config, document: &SkillDocument) -> anyhow::Result<()> { - let toolkit = document.toolkit.trim().to_ascii_lowercase(); - let connection_id = document.connection_id.trim(); - // A blank toolkit or connection would yield a scope with no platform - // prefix (`":conn"`, `"gmail:"`), which no retrieval kind matches; skip - // rather than write an unreachable tree. The skill store still holds the - // item. - if toolkit.is_empty() || connection_id.is_empty() { - tracing::debug!( - document_id = %document.document_id, - "[memory_sync] skipping memory-tree ingest: item has no toolkit/connection scope" - ); - return Ok(()); - } - let tree_scope = format!("{toolkit}:{connection_id}"); - let source_id = format!("{tree_scope}:{}", document.document_id); - let owner = format!("{toolkit}-sync:{connection_id}"); - let doc = crate::ingest_pipeline::IngestDocumentInput { - provider: format!("composio:{toolkit}"), - title: document.title.clone(), - body: document.content.clone(), - modified_at: chrono::Utc::now(), - source_ref: Some(document.document_id.clone()), - }; - let tags = vec!["composio_sync".to_string(), toolkit]; - crate::ingest_pipeline::ingest_document_with_scope( - config, - &source_id, - &owner, - tags, - doc, - Some(tree_scope), - ) - .await - .map(|_| ()) - .map_err(|error| anyhow::anyhow!("memory-tree ingest failed for source `{source_id}`: {error}")) -} - -#[async_trait] -impl SyncEventSink for PipelineHost { - async fn emit(&self, event: SyncEvent) -> anyhow::Result<()> { - crate::events::publish(crate::events::MemoryEvent::SyncStageChanged { - trigger: "tinycortex".into(), - stage: super::traits::stage_name(event.stage).into(), - provider: Some(event.toolkit), - connection_id: event.connection_id, - detail: event.message, - source_id: Some(event.source_id), - }); - Ok(()) - } -} - -#[async_trait] -impl SyncStateStore for PipelineHost { - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - self.memory - .kv_get(Some(namespace), key) - .await - .map_err(anyhow::Error::msg) - } - - async fn set( - &self, - namespace: &str, - key: &str, - value: &serde_json::Value, - ) -> anyhow::Result<()> { - self.memory - .kv_set(Some(namespace), key, value) - .await - .map_err(anyhow::Error::msg) - } -} - -/// The Composio connection settings from the host's config — the same -/// resolution the engine seam performs, onto the local types. -pub fn composio_config(config: &Config) -> Result { - if config.composio().mode.eq_ignore_ascii_case("direct") { - let api_key = crate::composio_host::api_key(config) - .or_else(|| config.composio().api_key.clone()) - .ok_or_else(|| "Composio direct API key is not configured".to_string())?; - Ok(ComposioSyncConfig { - mode: ComposioMode::Direct, - base_url: "https://backend.composio.dev/api/v3".into(), - api_key: Some(SecretString::new(api_key)), - bearer_token: None, - entity_id: Some(config.composio().entity_id.clone()), - gmail_query: config.composio().gmail_sync_query.clone(), - }) - } else { - // The seam first, the config second — the mirror of the direct branch - // above. Inside a loaded module `session_token` cannot answer (the - // module holds a load-time snapshot with no bearer in it), so without - // the seam this branch refuses for every proxied user. Outside a module - // no host is installed, the seam answers `None`, and this falls through - // to exactly the config read it always did. - let bearer = crate::composio_host::session_bearer(config) - .or_else(|| config.session_token().ok().flatten()) - .ok_or_else(|| "OpenHuman backend bearer token is not configured".to_string())?; - Ok(ComposioSyncConfig { - mode: ComposioMode::Proxied, - base_url: config.effective_backend_api_url(), - api_key: None, - bearer_token: Some(SecretString::new(bearer)), - entity_id: Some(config.composio().entity_id.clone()), - gmail_query: config.composio().gmail_sync_query.clone(), - }) - } -} - -/// The toolkits with a native pipeline here. Kept identical to the engine -/// seam's list; `sync_status` advertising draws from the provider registry. -pub fn syncable_composio_toolkits() -> &'static [&'static str] { - &["clickup", "github", "gmail", "linear", "notion", "slack"] -} - -/// Whether `toolkit` has a native pipeline (case-insensitive). -pub fn is_composio_toolkit_syncable(toolkit: &str) -> bool { - let slug = toolkit.trim().to_ascii_lowercase(); - syncable_composio_toolkits().contains(&slug.as_str()) -} - -fn build_composio_pipeline( - toolkit: &str, - connection_id: &str, - composio: ComposioSyncConfig, -) -> Result, String> { - // Fail closed before resolving credentials for any toolkit without a - // native pipeline (#4957) — the gate stays a single testable list. - // - // Normalise once and match on the normalised slug: the gate accepts - // `" Gmail "` (trim + lowercase), so matching on the raw input would let a - // padded or mixed-case toolkit through the gate and into `unreachable!`. - let slug = toolkit.trim().to_ascii_lowercase(); - if !syncable_composio_toolkits().contains(&slug.as_str()) { - return Err(format!("memory sync does not support toolkit '{toolkit}'")); - } - // Pull the Gmail scope filter out before the client consumes the config. - let gmail_filter = composio - .gmail_query - .as_deref() - .map(str::trim) - .filter(|q| !q.is_empty()) - .map(str::to_string); - let client = ComposioClient::new(composio); - Ok(match slug.as_str() { - "gmail" => { - let mut pipeline = GmailSyncPipeline::new(client, connection_id); - if let Some(filter) = gmail_filter { - pipeline = pipeline.with_filter(filter); - } - Arc::new(pipeline) - } - "github" => Arc::new(GitHubSyncPipeline::new(client, connection_id)), - "notion" => Arc::new(NotionSyncPipeline::new(client, connection_id)), - "linear" => Arc::new(LinearSyncPipeline::new(client, connection_id)), - "clickup" => Arc::new(ClickUpSyncPipeline::new(client, connection_id)), - "slack" => Arc::new(SlackSyncPipeline::new(client, connection_id)), - _ => unreachable!("gated by is_composio_toolkit_syncable"), - }) -} - -/// Run one Composio connection through the engine-free pipelines. -pub async fn run_composio_connection( - toolkit: &str, - connection_id: &str, - config: &Config, - max_items: Option, - sync_depth_days: Option, -) -> Result { - run_composio_connection_with_caps( - toolkit, - connection_id, - config, - SourceCaps { - max_items, - sync_depth_days, - ..SourceCaps::default() - }, - ) - .await -} - -/// The per-source limits a run honours. All `None` = the source's defaults. -#[derive(Clone, Copy, Debug, Default)] -pub struct SourceCaps { - pub max_items: Option, - pub sync_depth_days: Option, - pub max_tokens_per_sync: Option, - pub max_cost_per_sync_usd: Option, -} - -impl SourceCaps { - /// The caps a registry entry carries. - pub fn from_source(source: &tinymemory_sources::MemorySourceEntry) -> Self { - Self { - max_items: source.max_items, - sync_depth_days: source.sync_depth_days, - max_tokens_per_sync: source.max_tokens_per_sync, - max_cost_per_sync_usd: source.max_cost_per_sync_usd, - } - } -} - -/// Run one Composio connection through the engine-free pipelines, honouring -/// every per-source cap. -pub async fn run_composio_connection_with_caps( - toolkit: &str, - connection_id: &str, - config: &Config, - caps: SourceCaps, -) -> Result { - let memory = crate::global::client_if_ready() - .ok_or_else(|| PipelineFailure::without_usage("memory client is not ready"))?; - let composio = composio_config(config).map_err(PipelineFailure::without_usage)?; - let pipeline = build_composio_pipeline(toolkit, connection_id, composio) - .map_err(PipelineFailure::without_usage)?; - let pipeline_config = PipelineConfig { - composio: None, // the client already holds the connection settings - sync_depth_days: caps.sync_depth_days, - max_items: caps.max_items, - max_tokens_per_sync: caps.max_tokens_per_sync, - max_cost_per_sync_usd: caps.max_cost_per_sync_usd, - }; - let host = Arc::new(PipelineHost::new(memory, config.to_arc())); - let mut outcome = run_pipeline( - pipeline, - toolkit, - connection_id, - &pipeline_config, - &host.context(), - ) - .await?; - outcome.tree_ingest_failures = host.tree_ingest_failures(); - Ok(outcome) -} - -/// Run a bounded Gmail backfill through the engine-free pipelines. -pub async fn run_gmail_backfill( - connection_id: &str, - query: &str, - max_pages: usize, - page_size: usize, - config: &Config, -) -> Result { - let memory = crate::global::client_if_ready() - .ok_or_else(|| PipelineFailure::without_usage("memory client is not ready"))?; - let composio = composio_config(config).map_err(PipelineFailure::without_usage)?; - let pipeline: Arc = Arc::new( - GmailSyncPipeline::new(ComposioClient::new(composio), connection_id) - .with_limits(max_pages, page_size) - .with_query(query), - ); - let host = Arc::new(PipelineHost::new(memory, config.to_arc())); - // The backfill drives the Gmail pipeline, which keys its `SyncState` on - // `"gmail"`; naming the same toolkit here puts it behind the same guard as - // a periodic or RPC Gmail sync of this connection. - let mut outcome = run_pipeline( - pipeline, - "gmail", - connection_id, - &PipelineConfig::default(), - &host.context(), - ) - .await?; - outcome.tree_ingest_failures = host.tree_ingest_failures(); - Ok(outcome) -} - -/// Run the Slack search backfill through the engine-free pipelines. -pub async fn run_slack_search_backfill( - connection_id: &str, - backfill_days: i64, - config: &Config, -) -> Result { - let memory = crate::global::client_if_ready() - .ok_or_else(|| PipelineFailure::without_usage("memory client is not ready"))?; - let composio = composio_config(config).map_err(PipelineFailure::without_usage)?; - let client = ComposioClient::new(composio); - let pipeline: Arc = Arc::new(SlackSearchBackfillPipeline::new( - client, - connection_id, - backfill_days, - )); - let host = Arc::new(PipelineHost::new(memory, config.to_arc())); - // `SlackSearchBackfillPipeline` loads and saves the same - // `("slack", connection_id)` state the Slack sync pipeline does, so the two - // must share one guard or they clobber each other's cursor and budget. - let mut outcome = run_pipeline( - pipeline, - "slack", - connection_id, - &PipelineConfig::default(), - &host.context(), - ) - .await?; - outcome.tree_ingest_failures = host.tree_ingest_failures(); - Ok(outcome) -} - -/// The note a run carries when another run already holds its connection. -/// -/// Callers that distinguish "nothing to sync" from "did not sync" match on -/// this rather than on a message they would have to keep in step by hand. -pub const SYNC_ALREADY_RUNNING: &str = "sync already running for this connection"; - -/// One guard per connection, so two runs cannot clobber each other's state. -type ConnectionLock = Arc>; - -/// The process-wide guard table. -/// -/// `run_incremental_sync` loads the connection's `SyncState` once, mutates it -/// in memory for the whole run, and saves at the end; the Slack search -/// backfill does the same over the same `("slack", connection_id)` record. Two -/// runs of one connection therefore race on the cursor, the dedup set and the -/// daily budget, and whichever saves last wins — losing either the dedup set -/// (re-fetch, re-spend) or the budget count (overspend past the cap). The -/// periodic loop, the sync RPC and a trigger can each fire the same -/// connection, so the race is reachable as the code stands. -/// -/// This is the single-process answer, which is how the loop and the RPC paths -/// actually run. An optimistic version stamp on the KV record is what a -/// multi-process host would need instead. -/// -/// The table only ever grows, bounded by the number of connections the host -/// has seen — the same shape, and the same bound, as the periodic scheduler's -/// last-fired map. An entry is one `Arc` and an unlocked mutex. -fn connection_locks() -> &'static Mutex> { - static LOCKS: OnceLock>> = OnceLock::new(); - LOCKS.get_or_init(|| Mutex::new(HashMap::new())) -} - -/// The guard key for a connection. -/// -/// Normalised exactly as [`build_composio_pipeline`] normalises the toolkit -/// gate, so `" Gmail "` and `gmail` name one connection rather than two — and -/// so the Slack sync pipeline and the Slack search backfill, which share one -/// `SyncState` record, share one guard. -fn connection_key(toolkit: &str, connection_id: &str) -> (String, String) { - ( - toolkit.trim().to_ascii_lowercase(), - connection_id.trim().to_owned(), - ) -} - -/// Take the guard for one connection, or `None` if a run already holds it. -/// -/// Deliberately non-blocking. Queueing behind the running sync would stall the -/// periodic loop's whole tick — it walks connections sequentially — and then -/// run a second sync of a connection that has just been synced, which is the -/// Composio spend this guard exists to avoid. -fn try_hold_connection(toolkit: &str, connection_id: &str) -> Option> { - let lock = { - // A panic inside a run cannot corrupt the table: it holds `Arc`s, and - // the async guard is released by its own `Drop`. Recovering from the - // poison keeps one panicking sync from disabling every later one. - let mut locks = connection_locks() - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - Arc::clone( - locks - .entry(connection_key(toolkit, connection_id)) - .or_default(), - ) - }; - lock.try_lock_owned().ok() -} - -async fn run_pipeline( - pipeline: Arc, - toolkit: &str, - connection_id: &str, - config: &PipelineConfig, - context: &SyncContext, -) -> Result { - let Some(_connection) = try_hold_connection(toolkit, connection_id) else { - tracing::debug!( - toolkit, - connection_id, - "[memory_sync] a sync of this connection is already running; skipping" - ); - return Ok(SyncOutcome { - note: Some(SYNC_ALREADY_RUNNING.to_owned()), - ..SyncOutcome::default() - }); - }; - let pipeline_id = pipeline.id().to_owned(); - let mut dispatcher = SyncDispatcher::new(); - dispatcher - .register(pipeline) - .map_err(|error| PipelineFailure::without_usage(error.to_string()))?; - dispatcher - .tick(&pipeline_id, config, context) - .await - .map_err(|error| { - let usage = error.downcast_ref::(); - PipelineFailure { - message: error.to_string(), - actions_called: usage.map_or(0, |error| error.actions_called), - provider_cost_usd: usage.map_or(0.0, |error| error.provider_cost_usd), - } - }) -} - -#[cfg(test)] -#[path = "host_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/host_tests.rs b/crates/tinymemory-core/src/sync/pipelines/host_tests.rs deleted file mode 100644 index a84d0408..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/host_tests.rs +++ /dev/null @@ -1,496 +0,0 @@ -//! Tests for the surrounding module. - -use super::*; - -/// #4957: an unsupported toolkit is rejected *before* credentials are -/// resolved — moved here with the gate itself from the engine seam. -#[test] -fn unsupported_toolkit_is_rejected_before_resolving_credentials() { - let err = build_composio_pipeline("googlecalendar", "conn-1", ComposioSyncConfig::default()) - .err() - .expect("unsupported toolkit must be rejected"); - assert!( - err.contains("does not support toolkit 'googlecalendar'"), - "got: {err}" - ); -} - -#[test] -fn the_syncable_set_is_exactly_the_native_pipelines() { - for toolkit in syncable_composio_toolkits() { - assert!( - build_composio_pipeline(toolkit, "conn-1", ComposioSyncConfig::default()).is_ok(), - "advertised toolkit '{toolkit}' must build" - ); - } - assert!(!is_composio_toolkit_syncable("googlecalendar")); - assert!(is_composio_toolkit_syncable(" Gmail ")); -} - -/// The gate normalises; the build must match on the same normalised -/// slug, or a padded/mixed-case toolkit passes the gate and panics. -#[test] -fn a_padded_or_mixed_case_toolkit_builds_rather_than_panicking() { - for toolkit in [" Gmail ", "GMAIL", "gmail\t", " Slack"] { - assert!( - build_composio_pipeline(toolkit, "conn-1", ComposioSyncConfig::default()).is_ok(), - "{toolkit:?} passes the gate and must build" - ); - } -} - -/// The guard table is process-global and shared by every test in this -/// binary, so each test names connections nothing else touches. -#[test] -fn one_connection_admits_one_run_at_a_time() { - let held = try_hold_connection("gmail", "guard-single").expect("the first run takes the guard"); - assert!( - try_hold_connection("gmail", "guard-single").is_none(), - "a second run of the same connection must be refused, not queued" - ); - drop(held); - assert!( - try_hold_connection("gmail", "guard-single").is_some(), - "the guard must be released when the run ends" - ); -} - -/// The guard is per connection, not per toolkit: one slow Gmail sync must -/// not stop every other Gmail connection from syncing. -#[test] -fn different_connections_hold_independent_guards() { - let first = try_hold_connection("gmail", "guard-independent-a") - .expect("the first connection takes its guard"); - let second = try_hold_connection("gmail", "guard-independent-b") - .expect("a different connection has its own guard"); - drop((first, second)); -} - -/// `build_composio_pipeline` accepts `" Gmail "` by normalising it. The -/// guard key must normalise identically, or a padded toolkit syncs the -/// same connection concurrently with an unpadded one and they clobber each -/// other's state — the defect the guard exists to prevent. -#[test] -fn the_guard_key_normalises_the_toolkit_like_the_gate() { - assert_eq!( - connection_key(" Gmail ", " conn-1 "), - connection_key("gmail", "conn-1") - ); - let held = - try_hold_connection("gmail", "guard-normalised").expect("the first run takes the guard"); - assert!( - try_hold_connection(" GMAIL\t", "guard-normalised").is_none(), - "a padded, mixed-case toolkit names the same connection" - ); - drop(held); -} - -/// The Slack sync pipeline and the Slack search backfill load and save the -/// same `("slack", connection_id)` state, so they must contend. -#[test] -fn the_slack_backfill_shares_the_slack_sync_guard() { - assert_eq!( - connection_key("slack", "guard-slack"), - connection_key("Slack", "guard-slack") - ); - let held = try_hold_connection("slack", "guard-slack").expect("the sync takes the guard"); - assert!( - try_hold_connection("slack", "guard-slack").is_none(), - "the backfill must not run while a Slack sync of this connection is running" - ); - drop(held); -} - -/// The engine adapter's tree reconnect has this test -/// (`engine::sync`'s `composio_sync_document_reaches_memory_tree`); the -/// engine-free host that replaced it on the live path did not, and drifted -/// — it wrote a `composio:`-prefixed source id and no `path_scope`, so -/// every synced item became its own tree under a scope no platform prefix -/// matches. Chunks existed, recall could not reach them. Asserting the -/// addressing, not merely the row count, is what catches that. -#[tokio::test] -async fn a_synced_document_is_keyed_by_its_connection_scope() { - use tinymemory_api::host::test_support::TestHostConfig; - use tinymemory_api::host::MemoryHostConfig; - - crate::test_seams::init(); - let workspace = tempfile::tempdir().expect("workspace"); - let workspace_dir = workspace.path().join("workspace"); - let mut host_config = TestHostConfig::default(); - host_config.workspace_dir = workspace_dir.clone(); - let config = host_config.to_arc(); - let client: MemoryClientRef = Arc::new( - crate::store::MemoryClient::from_workspace_dir(workspace_dir) - .expect("memory client initialises against a fresh workspace"), - ); - let host = PipelineHost::new(client, config.clone()); - - // A fresh tree is empty, so a non-zero count after the store is - // attributable to this sync rather than to pre-existing state. - assert_eq!( - crate::store::chunks::store::count_chunks(&*config).expect("count chunks"), - 0, - "fresh workspace must start with an empty memory tree" - ); - - host.store(SkillDocument { - namespace_skill_id: "gmail".into(), - connection_id: "conn-1".into(), - document_id: "gmail:msg-1".into(), - title: "Quarterly planning".into(), - content: "Let's finalise the Q3 roadmap and align on the launch date.".into(), - toolkit: "gmail".into(), - metadata: serde_json::json!({ "source": "composio-provider-incremental" }), - }) - .await - .expect("storing a synced document must also ingest it into the memory tree"); - - let scoped = crate::store::chunks::store::list_chunks( - &*config, - &crate::store::chunks::store::ListChunksQuery { - source_id: Some("gmail:conn-1:gmail:msg-1".into()), - limit: Some(8), - ..Default::default() - }, - ) - .expect("list chunks by source id"); - assert!( - !scoped.is_empty(), - "ingested chunks must be keyed by `{{toolkit}}:{{connection_id}}:{{document_id}}` — \ - the scheme the memory-source status and diff snapshots query by" - ); - assert!( - scoped - .iter() - .all(|chunk| chunk.metadata.path_scope.as_deref() == Some("gmail:conn-1")), - "connector chunks must carry the `{{toolkit}}:{{connection_id}}` tree scope so \ - query_source resolves them (gmail → email)" - ); - assert!( - scoped - .iter() - .all(|chunk| chunk.metadata.owner == "gmail-sync:conn-1"), - "connector chunks must be owned by the connection that synced them" - ); -} - -/// A blank toolkit or connection cannot produce a scope any retrieval kind -/// matches, so the tree half is skipped rather than writing an unreachable -/// tree. The skill store, which committed first, still holds the item. -#[tokio::test] -async fn an_item_without_a_connection_scope_skips_the_tree_but_not_the_store() { - use tinymemory_api::host::test_support::TestHostConfig; - use tinymemory_api::host::MemoryHostConfig; - - crate::test_seams::init(); - let workspace = tempfile::tempdir().expect("workspace"); - let workspace_dir = workspace.path().join("workspace"); - let mut host_config = TestHostConfig::default(); - host_config.workspace_dir = workspace_dir.clone(); - let config = host_config.to_arc(); - let client: MemoryClientRef = Arc::new( - crate::store::MemoryClient::from_workspace_dir(workspace_dir) - .expect("memory client initialises against a fresh workspace"), - ); - let host = PipelineHost::new(client.clone(), config.clone()); - - host.store(SkillDocument { - namespace_skill_id: "gmail".into(), - connection_id: " ".into(), - document_id: "gmail:msg-2".into(), - title: "No connection".into(), - content: "This item has no connection scope.".into(), - toolkit: "gmail".into(), - metadata: serde_json::Value::Null, - }) - .await - .expect("a scopeless item must not fail the sync"); - - assert_eq!( - crate::store::chunks::store::count_chunks(&*config).expect("count chunks"), - 0, - "a scopeless item must not write a tree no retrieval can reach" - ); - let stored = client - .list_documents(Some("skill-gmail")) - .await - .expect("list skill documents"); - let documents = stored - .get("documents") - .and_then(|value| value.as_array()) - .cloned() - .unwrap_or_default(); - assert_eq!( - documents.len(), - 1, - "the skill store is the source of truth and must still hold the item" - ); -} - -/// A pipeline that records whether it was ticked, so the refusal path can -/// be shown to skip the run rather than to run and discard the result. -struct RecordingPipeline(Arc); - -#[async_trait] -impl SyncPipeline for RecordingPipeline { - fn id(&self) -> &str { - "test:recording" - } - - fn kind(&self) -> crate::sync::pipelines::traits::SyncPipelineKind { - crate::sync::pipelines::traits::SyncPipelineKind::Composio - } - - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - - async fn tick(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result { - self.0.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(SyncOutcome { - records_ingested: 7, - ..SyncOutcome::default() - }) - } -} - -/// End to end: with the connection held, `run_pipeline` returns the note -/// without ticking the pipeline — no fetch, no Composio spend, and no -/// second writer of the connection's `SyncState`. -#[tokio::test] -async fn a_held_connection_short_circuits_the_run() { - crate::test_seams::init(); - let workspace = tempfile::tempdir().expect("workspace"); - let client: MemoryClientRef = Arc::new( - crate::store::MemoryClient::from_workspace_dir(workspace.path().join("store")) - .expect("memory client initialises against a fresh workspace"), - ); - let host = Arc::new(PipelineHost::without_tree_ingest(client)); - let ticked = Arc::new(std::sync::atomic::AtomicBool::new(false)); - - let held = - try_hold_connection("gmail", "guard-short-circuit").expect("the first run takes the guard"); - let outcome = run_pipeline( - Arc::new(RecordingPipeline(ticked.clone())), - "gmail", - "guard-short-circuit", - &PipelineConfig::default(), - &host.context(), - ) - .await - .expect("a refused run is not a failure"); - - assert_eq!(outcome.note.as_deref(), Some(SYNC_ALREADY_RUNNING)); - assert_eq!(outcome.records_ingested, 0); - assert!( - !ticked.load(std::sync::atomic::Ordering::SeqCst), - "the refused run must not tick the pipeline" - ); - - // Released, the same call runs normally — the guard skips a concurrent - // run, it does not disable the connection. - drop(held); - let outcome = run_pipeline( - Arc::new(RecordingPipeline(ticked.clone())), - "gmail", - "guard-short-circuit", - &PipelineConfig::default(), - &host.context(), - ) - .await - .expect("the run succeeds once the guard is free"); - assert_eq!(outcome.records_ingested, 7); - assert!(ticked.load(std::sync::atomic::Ordering::SeqCst)); -} - -#[test] -fn pipeline_failure_and_source_caps_preserve_operational_details() { - let failure = PipelineFailure::without_usage("offline"); - assert_eq!(failure.to_string(), "offline"); - assert!(std::error::Error::source(&failure).is_none()); - assert_eq!(failure.actions_called, 0); - assert_eq!(failure.provider_cost_usd, 0.0); - - let source: tinymemory_sources::MemorySourceEntry = serde_json::from_value(serde_json::json!({ - "id": "source-1", - "kind": "composio", - "label": "Mail", - "enabled": true, - "max_items": 11, - "sync_depth_days": 4, - "max_tokens_per_sync": 500, - "max_cost_per_sync_usd": 0.25 - })) - .unwrap(); - let caps = SourceCaps::from_source(&source); - assert_eq!(caps.max_items, Some(11)); - assert_eq!(caps.sync_depth_days, Some(4)); - assert_eq!(caps.max_tokens_per_sync, Some(500)); - assert_eq!(caps.max_cost_per_sync_usd, Some(0.25)); -} - -#[test] -fn composio_config_covers_direct_proxied_and_missing_credentials() { - let mut direct = tinymemory_api::host::test_support::TestHostConfig::default(); - direct.composio.mode = "direct".into(); - direct.composio.entity_id = "entity-1".into(); - assert!(composio_config(&direct).is_err()); - direct.composio.api_key = Some("direct-secret".into()); - let config = composio_config(&direct).unwrap(); - assert_eq!(config.mode, ComposioMode::Direct); - assert_eq!(config.api_key.as_ref().unwrap().expose(), "direct-secret"); - assert_eq!(config.entity_id.as_deref(), Some("entity-1")); - - let mut proxied = tinymemory_api::host::test_support::TestHostConfig::default(); - assert!(composio_config(&proxied).is_err()); - proxied.session_token = Some("session-secret".into()); - proxied.api_url = Some("https://backend.example".into()); - let config = composio_config(&proxied).unwrap(); - assert_eq!(config.mode, ComposioMode::Proxied); - assert_eq!( - config.bearer_token.as_ref().unwrap().expose(), - "session-secret" - ); - assert!(config.api_key.is_none()); -} - -struct FailingPipeline { - usage: bool, -} - -#[async_trait] -impl SyncPipeline for FailingPipeline { - fn id(&self) -> &str { - if self.usage { - "test:usage" - } else { - "test:plain" - } - } - - fn kind(&self) -> crate::sync::pipelines::traits::SyncPipelineKind { - crate::sync::pipelines::traits::SyncPipelineKind::Composio - } - - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - - async fn tick(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result { - if self.usage { - Err(anyhow::Error::new(SyncRunError::new( - "spent failure", - 3, - 0.75, - ))) - } else { - anyhow::bail!("plain failure") - } - } -} - -#[tokio::test] -async fn run_pipeline_preserves_typed_usage_and_defaults_plain_errors() { - crate::test_seams::init(); - let workspace = tempfile::tempdir().unwrap(); - let memory: MemoryClientRef = Arc::new( - crate::store::MemoryClient::from_workspace_dir(workspace.path().join("store")).unwrap(), - ); - let host = Arc::new(PipelineHost::without_tree_ingest(memory)); - let typed = run_pipeline( - Arc::new(FailingPipeline { usage: true }), - "gmail", - "failure-usage", - &PipelineConfig::default(), - &host.context(), - ) - .await - .unwrap_err(); - assert_eq!(typed.message, "spent failure"); - assert_eq!(typed.actions_called, 3); - assert_eq!(typed.provider_cost_usd, 0.75); - - let plain = run_pipeline( - Arc::new(FailingPipeline { usage: false }), - "gmail", - "failure-plain", - &PipelineConfig::default(), - &host.context(), - ) - .await - .unwrap_err(); - assert_eq!(plain.message, "plain failure"); - assert_eq!(plain.actions_called, 0); - assert_eq!(plain.provider_cost_usd, 0.0); -} - -#[tokio::test] -async fn pipeline_host_state_event_and_delete_capabilities_round_trip() { - crate::test_seams::init(); - let workspace = tempfile::tempdir().unwrap(); - let memory: MemoryClientRef = Arc::new( - crate::store::MemoryClient::from_workspace_dir(workspace.path().join("store")).unwrap(), - ); - let host = Arc::new(PipelineHost::without_tree_ingest(memory.clone())); - SyncStateStore::set(&*host, "sync", "cursor", &serde_json::json!({"page": 2})) - .await - .unwrap(); - assert_eq!( - SyncStateStore::get(&*host, "sync", "cursor").await.unwrap(), - Some(serde_json::json!({"page": 2})) - ); - let sink = crate::events::RecordingSink::install(); - host.emit(SyncEvent { - source_id: "source-1".into(), - toolkit: "gmail".into(), - connection_id: Some("connection-1".into()), - stage: crate::sync::pipelines::traits::SyncStage::Completed, - message: Some("done".into()), - }) - .await - .unwrap(); - assert!(!sink.drain().is_empty()); - host.delete("gmail", "missing-document").await.unwrap(); -} - -/// An ordinary (non-corrupt) tree-ingest failure is tolerated — `store` -/// returns `Ok` and the skill store keeps the item — but it is COUNTED, so the -/// run's verdict can say "fetched, not tree-ingested" instead of reading as -/// full success (openhuman#5820). The broken-workspace lever is the same one -/// the engine adapter's tolerance test uses: the tree config's workspace sits -/// under a regular file, so the tree store cannot be created. -#[tokio::test] -async fn a_tolerated_tree_ingest_failure_is_counted() { - use tinymemory_api::host::test_support::TestHostConfig; - use tinymemory_api::host::MemoryHostConfig; - - crate::test_seams::init(); - let workspace = tempfile::tempdir().expect("workspace"); - let client: MemoryClientRef = Arc::new( - crate::store::MemoryClient::from_workspace_dir(workspace.path().join("skill-store")) - .expect("memory client initialises against a fresh workspace"), - ); - let blocker = workspace.path().join("blocker"); - std::fs::write(&blocker, b"not a directory").expect("write blocker file"); - let mut host_config = TestHostConfig::default(); - host_config.workspace_dir = blocker.join("workspace"); - let host = Arc::new(PipelineHost::new(client, host_config.to_arc())); - - assert_eq!(host.tree_ingest_failures(), 0); - host.store(SkillDocument { - namespace_skill_id: "gmail".into(), - connection_id: "conn-1".into(), - document_id: "gmail:msg-1".into(), - title: "Quarterly planning".into(), - content: "Let's finalise the Q3 roadmap.".into(), - toolkit: "gmail".into(), - metadata: serde_json::json!({ "source": "composio-provider-incremental" }), - }) - .await - .expect("a non-corrupt tree-ingest failure must stay tolerated"); - assert_eq!( - host.tree_ingest_failures(), - 1, - "the tolerated failure must be counted for the run's verdict" - ); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/mod.rs b/crates/tinymemory-core/src/sync/pipelines/mod.rs deleted file mode 100644 index dcd526f6..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/mod.rs +++ /dev/null @@ -1,17 +0,0 @@ -//! Engine-neutral sync pipelines (#18 §B1). -//! -//! The Composio orchestration that used to run inside the engine: fetch pages -//! within budget, normalise through `tinymemory-sync`, and write through the -//! [`traits::SyncContext`] sinks. A pipeline sees three capabilities — events, -//! documents, state — and whatever provider the host bound serves them, which -//! is the property §B5's acceptance test needs. -//! -//! The engine keeps its own copies for its internal pipelines (workspace -//! watcher, tree rebuild, repo summarisation — engine-tree features by -//! design). Sources of kind `Composio` route here; tree-coupled source kinds -//! still route through the engine seam. - -pub mod composio; -pub mod dispatcher; -pub mod host; -pub mod traits; diff --git a/crates/tinymemory-core/src/sync/pipelines/traits.rs b/crates/tinymemory-core/src/sync/pipelines/traits.rs deleted file mode 100644 index 311909e2..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/traits.rs +++ /dev/null @@ -1,207 +0,0 @@ -//! Host seams and pipeline contracts for live synchronization. - -use std::sync::Arc; - -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SyncPipelineKind { - Composio, - Workspace, - Mcp, -} - -impl SyncPipelineKind { - pub fn as_str(self) -> &'static str { - match self { - Self::Composio => "composio", - Self::Workspace => "workspace", - Self::Mcp => "mcp", - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SyncStage { - Requested, - Fetching, - Stored, - Ingesting, - Completed, - Failed, -} - -/// Stable wire name for each stage, shared by every event adapter. -pub fn stage_name(stage: SyncStage) -> &'static str { - match stage { - SyncStage::Requested => "requested", - SyncStage::Fetching => "fetching", - SyncStage::Stored => "stored", - SyncStage::Ingesting => "ingesting", - SyncStage::Completed => "completed", - SyncStage::Failed => "failed", - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SyncEvent { - pub source_id: String, - pub toolkit: String, - pub connection_id: Option, - pub stage: SyncStage, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub message: Option, -} - -#[async_trait] -pub trait SyncEventSink: Send + Sync { - async fn emit(&self, event: SyncEvent) -> anyhow::Result<()>; -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SkillDocument { - pub namespace_skill_id: String, - pub connection_id: String, - pub document_id: String, - pub title: String, - pub content: String, - pub toolkit: String, - #[serde(default)] - pub metadata: serde_json::Value, -} - -#[async_trait] -pub trait SkillDocSink: Send + Sync { - async fn store(&self, document: SkillDocument) -> anyhow::Result<()>; - async fn delete(&self, namespace_skill_id: &str, document_id: &str) -> anyhow::Result<()>; -} - -/// How the Composio client reaches the API: straight at it, or through the -/// backend proxy. The engine's enum, ported with the client — distinct from -/// `tinymemory_api::host::ComposioMode`, which is the *host seam's* -/// string-typed setting; the seam converts. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum ComposioMode { - /// Call api.composio.dev with the host's own key. - Direct, - /// Route through the backend proxy. - #[default] - Proxied, -} - -/// The Composio client's connection settings, owned here so the pipelines -/// take no engine config type (#18 §B1). The engine keeps its own copy for -/// its internal pipelines; the host constructs this one from its own config. -#[derive(Clone, Debug, Default)] -pub struct ComposioSyncConfig { - pub mode: ComposioMode, - pub base_url: String, - pub api_key: Option, - pub bearer_token: Option, - pub entity_id: Option, - /// Optional Gmail search query the Gmail pipeline ANDs onto every page - /// fetch (e.g. `label:brain`) so background sync only ingests matching - /// messages. `None` = whole inbox window. - pub gmail_query: Option, -} - -/// A string whose `Debug` never prints the value. -#[derive(Clone, Default, PartialEq, Eq)] -pub struct SecretString(String); - -impl SecretString { - pub fn new(value: impl Into) -> Self { - Self(value.into()) - } - - pub fn expose(&self) -> &str { - &self.0 - } - - pub fn is_empty(&self) -> bool { - self.0.trim().is_empty() - } -} - -impl std::fmt::Debug for SecretString { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("SecretString(redacted)") - } -} - -/// What a pipeline may read of the host's configuration: the Composio -/// connection settings and the sync-depth budget. Deliberately not the -/// host's whole config — a pipeline that needs more must argue for the -/// field here. -#[derive(Clone, Debug, Default)] -pub struct PipelineConfig { - pub composio: Option, - pub sync_depth_days: Option, - pub max_items: Option, - /// Stop the run once this many tokens (estimated from stored content) - /// have been ingested. `None` = unbounded. - pub max_tokens_per_sync: Option, - /// Stop the run once the provider has charged this much. `None` = - /// unbounded. - pub max_cost_per_sync_usd: Option, -} - -/// Host capabilities required by sync pipelines. -#[derive(Clone)] -pub struct SyncContext { - pub events: Arc, - pub documents: Arc, - pub state: Arc, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct SyncOutcome { - pub records_ingested: u32, - pub more_pending: bool, - #[serde(default)] - pub actions_called: u32, - #[serde(default)] - pub provider_cost_usd: f64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub note: Option, - /// Items whose skill-store write committed but whose memory-tree ingest - /// failed (non-corrupt failures — corruption aborts the run instead). - /// `records_ingested` counts those items as fetched-and-stored, so a - /// non-zero value here is the "fetch succeeded, tree did not" signal the - /// sync verdict must not report as full success (openhuman#5820). - #[serde(default)] - pub tree_ingest_failures: u32, -} - -#[derive(Debug, thiserror::Error)] -#[error("{message}")] -pub struct SyncRunError { - pub actions_called: u32, - pub provider_cost_usd: f64, - message: String, -} - -impl SyncRunError { - pub fn new(message: impl Into, actions_called: u32, provider_cost_usd: f64) -> Self { - Self { - actions_called, - provider_cost_usd, - message: message.into(), - } - } -} - -#[async_trait] -pub trait SyncPipeline: Send + Sync { - fn id(&self) -> &str; - fn kind(&self) -> SyncPipelineKind; - async fn init(&self, config: &PipelineConfig, context: &SyncContext) -> anyhow::Result<()>; - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result; -} diff --git a/crates/tinymemory-core/src/sync/usage.rs b/crates/tinymemory-core/src/sync/usage.rs new file mode 100644 index 00000000..21b31adf --- /dev/null +++ b/crates/tinymemory-core/src/sync/usage.rs @@ -0,0 +1,23 @@ +//! What one sync run cost. + +use serde::{Deserialize, Serialize}; + +/// Per-run accumulator for a source's billable provider calls. +/// +/// # Why it outlived the Composio tree +/// +/// It is what the sync audit log records, and the audit log is this crate's. +/// A run against a connected account has a price attached — the provider +/// charges per action — and an operator asking "why did this month cost that" +/// is asking a question about stored rows, not about whoever fetched them. +/// +/// Zero for the sources that cost nothing to read, which is most of them. +/// That is not a gap: a folder scan really did call nothing and spend nothing, +/// and the field says so rather than being absent. +#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] +pub struct ProviderUsage { + /// Calls that returned a response this run. + pub actions_called: u32, + /// Sum of each response's provider-reported cost. + pub cost_usd: f64, +} diff --git a/crates/tinymemory-core/src/sync/workspace/cadence.rs b/crates/tinymemory-core/src/sync/workspace/cadence.rs new file mode 100644 index 00000000..af9b558f --- /dev/null +++ b/crates/tinymemory-core/src/sync/workspace/cadence.rs @@ -0,0 +1,81 @@ +//! When the periodic scheduler should fire, and when it should hold off. +//! +//! # Why this is not with the connector +//! +//! None of it is specific to any source. "How often may this sync run" and +//! "should the scheduler run at all right now" are questions about the user's +//! cadence setting and the machine's pause policy, and the answers are the +//! same whether the source is a mailbox, a folder, or an RSS feed. +//! +//! It lived under the Composio tree because Composio was the first source +//! with a periodic loop. The loop is still here; only the fetching left. + +use std::time::Duration; + +use crate::scheduler_gate::{current_policy, PauseReason}; +use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; + +/// Resolve the effective periodic sync interval (seconds) for one connection, +/// combining the provider's own default with the user's global +/// memory-sync cadence ([`Config::memory_sync_interval_secs`], #3302). +/// +/// - `global == Some(0)` → `None`: "Manual only" — the scheduler skips this +/// source entirely (manual sync still works). +/// - `global == Some(n)` → `Some(max(n, provider_default))`: the user's +/// cadence overrides the provider default but is floored at it, so we never +/// sync *more* often than the provider intended. +/// - `global == None` → `Some(max(DEFAULT, provider_default))`: no explicit +/// user choice, so fall back to the 24h default cadence (also floored at the +/// provider default). +pub(crate) fn effective_interval_secs(provider_default: u64, global: Option) -> Option { + match global { + Some(0) => None, + Some(n) => Some(n.max(provider_default)), + None => Some(DEFAULT_MEMORY_SYNC_INTERVAL_SECS.max(provider_default)), + } +} + +/// Decide whether a connection is due for a periodic sync right now, given the +/// effective interval and how long ago it last synced this run. +/// +/// `since_last_sync == None` means we have no record of a sync this process +/// lifetime, so we fire immediately (the restart-recovery path). Kept pure so +/// the due-check can be simulated without driving the real `Instant` clock. +pub(crate) fn connection_is_due(interval_secs: u64, since_last_sync: Option) -> bool { + match since_last_sync { + Some(elapsed) => elapsed >= Duration::from_secs(interval_secs), + None => true, + } +} + +/// Inspect the scheduler-gate policy and decide whether this tick should +/// fire at all. Returns `Some(reason)` for paused states so the caller can +/// log a single, attributable line instead of doing the work and discovering +/// per-LLM-call later that everything's gated. +/// +/// Covers two reasons the memory subsystem treats as "do no background +/// work": +/// - [`PauseReason::UserDisabled`] — user flipped the Memory Tree toggle off +/// in Settings (#1856 Part 1). The 20-min Composio fetch loop honouring +/// this flag is the explicit follow-up listed in the #2719 PR body. +/// - [`PauseReason::SignedOut`] — no live session; periodic work would just +/// 401-loop against the backend. +/// +/// Other [`PauseReason`] variants: +/// - `OnBattery` / `CpuPressure` (future, per #1073) — intentionally **not** +/// gated here; periodic Composio fetch is network-light, so battery / CPU +/// pressure shouldn't stop the user's data flowing in. Those signals +/// already throttle LLM-bound work through the regular gate. +/// - `Unknown` — documented in `scheduler_gate::policy` as a safe fallback; +/// `Policy::pause_reason()` returns it only when the gate state is in a +/// transitional / not-yet-resolved condition. Letting the tick proceed +/// here keeps periodic sync running through brief transitions instead of +/// pausing on stale unresolved state. +pub(crate) fn periodic_pause_reason() -> Option { + // Delegate the `Policy::Paused { .. }` → `PauseReason` extraction to + // the existing `Policy::pause_reason()` helper (avoids re-implementing + // the same destructure twice). The allow-list below is the only thing + // this site has to own — future `PauseReason` variants stay opt-in. + let reason = current_policy().pause_reason()?; + matches!(reason, PauseReason::UserDisabled | PauseReason::SignedOut).then_some(reason) +} diff --git a/crates/tinymemory-core/src/sync/workspace/mod.rs b/crates/tinymemory-core/src/sync/workspace/mod.rs index a21d477d..d7e868d1 100644 --- a/crates/tinymemory-core/src/sync/workspace/mod.rs +++ b/crates/tinymemory-core/src/sync/workspace/mod.rs @@ -20,6 +20,7 @@ //! workspace-kind memory sources (GitHub repos, folders, RSS, web pages) //! syncing without manual "Sync now" clicks. +pub mod cadence; pub mod periodic; pub use periodic::start_workspace_periodic_sync; diff --git a/crates/tinymemory-core/src/sync/workspace/periodic.rs b/crates/tinymemory-core/src/sync/workspace/periodic.rs index 7f2de2fa..f6d41d18 100644 --- a/crates/tinymemory-core/src/sync/workspace/periodic.rs +++ b/crates/tinymemory-core/src/sync/workspace/periodic.rs @@ -37,7 +37,7 @@ use crate::scheduler_gate::resume_notify; use crate::sources::sync::sync_source; use crate::sources::types::{MemorySourceEntry, SourceKind}; use crate::sync::audit::{read_audit_log, SyncAuditEntry}; -use crate::sync::composio::periodic::{ +use crate::sync::workspace::cadence::{ connection_is_due, effective_interval_secs, periodic_pause_reason, }; use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; diff --git a/crates/tinymemory-core/src/test_seams.rs b/crates/tinymemory-core/src/test_seams.rs index 17ca16b9..24a27007 100644 --- a/crates/tinymemory-core/src/test_seams.rs +++ b/crates/tinymemory-core/src/test_seams.rs @@ -51,7 +51,6 @@ pub(crate) fn init() { crate::embedding_host::TestEmbeddingHost::install(); crate::config_loader::set_config_loader(Arc::new(TestConfigLoader)); crate::chat_host::set_chat_host(Arc::new(TestChatHost)); - crate::composio_host::set_composio_host(Arc::new(TestComposioHost)); }); } @@ -93,38 +92,6 @@ impl crate::chat_host::ChatHost for TestChatHost { /// /// Every method reports the no-backend-session state, which is the branch the /// core's own tests exercise; a stub that succeeded would need to fake Composio -/// itself. -#[derive(Debug)] -struct TestComposioHost; - -#[async_trait] -impl crate::composio_host::ComposioHost for TestComposioHost { - async fn list_connections( - &self, - _config: &Config, - ) -> Result, String> { - Err(NO_SESSION.to_string()) - } - - async fn execute( - &self, - _config: &Config, - _tool: &str, - _arguments: Option, - _entity_id: &str, - _connection_id: Option<&str>, - ) -> Result { - Err(NO_SESSION.to_string()) - } - - fn api_key(&self, _config: &Config) -> Option { - None - } - - fn is_available(&self, _config: &Config) -> bool { - false - } -} /// The message [`TestComposioHost`] reports. Matches the shape the real backend /// client produces when no session token is stored, which is what the tests diff --git a/crates/tinymemory-core/tests/composio_gmail_non_tinycortex_e2e.rs b/crates/tinymemory-core/tests/composio_gmail_non_tinycortex_e2e.rs deleted file mode 100644 index 063efe0b..00000000 --- a/crates/tinymemory-core/tests/composio_gmail_non_tinycortex_e2e.rs +++ /dev/null @@ -1,239 +0,0 @@ -//! Issue #18 §B5 / §E4 — the acceptance test for the sync section: -//! **Composio Gmail sync completes end to end against a driver that is not -//! TinyCortex.** -//! -//! The pieces under test, and what each proves: -//! -//! - A mock Composio (wiremock, loopback-only) serves two pages of -//! `GMAIL_FETCH_EMAILS` — pagination, cursor advance and dedup are real. -//! - The pipeline is `sync::pipelines::composio::GmailSyncPipeline`, run -//! through the real `SyncDispatcher` — the exact production path. -//! - The host is `PipelineHost::without_tree_ingest` over a `MemoryClient` -//! bound to the **namespace store** — the driver #42 (§A3) registers as its -//! own non-TinyCortex class. No engine is initialised, no tree exists, and -//! the pipeline code under `core/src/sync/` names no engine module. The -//! engine's `KvStore` appears only as a storage *library* inside the -//! namespace store's SQLite file — it is not the bound driver, and nothing -//! in the pipeline knows it is there. -//! -//! Offline by construction: the only socket is wiremock's 127.0.0.1 listener. - -use std::sync::Arc; - -use serde_json::json; -use wiremock::matchers::{body_partial_json, method, path}; -use wiremock::{Match, Mock, MockServer, Request, ResponseTemplate}; - -/// Matches the *first* fetch only: an execute body whose arguments carry no -/// `page_token`. `body_partial_json` cannot express absence, and without this -/// the page-1 mount also matches the page-2 request (which still contains -/// `max_results`), serving page 1 twice — dedup then eats the repeats and the -/// test fails honestly but confusingly. -struct NoPageToken; - -impl Match for NoPageToken { - fn matches(&self, request: &Request) -> bool { - serde_json::from_slice::(&request.body) - .map(|body| body["arguments"].get("page_token").is_none()) - .unwrap_or(false) - } -} - -use tinymemory_core::store::MemoryClient; - -/// The one piece of host wiring `MemoryClient` requires: an embedding host. -/// Noop — this test is about the sync path, and recall is not asserted. -#[derive(Debug)] -struct NoopEmbeddingHost; - -impl tinymemory_api::host::EmbeddingHost for NoopEmbeddingHost { - fn resolve_api_key(&self, _provider: &str) -> Option { - None - } - - fn ollama_base_url(&self) -> String { - "http://127.0.0.1:1".into() - } - - fn default_embedding_provider( - &self, - ) -> std::sync::Arc { - std::sync::Arc::new(tinymemory_api::host::NoopEmbedding) - } - - fn create_embedding_provider_with_credentials( - &self, - _provider: &str, - _model: &str, - _dims: usize, - _api_key: &str, - _custom_endpoint: Option<&str>, - ) -> Result, String> { - Ok(Box::new(tinymemory_api::host::NoopEmbedding)) - } - - fn model_supports_dimensions(&self, _model: &str) -> bool { - false - } - - fn cloud_embedding_provider( - &self, - _model: &str, - _dims: usize, - ) -> Result, String> { - Ok(Box::new(tinymemory_api::host::NoopEmbedding)) - } - - fn default_cloud_embedding_model(&self) -> &str { - "noop" - } - - fn default_cloud_embedding_dimensions(&self) -> usize { - 8 - } - - fn ollama_embedding_provider( - &self, - _base_url: &str, - _model: &str, - _dims: usize, - ) -> Result, String> { - Ok(Box::new(tinymemory_api::host::NoopEmbedding)) - } -} -// `load`/`save` are the extension trait, not inherent methods: `SyncState` -// itself moved to the contract crate, which stays free of I/O, so persistence -// lives here in the engine and arrives through `PersistedSyncState`. -use tinymemory_core::sync::composio::providers::sync_state::{ - PersistedSyncState, SyncState, KV_NAMESPACE, -}; -use tinymemory_core::sync::pipelines::composio::ComposioClient; -use tinymemory_core::sync::pipelines::composio::GmailSyncPipeline; -use tinymemory_core::sync::pipelines::dispatcher::SyncDispatcher; -use tinymemory_core::sync::pipelines::host::PipelineHost; -use tinymemory_core::sync::pipelines::traits::{ - ComposioMode, ComposioSyncConfig, PipelineConfig, SecretString, SyncPipeline, -}; - -fn message(id: &str, subject: &str, body_md: &str) -> serde_json::Value { - json!({ - "id": id, - "subject": subject, - "from": "sender@example.com", - "markdown": body_md, - "messageTimestamp": "2026-01-02T03:04:05Z", - }) -} - -#[tokio::test(flavor = "multi_thread")] -async fn composio_gmail_sync_completes_against_the_namespace_driver() { - // ── The mock Composio ──────────────────────────────────────────────── - let server = MockServer::start().await; - - // Page 1: two messages and a cursor. Matched on the *absence* of a page - // token in the arguments, so retries stay deterministic. - Mock::given(method("POST")) - .and(path("/tools/execute/GMAIL_FETCH_EMAILS")) - .and(body_partial_json(json!({"arguments": {"max_results": 25}}))) - .and(NoPageToken) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "successful": true, - "data": { - "messages": [ - message("m1", "First", "hello one"), - message("m2", "Second", "hello two"), - ], - "nextPageToken": "page-2", - } - }))) - .mount(&server) - .await; - - // Page 2: one message, no cursor — the sync must stop here. - Mock::given(method("POST")) - .and(path("/tools/execute/GMAIL_FETCH_EMAILS")) - .and(body_partial_json( - json!({"arguments": {"page_token": "page-2"}}), - )) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "successful": true, - "data": { - "messages": [message("m3", "Third", "hello three")], - } - }))) - .mount(&server) - .await; - - // ── The non-TinyCortex driver ──────────────────────────────────────── - tinymemory_core::embedding_host::set_embedding_host(Arc::new(NoopEmbeddingHost)); - let workspace = tempfile::tempdir().expect("workspace"); - let memory = Arc::new( - MemoryClient::from_workspace_dir(workspace.path().to_path_buf()) - .expect("bind the namespace store"), - ); - - // ── The engine-free pipeline, on the production dispatcher ─────────── - let composio = ComposioSyncConfig { - mode: ComposioMode::Direct, - base_url: server.uri(), - api_key: Some(SecretString::new("test-key")), - bearer_token: None, - entity_id: Some("entity-1".into()), - gmail_query: None, - }; - let pipeline = Arc::new(GmailSyncPipeline::new( - ComposioClient::new(composio), - "conn-1", - )); - let pipeline_id = pipeline.id().to_owned(); - - let host = Arc::new(PipelineHost::without_tree_ingest(memory.clone())); - let mut dispatcher = SyncDispatcher::new(); - dispatcher.register(pipeline).expect("register pipeline"); - let outcome = dispatcher - .tick(&pipeline_id, &PipelineConfig::default(), &host.context()) - .await - .expect("gmail sync must complete"); - - // ── End to end: the outcome ────────────────────────────────────────── - assert_eq!( - outcome.records_ingested, 3, - "all three messages ingest; outcome={outcome:?}" - ); - assert!(!outcome.more_pending, "page 2 carried no cursor"); - - // ── End to end: the documents landed in the bound store ────────────── - let docs = memory - .list_documents(Some("skill-gmail")) - .await - .expect("list synced documents"); - let listed = docs - .as_array() - .or_else(|| docs.get("documents").and_then(|d| d.as_array())) - .map(|a| a.len()) - .unwrap_or_default(); - assert_eq!(listed, 3, "three documents in skill-gmail: {docs}"); - - // ── End to end: the canonical markdown, not raw JSON ───────────────── - let doc = memory - .get_document("skill-gmail", "gmail:m1") - .await - .expect("read gmail:m1") - .expect("gmail:m1 stored"); - assert!( - doc.content.contains("From: sender@example.com") && doc.content.contains("hello one"), - "canonical markdown stored, got: {}", - doc.content - ); - - // ── End to end: cursor + dedup state persisted through the KV seam ─── - let state = SyncState::load(&*host, "gmail", "conn-1") - .await - .expect("load persisted state"); - assert!(state.is_synced("m1") && state.is_synced("m3"), "dedup ids"); - let raw = memory - .kv_get(Some(KV_NAMESPACE), "gmail:conn-1") - .await - .expect("kv read"); - assert!(raw.is_some(), "sync state persisted under {KV_NAMESPACE}"); -} diff --git a/crates/tinymemory-core/tests/host_seams.rs b/crates/tinymemory-core/tests/host_seams.rs index 83a0b867..6f0a0d8d 100644 --- a/crates/tinymemory-core/tests/host_seams.rs +++ b/crates/tinymemory-core/tests/host_seams.rs @@ -24,10 +24,6 @@ mod chat_host { pub use tinymemory_core::chat_host::*; } -mod composio_host { - pub use tinymemory_core::composio_host::*; -} - type Config = tinymemory_core::Config; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; @@ -265,24 +261,15 @@ async fn nlp_host_reports_unwired_then_returns_host_response() { async fn required_host_seams_fail_loudly_when_unwired() { let _guard = seam_guard().await; let chat = crate::chat_host::chat_host(); - let composio = crate::composio_host::composio_host(); let _chat_restore = Restore::new(move || match chat { Some(host) => crate::chat_host::set_chat_host(host), None => crate::chat_host::clear_chat_host(), }); - let _composio_restore = Restore::new(move || match composio { - Some(host) => crate::composio_host::set_composio_host(host), - None => crate::composio_host::clear_composio_host(), - }); crate::chat_host::clear_chat_host(); - crate::composio_host::clear_composio_host(); assert!(crate::chat_host::require_chat_host() .expect_err("chat host must be required") .contains("no ChatHost installed")); - assert!(crate::composio_host::require_composio_host() - .expect_err("Composio host must be required") - .contains("no ComposioHost installed")); let config = TestHostConfig::default(); assert_eq!( crate::chat_host::provider_for_role("memory", &config), @@ -292,6 +279,4 @@ async fn required_host_seams_fail_loudly_when_unwired() { crate::chat_host::summarizer_available(&config), (false, "no chat host installed — summarisation cannot run") ); - assert!(!crate::composio_host::is_available(&config)); - assert_eq!(crate::composio_host::api_key(&config), None); } diff --git a/crates/tinymemory-tinycortex/src/engine/mod.rs b/crates/tinymemory-tinycortex/src/engine/mod.rs index 88e2f9d3..13bcf05b 100644 --- a/crates/tinymemory-tinycortex/src/engine/mod.rs +++ b/crates/tinymemory-tinycortex/src/engine/mod.rs @@ -2392,20 +2392,16 @@ impl MemoryProvider for TinycortexProvider { // agrees only while the field *names* agree on both sides, and it fails at // runtime rather than at compile time when they stop. -/// Toolkits with no native pipeline are refused before anything is dispatched. -/// -/// The pipeline builder refuses them too, but as a `PipelineFailure` carrying a -/// message — and by the time it does, this adapter can no longer tell "you -/// asked for a provider that does not exist" apart from "the provider failed". -/// The contract promises [`MemoryError::Invalid`] for the first, and on a call -/// that costs money the difference decides whether a caller retries. -fn ensure_syncable_toolkit(toolkit: &str) -> Result<(), MemoryError> { - if tinymemory_core::sync::pipelines::host::is_composio_toolkit_syncable(toolkit) { - return Ok(()); - } - Err(MemoryError::Invalid(format!( - "memory sync has no pipeline for toolkit '{toolkit}'" - ))) +/// Composio connections are no longer dispatched from an in-process +/// pipeline: reaching a connected account needs a credential this crate does +/// not hold and must not. The host fetches through the `tinyconnectors` +/// module and hands the records back through +/// `MemorySourceSink::accept_source_items`, so every toolkit-keyed entry +/// point below refuses rather than dispatching. +fn refuse_composio_dispatch(action: &str) -> MemoryError { + MemoryError::Invalid(format!( + "{action} is synced through the connector module, not this engine" + )) } /// Carry one audit row across as the contract's own shape. @@ -2473,37 +2469,10 @@ impl MemorySourceSync for TinycortexProvider { async fn run_connection_sync( &self, toolkit: &str, - connection_id: &str, + _connection_id: &str, ) -> Result { - ensure_syncable_toolkit(toolkit)?; - // The registry lookup, the per-source budgets and the pipeline dispatch - // all happen inside this call — which is why the contract carries no - // budget arguments: they are already recorded against the source. - let outcome = tinymemory_core::tinycortex::run_composio_connection( - toolkit, - connection_id, - &self.config, - ) - .await - .map_err(|failure| { - // The usage travels in the message rather than being dropped: a run - // that failed after calling four provider actions and spending real - // money has to say so somewhere, and `IngestOutcome`-style partial - // success is not available on an error path. - MemoryError::Other(anyhow::anyhow!( - "sync {toolkit} connection: {} (actions_called={}, provider_cost_usd={})", - failure.message, - failure.actions_called, - failure.provider_cost_usd - )) - })?; - Ok(SyncRunOutcome { - records_ingested: outcome.records_ingested, - more_pending: outcome.more_pending, - actions_called: outcome.actions_called, - provider_cost_usd: outcome.provider_cost_usd, - note: outcome.note, - }) + let _ = toolkit; + Err(refuse_composio_dispatch("a composio connection")) } async fn run_source_sync(&self, source_id: &str) -> Result { @@ -2544,99 +2513,30 @@ impl MemorySourceSync for TinycortexProvider { async fn bootstrap_connection( &self, toolkit: &str, - connection_id: &str, + _connection_id: &str, ) -> Result<(), MemoryError> { - use tinymemory_core::sync::composio::providers::{get_provider, ProviderContext}; - - // Same gate as `run_connection_sync`, and deliberately before the - // provider lookup: a toolkit with no pipeline cannot bootstrap into - // anything a later sync would read, so reporting it here names the - // real problem rather than "no provider". - ensure_syncable_toolkit(toolkit)?; - - let provider = get_provider(toolkit).ok_or_else(|| { - MemoryError::Invalid(format!("no composio provider registered for '{toolkit}'")) - })?; - - // `from_config` answers `None` when no Composio client resolves in - // either mode — the not-signed-in case. That is `Invalid` rather than a - // silent `Ok`: a caller that just authorised a connection and gets a - // success back would believe the profile was fetched. - let ctx = ProviderContext::from_config( - self.config.to_arc(), - toolkit, - Some(connection_id.to_string()), - ) - .ok_or_else(|| { - MemoryError::Invalid(format!( - "no viable composio client for '{toolkit}'; connection {connection_id} \ - cannot bootstrap" - )) - })?; - - // `max_items` / `sync_depth_days` are left at their defaults on - // purpose. They cap how much a *sync* walks; a bootstrap fetches one - // profile and registers what the provider needs, and giving it a walk - // budget would imply it walks. - provider.on_connection_created(&ctx).await.map_err(|error| { - MemoryError::Other(anyhow::anyhow!( - "bootstrap {toolkit} connection {connection_id}: {error}" - )) - }) + let _ = toolkit; + Err(refuse_composio_dispatch( + "bootstrapping a composio connection", + )) } - async fn is_toolkit_syncable(&self, toolkit: &str) -> Result { - // The same predicate `ensure_syncable_toolkit` gates on, exposed rather - // than inferred: a caller that learned this from a failed sync would - // already have registered the source it should not have. - Ok(tinymemory_core::sync::pipelines::host::is_composio_toolkit_syncable(toolkit)) + async fn is_toolkit_syncable(&self, _toolkit: &str) -> Result { + // No toolkit has an in-process pipeline any more: every composio + // toolkit is now synced through the connector module, which this + // contract entry point does not reach into. + Ok(false) } async fn source_sync_state( &self, toolkit: &str, - connection_id: &str, + _connection_id: &str, ) -> Result, MemoryError> { - use tinymemory_core::sync::composio::providers::sync_state::{SyncState, STATE_NAMESPACE}; - - // Read the row rather than calling `SyncState::load`, which materialises - // a fresh default when nothing is persisted. That default is right for a - // *run* — it is the state a first sync starts from — and wrong here: the - // contract distinguishes "never synced" from "synced and holding no - // cursor", and `load` cannot. - // - // The namespace and the key come from the engine's own constant and its - // own `key`, so this read cannot address a different row than the writes - // do; a literal here would be a second spelling of a durable key. - let key = SyncState::key(toolkit, connection_id); - let stored = self - .client - .kv_get(Some(STATE_NAMESPACE), &key) - .await - .map_err(|error| Self::other("read composio sync state", error))?; - let Some(stored) = stored else { - return Ok(None); - }; - let state: SyncState = serde_json::from_value(stored) - .map_err(|error| Self::other("decode composio sync state", error))?; - - // `remaining()` applies the engine's own day-rollover rule, so a budget - // last written yesterday reads as fully available today. Deriving the - // used count from it rather than from `requests_used` keeps that rule in - // one place — a second date comparison here would show yesterday's spend - // as today's the moment the two disagreed about what a day is. - let limit = state.daily_budget.limit; - let used = limit.saturating_sub(state.daily_budget.remaining()); - Ok(Some(SourceSyncState { - toolkit: state.toolkit, - connection_id: state.connection_id, - cursor: state.cursor, - synced_item_count: u64::try_from(state.synced_ids.len()).unwrap_or(u64::MAX), - last_seen_id: state.last_seen_id, - last_sync_at_ms: state.last_sync_at_ms, - daily_requests_used: used, - daily_request_limit: limit, - })) + let _ = toolkit; + Err(refuse_composio_dispatch( + "reading a composio connection's sync state", + )) } async fn sync_audit_log( diff --git a/crates/tinymemory-tinycortex/src/engine/test.rs b/crates/tinymemory-tinycortex/src/engine/test.rs index e508beaf..7a4455e0 100644 --- a/crates/tinymemory-tinycortex/src/engine/test.rs +++ b/crates/tinymemory-tinycortex/src/engine/test.rs @@ -23,9 +23,9 @@ use tinymemory_api::provider::types::IngestItem; use tinymemory_api::types::MemoryTaint; use super::{ - advertised_capabilities, audit_entry, diagnosis_failure, ensure_syncable_toolkit, - facet_type_to_engine, handle_to_contract, handle_to_engine, parse_person_id, scope_to_engine, - validate_ingest_item, EngineRuntimeConfig, + advertised_capabilities, audit_entry, diagnosis_failure, facet_type_to_engine, + handle_to_contract, handle_to_engine, parse_person_id, refuse_composio_dispatch, + scope_to_engine, validate_ingest_item, EngineRuntimeConfig, }; fn ingest_item(content: &str, mime: Option<&str>, taint: MemoryTaint) -> IngestItem { @@ -338,27 +338,18 @@ fn people_profile_and_scope_boundary_conversions_are_total_and_fail_closed() { } #[test] -fn an_unsyncable_toolkit_is_refused_before_anything_is_dispatched() { - // The pipeline builder refuses these too, but as a message inside a - // `PipelineFailure` — by which point this adapter can no longer tell - // "there is no such provider" from "the provider failed". On a call that - // spends money, the caller acts differently on each. - let error = ensure_syncable_toolkit("definitely-not-a-provider") - .expect_err("a toolkit with no pipeline must be refused"); +fn composio_dispatch_is_refused_regardless_of_toolkit() { + // Composio connections are read by the connector module, not this + // engine: reaching a connected account needs a credential this crate + // does not hold and must not. Every toolkit-keyed dispatch entry point + // refuses unconditionally now, rather than gating on which toolkit used + // to have a native pipeline. + let error = refuse_composio_dispatch("a composio connection"); assert!( matches!(error, MemoryError::Invalid(_)), "expected Invalid, got {error:?}" ); - - // Every toolkit the engine-free pipelines actually build, and the - // case/whitespace forms a caller may send: the gate normalises, so a - // padded slug must not be refused here and then accepted downstream. - for toolkit in ["gmail", "Slack", " github ", "notion", "linear", "clickup"] { - assert!( - ensure_syncable_toolkit(toolkit).is_ok(), - "`{toolkit}` has a native pipeline and must not be refused" - ); - } + assert!(error.to_string().contains("connector module")); } #[test]