From cb9221bf9cd0ae7d203b123ce477148e769fe9dd Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:40:02 +0300 Subject: [PATCH 01/16] chore: remove composio integration The entire Composio integration has been removed from the codebase, including all provider implementations, sync pipelines, API types, and host wiring. This was done because the integration is no longer supported or needed. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinymemory-api/src/host/composio.rs | 878 ------------ crates/tinymemory-core/src/composio_host.rs | 153 -- .../src/sources/readers/composio.rs | 110 -- .../tinymemory-core/src/sync/composio/mod.rs | 214 --- .../src/sync/composio/periodic.rs | 1274 ----------------- .../src/sync/composio/providers/catalogs.rs | 38 - .../composio/providers/catalogs_business.rs | 524 ------- .../composio/providers/catalogs_google.rs | 360 ----- .../composio/providers/catalogs_messaging.rs | 386 ----- .../composio/providers/catalogs_microsoft.rs | 207 --- .../providers/catalogs_productivity.rs | 600 -------- .../providers/catalogs_social_media.rs | 248 ---- .../sync/composio/providers/clickup/mod.rs | 27 - .../composio/providers/clickup/provider.rs | 271 ---- .../sync/composio/providers/clickup/tests.rs | 155 -- .../sync/composio/providers/clickup/tools.rs | 124 -- .../sync/composio/providers/descriptions.rs | 61 - .../src/sync/composio/providers/github/mod.rs | 26 - .../composio/providers/github/provider.rs | 547 ------- .../sync/composio/providers/github/tests.rs | 611 -------- .../sync/composio/providers/github/tools.rs | 189 --- .../src/sync/composio/providers/gmail/mod.rs | 11 - .../sync/composio/providers/gmail/provider.rs | 177 --- .../sync/composio/providers/gmail/tests.rs | 29 - .../sync/composio/providers/gmail/tools.rs | 145 -- .../src/sync/composio/providers/helpers.rs | 56 - .../src/sync/composio/providers/linear/mod.rs | 16 - .../composio/providers/linear/provider.rs | 241 ---- .../sync/composio/providers/linear/tests.rs | 172 --- .../sync/composio/providers/linear/tools.rs | 90 -- .../src/sync/composio/providers/mod.rs | 615 -------- .../src/sync/composio/providers/notion/mod.rs | 11 - .../composio/providers/notion/provider.rs | 411 ------ .../sync/composio/providers/notion/tests.rs | 119 -- .../sync/composio/providers/notion/tools.rs | 196 --- .../src/sync/composio/providers/profile.rs | 821 ----------- .../src/sync/composio/providers/profile_md.rs | 718 ---------- .../src/sync/composio/providers/registry.rs | 151 -- .../sync/composio/providers/scope_lookup.rs | 78 - .../src/sync/composio/providers/slack/mod.rs | 19 - .../sync/composio/providers/slack/provider.rs | 337 ----- .../sync/composio/providers/slack/types.rs | 68 - .../src/sync/composio/providers/sync_state.rs | 326 ----- .../src/sync/composio/providers/tool_scope.rs | 200 --- .../src/sync/composio/providers/traits.rs | 405 ------ .../src/sync/composio/providers/types.rs | 523 ------- .../sync/composio/providers/user_scopes.rs | 159 -- .../composio/providers/user_scopes_tests.rs | 103 -- .../src/sync/pipelines/composio/client.rs | 393 ----- .../src/sync/pipelines/composio/connect.rs | 364 ----- .../sync/pipelines/composio/connect_tests.rs | 305 ---- .../src/sync/pipelines/composio/gmail.rs | 380 ----- .../sync/pipelines/composio/gmail_tests.rs | 101 -- .../src/sync/pipelines/composio/mod.rs | 22 - .../sync/pipelines/composio/orchestrator.rs | 528 ------- .../pipelines/composio/orchestrator_tests.rs | 236 --- .../src/sync/pipelines/composio/page_size.rs | 79 - .../pipelines/composio/page_size_tests.rs | 51 - .../pipelines/composio/providers/clickup.rs | 196 --- .../pipelines/composio/providers/common.rs | 110 -- .../pipelines/composio/providers/github.rs | 178 --- .../composio/providers/google_calendar.rs | 173 --- .../composio/providers/google_docs.rs | 216 --- .../composio/providers/google_drive.rs | 179 --- .../composio/providers/google_sheets.rs | 186 --- .../pipelines/composio/providers/linear.rs | 193 --- .../sync/pipelines/composio/providers/mod.rs | 28 - .../pipelines/composio/providers/notion.rs | 193 --- .../pipelines/composio/providers/outlook.rs | 205 --- .../pipelines/composio/providers/slack.rs | 467 ------ .../composio/providers/slack_parse.rs | 91 -- .../pipelines/composio/providers/todoist.rs | 226 --- 72 files changed, 18300 deletions(-) delete mode 100644 crates/tinymemory-api/src/host/composio.rs delete mode 100644 crates/tinymemory-core/src/composio_host.rs delete mode 100644 crates/tinymemory-core/src/sources/readers/composio.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/mod.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/periodic.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/catalogs.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/catalogs_business.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/catalogs_google.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/catalogs_messaging.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/catalogs_microsoft.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/catalogs_productivity.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/catalogs_social_media.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/clickup/mod.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/clickup/provider.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/clickup/tests.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/clickup/tools.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/descriptions.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/github/mod.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/github/provider.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/github/tests.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/github/tools.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/gmail/mod.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/gmail/provider.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/gmail/tests.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/gmail/tools.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/helpers.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/linear/mod.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/linear/provider.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/linear/tests.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/linear/tools.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/mod.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/notion/mod.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/notion/provider.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/notion/tests.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/notion/tools.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/profile.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/profile_md.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/registry.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/scope_lookup.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/slack/mod.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/slack/provider.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/slack/types.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/sync_state.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/tool_scope.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/traits.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/types.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/user_scopes.rs delete mode 100644 crates/tinymemory-core/src/sync/composio/providers/user_scopes_tests.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/client.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/connect.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/connect_tests.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/gmail.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/gmail_tests.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/mod.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/orchestrator.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/orchestrator_tests.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/page_size.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/page_size_tests.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/clickup.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/common.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/github.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/google_calendar.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/google_docs.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/google_drive.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/google_sheets.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/linear.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/mod.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/notion.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/outlook.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/slack.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/slack_parse.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/composio/providers/todoist.rs diff --git a/crates/tinymemory-api/src/host/composio.rs b/crates/tinymemory-api/src/host/composio.rs deleted file mode 100644 index ddb7b705..00000000 --- a/crates/tinymemory-api/src/host/composio.rs +++ /dev/null @@ -1,878 +0,0 @@ -//! Composio value types — connections, capabilities, execute responses. -//! -//! Moved here from the host's `integrations::composio::types` because the -//! extracted memory sync pipelines read these fields directly on every run, and -//! a trait accessor per field would be absurd. They are inert serde data with -//! no behaviour and no dependencies beyond `serde`, so the contract crate's -//! dependency-light guarantee is unaffected. -//! -//! The Composio *client* deliberately did not come with them — see -//! `tinymemory_core::composio_host`. Its `Direct` variant wraps a host agent -//! tool, and mode dispatch is host policy. -//! -//! Domain types for the Composio integration. -//! -//! These mirror the response envelopes emitted by the openhuman backend under -//! `/agent-integrations/composio/*`. See: -//! - `src/routes/agentIntegrations/composio.ts` -//! - `src/controllers/agentIntegrations/composio/*.ts` -//! in the backend repo for the authoritative shapes. - -use serde::{Deserialize, Deserializer, Serialize}; - -/// Accepts either a JSON string or an object whose first matching field -/// (`slug`/`id`/`name`/`key`) is a string. Lets us tolerate upstream -/// shape drift where a previously-stringy field is now nested in an -/// object — e.g. `"toolkit": {"slug": "gmail", "logo": "…"}`. -fn de_string_or_object<'de, D: Deserializer<'de>>(d: D) -> Result { - use serde::de::Error; - let v = serde_json::Value::deserialize(d)?; - match v { - serde_json::Value::String(s) => Ok(s), - serde_json::Value::Object(map) => { - for key in ["slug", "id", "name", "key"] { - if let Some(serde_json::Value::String(s)) = map.get(key) { - return Ok(s.clone()); - } - } - Err(D::Error::custom( - "expected string or object with slug/id/name/key field", - )) - } - other => Err(D::Error::custom(format!( - "expected string, got {}", - match other { - serde_json::Value::Null => "null", - serde_json::Value::Bool(_) => "bool", - serde_json::Value::Number(_) => "number", - serde_json::Value::Array(_) => "array", - _ => "unknown", - } - ))), - } -} - -/// Like [`de_string_or_object`] but optional and resilient: missing / -/// null / unrecognized object shapes return `None` instead of erroring. -fn de_opt_string_or_object<'de, D: Deserializer<'de>>(d: D) -> Result, D::Error> { - let v = Option::::deserialize(d)?; - Ok(match v { - None | Some(serde_json::Value::Null) => None, - Some(serde_json::Value::String(s)) => Some(s), - Some(serde_json::Value::Object(map)) => { - let mut found = None; - for key in ["state", "value", "slug", "id", "name", "key"] { - if let Some(serde_json::Value::String(s)) = map.get(key) { - found = Some(s.clone()); - break; - } - } - found - } - _ => None, - }) -} - -// ── Toolkits ──────────────────────────────────────────────────────── - -/// One toolkit from the live Composio catalog, forwarded verbatim from the -/// backend (`GET /agent-integrations/composio/toolkits`). -/// -/// The core does not interpret these fields — it passes them straight through -/// to the desktop UI so the app no longer hardcodes toolkit display metadata -/// (see the workspace `COMPOSIO_DYNAMIC_CATALOG_PLAN.md`). Everything except -/// `slug` is best-effort; backends predating the dynamic catalog omit the -/// whole `catalog` array, in which case the UI falls back to local metadata. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioToolkitCatalogEntry { - /// Toolkit slug as Composio emits it, e.g. `"googlecalendar"`. - pub slug: String, - /// Human-readable name, e.g. `"Google Calendar"`. - #[serde(default)] - pub name: String, - /// Composio-hosted logo URL (`meta.logo`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub logo: Option, - /// Short description (`meta.description`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub description: Option, - /// Composio category names (`meta.categories`). - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub categories: Vec, - /// Whether the user can connect/use this toolkit (passed the backend gate). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub enabled: Option, -} - -/// Response body of `GET /agent-integrations/composio/toolkits`. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioToolkitsResponse { - /// Server-enforced toolkit allowlist, e.g. `["gmail", "notion"]`. - #[serde(default)] - pub toolkits: Vec, - /// Rich render model from the live Composio catalog. Optional — empty when - /// the backend predates the dynamic catalog. Forwarded as-is to the UI. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub catalog: Vec, -} - -/// One row in OpenHuman's local Composio capability matrix. -/// -/// Unlike `ComposioToolkitsResponse`, this is not tied to a signed-in -/// backend/direct Composio session. It describes what this core build knows -/// how to do for each toolkit: whether the toolkit has a native provider -/// implementation, a curated tool catalog, profile/sync hooks, and memory -/// ingestion support. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioCapability { - pub toolkit: String, - pub description: String, - pub native_provider: bool, - pub curated_tools: bool, - pub curated_tool_count: usize, - pub tool_execution: bool, - pub user_profile: bool, - pub initial_sync: bool, - pub periodic_sync: bool, - pub sync_interval_secs: Option, - pub trigger_webhooks: bool, - pub memory_ingest: bool, -} - -/// Response body of `composio.list_capabilities`. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioCapabilitiesResponse { - #[serde(default)] - pub capabilities: Vec, -} - -/// Response body of `composio.list_agent_ready_toolkits`. -/// -/// Sorted slugs that have a curated agent catalog — the frontend -/// uses this to decide whether to label a connected toolkit as -/// "preview / agent integration coming soon". See #2283. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioAgentReadyToolkitsResponse { - #[serde(default)] - pub toolkits: Vec, -} - -// ── Connections ───────────────────────────────────────────────────── - -/// One connected Composio account (OAuth integration instance). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioConnection { - /// Composio connection id (what you DELETE to disconnect). - pub id: String, - /// Toolkit slug, e.g. `"gmail"`. - pub toolkit: String, - /// Connection status — `"ACTIVE"`, `"CONNECTED"`, `"PENDING"`, … - pub status: String, - /// ISO timestamp (backend passes this through from Composio). - #[serde(rename = "createdAt", default, skip_serializing_if = "Option::is_none")] - pub created_at: Option, - /// Account email — populated from the cached provider profile when - /// the toolkit reports an email address (e.g. Gmail, Google Calendar, - /// Google Sheets). Lets the UI picker show "Gmail · user@example.com" - /// instead of a generic "Account N" label. - #[serde( - rename = "accountEmail", - default, - skip_serializing_if = "Option::is_none" - )] - pub account_email: Option, - /// Workspace or team display name — populated for workspace-based - /// services (e.g. Slack: user display name / team name, Notion: workspace - /// name). Used by the picker when no email is available. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub workspace: Option, - /// Screen name or handle — populated for username-based services - /// (e.g. GitHub login, Twitter handle). Used by the picker as a - /// last-resort identity hint after email and workspace. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub username: Option, -} - -impl ComposioConnection { - /// Return the toolkit slug in the canonical form used by provider - /// lookup, prompt injection, and tool-action prefix matching. - pub fn normalized_toolkit(&self) -> String { - self.toolkit.trim().to_ascii_lowercase() - } - - /// Whether this row represents a usable connection. - /// - /// The web UI already treats status case-insensitively. Keep the - /// core-side chat/runtime filters aligned so a backend spelling such - /// as `connected` cannot display as connected in Settings while - /// disappearing from the agent's integration surface. - pub fn is_active(&self) -> bool { - let status = self.status.trim(); - status.eq_ignore_ascii_case("ACTIVE") || status.eq_ignore_ascii_case("CONNECTED") - } -} - -/// Response body of `GET /agent-integrations/composio/connections`. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioConnectionsResponse { - #[serde(default)] - pub connections: Vec, -} - -/// Response body of `POST /agent-integrations/composio/authorize`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioAuthorizeResponse { - /// Composio-hosted OAuth URL the user opens in a browser. - #[serde(rename = "connectUrl")] - pub connect_url: String, - /// Composio connection id created by this authorize call. - #[serde(rename = "connectionId")] - pub connection_id: String, -} - -/// Response body of `DELETE /agent-integrations/composio/connections/:id`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioDeleteResponse { - #[serde(default)] - pub deleted: bool, - #[serde(default)] - pub memory_chunks_deleted: usize, -} - -// ── Tools ─────────────────────────────────────────────────────────── - -/// OpenAI function-calling schema returned by the backend for each tool. -/// -/// The backend wraps Composio's upstream shape; we keep the `type` + -/// `function` envelope so callers can forward directly into an LLM. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioToolSchema { - #[serde(rename = "type", default = "default_function_type")] - pub kind: String, - pub function: ComposioToolFunction, -} - -fn default_function_type() -> String { - "function".to_string() -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioToolFunction { - /// Composio action slug, e.g. `"GMAIL_SEND_EMAIL"`. - pub name: String, - /// Human-readable description shown to the model. - #[serde(default)] - pub description: Option, - /// JSON schema for the tool's INPUT parameters. - #[serde(default)] - pub parameters: Option, - /// JSON schema describing the tool's OUTPUT/return-value shape, when the - /// upstream listing publishes one. Composio's v3 `/tools` endpoint calls - /// this `output_parameters` — documented as "Schema definition of return - /// values from the tool" - /// () — - /// alongside `input_parameters`. `None` means "unknown" (not "empty"): - /// the backend-proxied `/agent-integrations/composio/tools` path is - /// opaque to this crate and may not forward it, and not every Composio - /// action publishes an output schema. - #[serde(default)] - pub output_parameters: Option, -} - -/// Response body of `GET /agent-integrations/composio/tools`. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioToolsResponse { - #[serde(default)] - pub tools: Vec, -} - -// ── Execute ───────────────────────────────────────────────────────── - -/// Response body of `POST /agent-integrations/composio/execute`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioExecuteResponse { - /// Raw result from the upstream provider. - #[serde(default)] - pub data: serde_json::Value, - /// Did the provider report success? - #[serde(default)] - pub successful: bool, - /// Provider error message if any. - #[serde(default)] - pub error: Option, - /// Amount charged to the caller (base + margin) in USD. - #[serde(rename = "costUsd", default)] - pub cost_usd: f64, - /// Backend-rendered compact markdown for known tools (set by - /// backend PR tinyhumansai/backend#683). When present and non-empty - /// callers should prefer this over `data` for LLM/CLI consumption. - #[serde(rename = "markdownFormatted", default)] - pub markdown_formatted: Option, -} - -// ── GitHub repos + triggers ───────────────────────────────────────── - -/// One repository returned by `GET /agent-integrations/composio/github/repos`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioGithubRepo { - pub owner: String, - pub repo: String, - #[serde(rename = "fullName")] - pub full_name: String, - #[serde(default)] - pub private: Option, - #[serde(rename = "defaultBranch", default)] - pub default_branch: Option, - #[serde(rename = "htmlUrl", default)] - pub html_url: Option, -} - -/// Response body of `GET /agent-integrations/composio/github/repos`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioGithubReposResponse { - #[serde(rename = "connectionId")] - pub connection_id: String, - #[serde(default, rename = "repositories")] - pub repositories: Vec, -} - -/// Response body of `POST /agent-integrations/composio/triggers`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioCreateTriggerResponse { - #[serde(rename = "triggerId")] - pub trigger_id: String, - #[serde(default)] - pub status: Option, -} - -// ── Trigger management (catalog + active list + enable/disable) ───── - -/// Per-repo descriptor used by GitHub-scoped available triggers. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioAvailableTriggerRepo { - pub owner: String, - pub repo: String, -} - -/// One entry in `GET /agent-integrations/composio/triggers/available`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioAvailableTrigger { - pub slug: String, - /// `"static"` or `"github_repo"`. - pub scope: String, - #[serde( - rename = "defaultConfig", - default, - skip_serializing_if = "Option::is_none" - )] - pub default_config: Option, - #[serde( - rename = "requiredConfigKeys", - default, - skip_serializing_if = "Option::is_none" - )] - pub required_config_keys: Option>, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub repo: Option, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioAvailableTriggersResponse { - #[serde(default)] - pub triggers: Vec, -} - -/// One entry in `GET /agent-integrations/composio/triggers`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioActiveTrigger { - #[serde(deserialize_with = "de_string_or_object")] - pub id: String, - #[serde(deserialize_with = "de_string_or_object")] - pub slug: String, - #[serde(deserialize_with = "de_string_or_object")] - pub toolkit: String, - #[serde(rename = "connectionId", deserialize_with = "de_string_or_object")] - pub connection_id: String, - #[serde( - rename = "triggerConfig", - default, - skip_serializing_if = "Option::is_none" - )] - pub trigger_config: Option, - #[serde( - default, - skip_serializing_if = "Option::is_none", - deserialize_with = "de_opt_string_or_object" - )] - pub state: Option, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioActiveTriggersResponse { - #[serde(default)] - pub triggers: Vec, -} - -/// Response body of `POST /agent-integrations/composio/triggers` (enable). -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioEnableTriggerResponse { - #[serde(rename = "triggerId")] - pub trigger_id: String, - pub slug: String, - #[serde(rename = "connectionId")] - pub connection_id: String, -} - -/// Response body of `DELETE /agent-integrations/composio/triggers/:id`. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioDisableTriggerResponse { - #[serde(default)] - pub deleted: bool, -} - -// ── Triggers ──────────────────────────────────────────────────────── - -/// Payload of the `composio:trigger` Socket.IO event emitted by the backend -/// when a Composio webhook is received, HMAC-verified, and delivered to the -/// user's active sockets. -/// -/// See `src/controllers/agentIntegrations/composio/handleWebhook.ts` in the -/// backend repo. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioTriggerEvent { - /// Toolkit slug, e.g. `"gmail"`. - #[serde(default)] - pub toolkit: String, - /// Trigger slug, e.g. `"GMAIL_NEW_GMAIL_MESSAGE"`. - #[serde(default)] - pub trigger: String, - /// Trigger-specific payload (provider-defined shape). - #[serde(default)] - pub payload: serde_json::Value, - /// Metadata the backend attaches: `{ id, uuid }`. - #[serde(default)] - pub metadata: ComposioTriggerMetadata, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioTriggerMetadata { - #[serde(default)] - pub id: String, - #[serde(default)] - pub uuid: String, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioTriggerHistoryEntry { - /// Unix timestamp in milliseconds when the trigger reached the core. - pub received_at_ms: u64, - /// Toolkit slug, e.g. `"gmail"`. - pub toolkit: String, - /// Trigger slug, e.g. `"GMAIL_NEW_GMAIL_MESSAGE"`. - pub trigger: String, - /// Backend metadata id for this event. - pub metadata_id: String, - /// Backend metadata UUID for this event. - pub metadata_uuid: String, - /// Raw provider payload as forwarded by the backend socket event. - pub payload: serde_json::Value, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ComposioTriggerHistoryResult { - /// Directory containing daily JSONL archives. - pub archive_dir: String, - /// Today's JSONL file path. - pub current_day_file: String, - /// Recent triggers, newest first. - pub entries: Vec, -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn connection_is_active_matches_ui_status_normalization() { - for status in ["ACTIVE", "CONNECTED", "active", "connected", " connected "] { - let conn = ComposioConnection { - id: "c1".into(), - toolkit: "slack".into(), - status: status.into(), - created_at: None, - account_email: None, - workspace: None, - username: None, - }; - assert!(conn.is_active(), "status {status:?} should be active"); - } - - for status in ["PENDING", "INITIATED", "FAILED", ""] { - let conn = ComposioConnection { - id: "c1".into(), - toolkit: "slack".into(), - status: status.into(), - created_at: None, - account_email: None, - workspace: None, - username: None, - }; - assert!(!conn.is_active(), "status {status:?} should not be active"); - } - } - - #[test] - fn connection_normalizes_toolkit_for_runtime_matching() { - let conn = ComposioConnection { - id: "c1".into(), - toolkit: " Slack ".into(), - status: "ACTIVE".into(), - created_at: None, - account_email: None, - workspace: None, - username: None, - }; - assert_eq!(conn.normalized_toolkit(), "slack"); - } - - #[test] - fn toolkits_response_defaults_to_empty() { - let resp: ComposioToolkitsResponse = serde_json::from_str("{}").unwrap(); - assert!(resp.toolkits.is_empty()); - } - - #[test] - fn toolkits_response_roundtrips() { - let resp = ComposioToolkitsResponse { - toolkits: vec!["gmail".into(), "notion".into()], - ..Default::default() - }; - let value = serde_json::to_value(&resp).unwrap(); - // Empty catalog is skipped on the wire — back-compat with old cores. - assert_eq!(value, json!({ "toolkits": ["gmail", "notion"] })); - let back: ComposioToolkitsResponse = serde_json::from_value(value).unwrap(); - assert_eq!(back.toolkits, vec!["gmail", "notion"]); - assert!(back.catalog.is_empty()); - } - - #[test] - fn toolkits_response_forwards_catalog() { - // A backend that sends the dynamic catalog must deserialize and - // re-serialize verbatim so the field reaches the desktop UI. - let raw = json!({ - "toolkits": ["gmail"], - "catalog": [ - { - "slug": "gmail", - "name": "Gmail", - "logo": "https://logos.composio.dev/api/gmail", - "description": "Send and read email", - "categories": ["productivity"], - "enabled": true - } - ] - }); - let resp: ComposioToolkitsResponse = serde_json::from_value(raw).unwrap(); - assert_eq!(resp.catalog.len(), 1); - let entry = &resp.catalog[0]; - assert_eq!(entry.slug, "gmail"); - assert_eq!(entry.name, "Gmail"); - assert_eq!(entry.enabled, Some(true)); - assert_eq!(entry.categories, vec!["productivity".to_string()]); - - // Round-trips back out with the catalog intact. - let value = serde_json::to_value(&resp).unwrap(); - assert_eq!(value["catalog"][0]["slug"], "gmail"); - assert_eq!(value["catalog"][0]["enabled"], true); - } - - #[test] - fn connection_parses_and_serializes_camelcase_created_at() { - let raw = json!({ - "id": "conn_1", - "toolkit": "gmail", - "status": "ACTIVE", - "createdAt": "2026-02-01T00:00:00Z" - }); - let conn: ComposioConnection = serde_json::from_value(raw.clone()).unwrap(); - assert_eq!(conn.id, "conn_1"); - assert_eq!(conn.toolkit, "gmail"); - assert_eq!(conn.status, "ACTIVE"); - assert_eq!(conn.created_at.as_deref(), Some("2026-02-01T00:00:00Z")); - - // Round-trip must use camelCase too. - let serialized = serde_json::to_value(&conn).unwrap(); - assert!(serialized.get("createdAt").is_some()); - } - - #[test] - fn connection_without_created_at_omits_field_when_serialized() { - let conn = ComposioConnection { - id: "x".into(), - toolkit: "notion".into(), - status: "PENDING".into(), - created_at: None, - account_email: None, - workspace: None, - username: None, - }; - let s = serde_json::to_value(&conn).unwrap(); - assert!( - s.get("createdAt").is_none(), - "createdAt must be skipped when None" - ); - } - - #[test] - fn authorize_response_uses_camelcase_keys() { - let raw = json!({ - "connectUrl": "https://composio.dev/oauth/abc", - "connectionId": "conn_2" - }); - let resp: ComposioAuthorizeResponse = serde_json::from_value(raw).unwrap(); - assert_eq!(resp.connect_url, "https://composio.dev/oauth/abc"); - assert_eq!(resp.connection_id, "conn_2"); - - let s = serde_json::to_value(&resp).unwrap(); - assert!(s.get("connectUrl").is_some()); - assert!(s.get("connectionId").is_some()); - } - - #[test] - fn tool_schema_defaults_type_field_to_function() { - let raw = json!({ - "function": { - "name": "GMAIL_SEND_EMAIL", - "description": "Send an email", - "parameters": { "type": "object" } - } - }); - let tool: ComposioToolSchema = serde_json::from_value(raw).unwrap(); - assert_eq!(tool.kind, "function"); - assert_eq!(tool.function.name, "GMAIL_SEND_EMAIL"); - assert_eq!(tool.function.description.as_deref(), Some("Send an email")); - assert!(tool.function.parameters.is_some()); - } - - #[test] - fn tool_function_tolerates_missing_description_and_parameters() { - let raw = json!({ "function": { "name": "SLUG_ONLY" } }); - let tool: ComposioToolSchema = serde_json::from_value(raw).unwrap(); - assert_eq!(tool.function.name, "SLUG_ONLY"); - assert!(tool.function.description.is_none()); - assert!(tool.function.parameters.is_none()); - } - - #[test] - fn execute_response_parses_cost_and_error() { - let raw = json!({ - "data": { "messageId": "m-1" }, - "successful": true, - "error": null, - "costUsd": 0.0025 - }); - let resp: ComposioExecuteResponse = serde_json::from_value(raw).unwrap(); - assert!(resp.successful); - assert!(resp.error.is_none()); - assert!((resp.cost_usd - 0.0025).abs() < f64::EPSILON); - } - - #[test] - fn execute_response_defaults_when_fields_missing() { - let resp: ComposioExecuteResponse = serde_json::from_str("{}").unwrap(); - assert!(!resp.successful); - assert!(resp.error.is_none()); - assert_eq!(resp.cost_usd, 0.0); - assert!(resp.data.is_null()); - } - - #[test] - fn available_trigger_deserializes_and_serializes_camelcase_fields() { - let raw = json!({ - "slug": "GMAIL_NEW_GMAIL_MESSAGE", - "scope": "static", - "defaultConfig": { "labelIds": ["INBOX"] }, - "requiredConfigKeys": ["labelIds"], - "repo": { "owner": "acme", "repo": "inbox" } - }); - let trigger: ComposioAvailableTrigger = serde_json::from_value(raw).unwrap(); - assert_eq!(trigger.slug, "GMAIL_NEW_GMAIL_MESSAGE"); - assert_eq!(trigger.scope, "static"); - assert_eq!( - trigger.default_config, - Some(json!({ "labelIds": ["INBOX"] })) - ); - assert_eq!( - trigger.required_config_keys, - Some(vec!["labelIds".to_string()]) - ); - let repo = trigger.repo.as_ref().expect("repo"); - assert_eq!(repo.owner, "acme"); - assert_eq!(repo.repo, "inbox"); - - let value = serde_json::to_value(&trigger).unwrap(); - assert!(value.get("defaultConfig").is_some()); - assert!(value.get("requiredConfigKeys").is_some()); - } - - #[test] - fn active_trigger_parses_connection_id_and_optional_fields() { - let raw = json!({ - "id": "ti_1", - "slug": "GMAIL_NEW_GMAIL_MESSAGE", - "toolkit": "gmail", - "connectionId": "c-1", - "triggerConfig": { "labelIds": "INBOX" }, - "state": "active" - }); - let trigger: ComposioActiveTrigger = serde_json::from_value(raw).unwrap(); - assert_eq!(trigger.id, "ti_1"); - assert_eq!(trigger.slug, "GMAIL_NEW_GMAIL_MESSAGE"); - assert_eq!(trigger.connection_id, "c-1"); - assert_eq!(trigger.trigger_config, Some(json!({"labelIds":"INBOX"}))); - assert_eq!(trigger.state.as_deref(), Some("active")); - - let value = serde_json::to_value(&trigger).unwrap(); - assert!(value.get("connectionId").is_some()); - assert!(value.get("triggerConfig").is_some()); - assert!(value.get("state").is_some()); - } - - #[test] - fn trigger_enable_response_uses_camelcase_and_optional_defaults() { - let raw = json!({ - "triggerId": "ti_9", - "slug": "GMAIL_NEW_GMAIL_MESSAGE", - "connectionId": "c-9" - }); - let resp: ComposioEnableTriggerResponse = serde_json::from_value(raw).unwrap(); - assert_eq!(resp.trigger_id, "ti_9"); - assert_eq!(resp.slug, "GMAIL_NEW_GMAIL_MESSAGE"); - assert_eq!(resp.connection_id, "c-9"); - - let serialized = serde_json::to_value(&resp).unwrap(); - assert_eq!(serialized.get("triggerId").unwrap(), "ti_9"); - assert_eq!(serialized.get("connectionId").unwrap(), "c-9"); - } - - #[test] - fn delete_trigger_response_defaults_deleted_to_false() { - let raw = json!({}); - let resp: ComposioDisableTriggerResponse = serde_json::from_value(raw).unwrap(); - assert!(!resp.deleted); - } - - #[test] - fn trigger_event_defaults_empty_fields_to_empty_strings() { - let ev: ComposioTriggerEvent = serde_json::from_str("{}").unwrap(); - assert_eq!(ev.toolkit, ""); - assert_eq!(ev.trigger, ""); - assert_eq!(ev.metadata.id, ""); - assert_eq!(ev.metadata.uuid, ""); - assert!(ev.payload.is_null()); - } - - #[test] - fn trigger_event_parses_full_payload() { - let raw = json!({ - "toolkit": "gmail", - "trigger": "GMAIL_NEW_GMAIL_MESSAGE", - "payload": { "subject": "hi" }, - "metadata": { "id": "evt-1", "uuid": "uuid-1" } - }); - let ev: ComposioTriggerEvent = serde_json::from_value(raw).unwrap(); - assert_eq!(ev.toolkit, "gmail"); - assert_eq!(ev.trigger, "GMAIL_NEW_GMAIL_MESSAGE"); - assert_eq!(ev.metadata.id, "evt-1"); - assert_eq!(ev.metadata.uuid, "uuid-1"); - assert_eq!(ev.payload["subject"], "hi"); - } - - #[test] - fn active_trigger_accepts_string_fields() { - let v = json!({ - "id": "t1", - "slug": "GMAIL_NEW_MAIL", - "toolkit": "gmail", - "connectionId": "c1", - "state": "ACTIVE", - }); - let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); - assert_eq!(trig.id, "t1"); - assert_eq!(trig.slug, "GMAIL_NEW_MAIL"); - assert_eq!(trig.toolkit, "gmail"); - assert_eq!(trig.connection_id, "c1"); - assert_eq!(trig.state.as_deref(), Some("ACTIVE")); - } - - #[test] - fn active_trigger_accepts_object_fields() { - // Mirrors upstream API drift where these fields arrive as objects - // rather than plain strings. - let v = json!({ - "id": {"id": "t1"}, - "slug": {"slug": "GMAIL_NEW_MAIL"}, - "toolkit": {"slug": "gmail", "logo": "https://…"}, - "connectionId": {"id": "c1"}, - "state": {"state": "ACTIVE", "slug": "should-be-ignored"}, - }); - let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); - assert_eq!(trig.id, "t1"); - assert_eq!(trig.slug, "GMAIL_NEW_MAIL"); - assert_eq!(trig.toolkit, "gmail"); - assert_eq!(trig.connection_id, "c1"); - // `state` priority must prefer the literal `state` key over metadata. - assert_eq!(trig.state.as_deref(), Some("ACTIVE")); - } - - #[test] - fn active_trigger_state_falls_back_to_value() { - let v = json!({ - "id": "t1", - "slug": "X", - "toolkit": "gmail", - "connectionId": "c1", - "state": {"value": "PENDING"}, - }); - let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); - assert_eq!(trig.state.as_deref(), Some("PENDING")); - } - - #[test] - fn active_trigger_state_missing_or_unknown_returns_none() { - let v = json!({ - "id": "t1", - "slug": "X", - "toolkit": "gmail", - "connectionId": "c1", - }); - let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); - assert!(trig.state.is_none()); - - let v = json!({ - "id": "t1", - "slug": "X", - "toolkit": "gmail", - "connectionId": "c1", - "state": {"unrelated": 42}, - }); - let trig: ComposioActiveTrigger = serde_json::from_value(v).unwrap(); - assert!(trig.state.is_none()); - } - - #[test] - fn active_trigger_required_field_rejects_unsupported_object() { - // Object without any of slug/id/name/key must fail loudly so we - // notice further upstream shape drift instead of silently dropping - // the trigger. - let v = json!({ - "id": {"unrelated": 42}, - "slug": "X", - "toolkit": "gmail", - "connectionId": "c1", - }); - let err = serde_json::from_value::(v).unwrap_err(); - assert!(err.to_string().contains("expected string or object")); - } -} diff --git a/crates/tinymemory-core/src/composio_host.rs b/crates/tinymemory-core/src/composio_host.rs deleted file mode 100644 index 3258901e..00000000 --- a/crates/tinymemory-core/src/composio_host.rs +++ /dev/null @@ -1,153 +0,0 @@ -//! [`ComposioHost`] — the Composio integration, as the memory sync layer sees it. -//! -//! The sync pipelines need three things from Composio: which connections are -//! active, the ability to execute a tool against one, and the direct-mode API -//! key. Everything else about the integration — OAuth, the backend session -//! token, per-toolkit allowlists, HMAC-verified trigger fan-out, and the choice -//! between backend-proxied and direct mode — is host concern. -//! -//! # Why the client itself stayed in the host -//! -//! `ComposioClientKind::Direct` wraps an `Arc`, -//! a host *agent tool*. There is no way to name that from here, and no reason -//! to: mode dispatch reads `config.composio.mode` and fails loud on a typo, -//! which is exactly the kind of policy the README's split assigns to the host. -//! -//! So this trait is deliberately **behavioural, not structural**. It hides -//! `ComposioClientKind` entirely — the core never learns that two modes exist, -//! and the three call sites that used to do -//! `create_composio_client(config)?` → `match kind` → call collapse to one -//! method each. -//! -//! The value types those methods return did move, to -//! [`tinymemory_api::host::composio`], because the pipelines read their fields -//! directly. -//! -//! # Unwired is an error -//! -//! Same reasoning as [`crate::embedding_host`]: a sync run that quietly saw -//! zero connections would look like "nothing to sync" rather than "not wired -//! up", and the difference would only surface as missing memory days later. - -use std::sync::Arc; - -use async_trait::async_trait; -use parking_lot::RwLock; - -use crate::Config; - -pub use tinymemory_api::host::composio::{ - ComposioCapability, ComposioConnection, ComposioExecuteResponse, -}; - -/// The Composio operations the memory sync layer performs. -#[async_trait] -pub trait ComposioHost: Send + Sync + std::fmt::Debug { - /// Every connection the signed-in user has, active or not. - /// - /// Filtering to active ones is the caller's job — `ComposioConnection` - /// carries the status and treats an empty one as inactive, so a malformed - /// upstream row is never presented as connected. - /// - /// # Errors - /// - /// Returns `Err` when no client can be built (no backend session, bad mode - /// string) or the upstream call fails. - async fn list_connections(&self, config: &Config) -> Result, String>; - - /// Execute `tool` against a connection. - /// - /// # Errors - /// - /// Returns `Err` when no client can be built or the call fails. A provider - /// that answers with `successful: false` is **not** an error — that is - /// reported in the returned [`ComposioExecuteResponse`]. - async fn execute( - &self, - config: &Config, - tool: &str, - arguments: Option, - entity_id: &str, - connection_id: Option<&str>, - ) -> Result; - - /// The direct-mode Composio API key from the host's credential store, or - /// `None` when direct mode is not configured. - fn api_key(&self, config: &Config) -> Option; - - /// Whether *some* viable client resolves for the current config. - /// - /// The sync layer uses this as its "is the user signed in?" probe. It must - /// answer for **either** mode: direct-mode users typically have no backend - /// session token, and probing for one alone would falsely skip them. - fn is_available(&self, config: &Config) -> bool; -} - -static HOST: RwLock>> = RwLock::new(None); - -const NOT_INSTALLED: &str = - "no ComposioHost installed — the host must call memory::composio_host::set_composio_host \ - during startup wiring, before any sync runs"; - -/// Install the host's Composio integration. Called once during startup wiring. -pub fn set_composio_host(host: Arc) { - *HOST.write() = Some(host); -} - -/// Remove any installed host. For tests. -pub fn clear_composio_host() { - *HOST.write() = None; -} - -/// The installed host, or `None` when nothing has been wired up. -#[must_use] -pub fn composio_host() -> Option> { - HOST.read().clone() -} - -/// The installed host. -/// -/// # Errors -/// -/// Returns `Err` when no host has been installed. -pub fn require_composio_host() -> Result, String> { - composio_host().ok_or_else(|| NOT_INSTALLED.to_string()) -} - -/// Active-or-not connections for the signed-in user. -/// -/// # Errors -/// -/// Returns `Err` when no host is installed, or the upstream call fails. -pub async fn list_connections(config: &Config) -> Result, String> { - require_composio_host()?.list_connections(config).await -} - -/// Execute a Composio tool. -/// -/// # Errors -/// -/// Returns `Err` when no host is installed, or the call fails. -pub async fn execute( - config: &Config, - tool: &str, - arguments: Option, - entity_id: &str, - connection_id: Option<&str>, -) -> Result { - require_composio_host()? - .execute(config, tool, arguments, entity_id, connection_id) - .await -} - -/// The direct-mode API key, or `None` when unset or unwired. -#[must_use] -pub fn api_key(config: &Config) -> Option { - composio_host()?.api_key(config) -} - -/// Whether a viable Composio client resolves. `false` when unwired. -#[must_use] -pub fn is_available(config: &Config) -> bool { - composio_host().is_some_and(|host| host.is_available(config)) -} diff --git a/crates/tinymemory-core/src/sources/readers/composio.rs b/crates/tinymemory-core/src/sources/readers/composio.rs deleted file mode 100644 index 3bdc6e15..00000000 --- a/crates/tinymemory-core/src/sources/readers/composio.rs +++ /dev/null @@ -1,110 +0,0 @@ -//! Composio source reader — delegates to the existing composio sync layer. -//! -//! For Composio sources, `list_items` returns the sync targets and -//! `read_item` is not meaningful (sync is provider-driven, not -//! item-by-item). The reader exists so the registry can uniformly -//! query all source kinds. - -use async_trait::async_trait; - -#[cfg(test)] -use tinymemory_api::host::test_support::TestHostConfig; - -use crate::sources::types::{ - ContentType, MemorySourceEntry, SourceContent, SourceItem, SourceKind, -}; -use crate::Config; - -use super::SourceReader; - -pub struct ComposioReader; - -#[async_trait] -impl SourceReader for ComposioReader { - fn kind(&self) -> SourceKind { - SourceKind::Composio - } - - async fn list_items( - &self, - source: &MemorySourceEntry, - _config: &Config, - ) -> Result, String> { - let toolkit = source.toolkit.as_deref().unwrap_or("unknown"); - let connection_id = source.connection_id.as_deref().unwrap_or("unknown"); - - tracing::debug!( - toolkit = %toolkit, - connection_id = %connection_id, - "[memory_sources:composio] list_items" - ); - - Ok(vec![SourceItem { - id: connection_id.to_string(), - title: format!("{toolkit} connection"), - updated_at_ms: None, - }]) - } - - async fn read_item( - &self, - source: &MemorySourceEntry, - item_id: &str, - _config: &Config, - ) -> Result { - let toolkit = source.toolkit.as_deref().unwrap_or("unknown"); - Ok(SourceContent { - id: item_id.to_string(), - title: format!("{toolkit} sync data"), - body: format!( - "Composio {toolkit} data is synced via the provider sync pipeline, not read item-by-item." - ), - content_type: ContentType::Plaintext, - metadata: serde_json::json!({ - "toolkit": toolkit, - "connection_id": source.connection_id, - }), - }) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::sources::types::MemorySourceEntry; - - fn test_source() -> MemorySourceEntry { - MemorySourceEntry { - id: "src_1".into(), - kind: SourceKind::Composio, - label: "Gmail".into(), - enabled: true, - toolkit: Some("gmail".into()), - connection_id: Some("cmp_123".into()), - path: None, - glob: None, - url: None, - branch: None, - paths: Vec::new(), - query: None, - since_days: None, - max_items: None, - max_commits: None, - max_issues: None, - max_prs: None, - selector: None, - max_tokens_per_sync: None, - max_cost_per_sync_usd: None, - sync_depth_days: None, - } - } - - #[tokio::test] - async fn list_items_returns_connection_as_item() { - let reader = ComposioReader; - let config = TestHostConfig::default(); - let items = reader.list_items(&test_source(), &config).await.unwrap(); - assert_eq!(items.len(), 1); - assert_eq!(items[0].id, "cmp_123"); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/mod.rs b/crates/tinymemory-core/src/sync/composio/mod.rs deleted file mode 100644 index 6d5d9072..00000000 --- a/crates/tinymemory-core/src/sync/composio/mod.rs +++ /dev/null @@ -1,214 +0,0 @@ -//! Composio-backed sync pipelines. -//! -//! This module owns the "pull upstream provider data into memory" side of -//! Composio integrations: -//! -//! - provider sync implementations (`providers/*/provider.rs`, `sync.rs`) -//! - periodic scheduler (`periodic.rs`) -//! - trigger / connection-created event subscribers (`bus.rs`) -//! - sync-state persistence and profile-to-memory shaping -//! -//! The host's sibling `integrations::composio` domain still owns auth, -//! connection management, action execution, and general Composio RPC/tool -//! surfaces. This submodule is specifically the memory-sync half of that -//! integration boundary. - -use std::sync::Arc; -pub mod periodic; -pub mod providers; - -use crate::composio_host::{self, ComposioConnection}; -use crate::Config; - -pub use periodic::{record_sync_success, start_periodic_sync}; -pub use providers::{ - all_providers as all_composio_sync_providers, get_provider as get_composio_sync_provider, - init_default_providers as init_default_composio_sync_providers, ComposioProvider, - ComposioUsage, ProviderContext, ProviderUserProfile, SyncOutcome, SyncReason, -}; - -/// One provider-backed connection that the memory sync layer can execute. -#[derive(Debug, Clone)] -pub struct SyncTarget { - pub toolkit: String, - pub connection_id: String, -} - -/// List active Composio connections that have a native memory-sync provider. -/// -/// When memory_sources entries exist with `kind=composio` and `enabled=true`, -/// those are used as the authoritative source list (user curated). When no -/// memory_sources composio entries exist, falls back to scanning all active -/// Composio connections (legacy behavior). -pub async fn list_sync_targets(config: &Config) -> Result, String> { - init_default_composio_sync_providers(); - - // Try memory_sources registry first (user-curated list). - let registry_sources = - crate::sources::list_enabled_by_kind(crate::sources::SourceKind::Composio) - .await - .unwrap_or_default(); - - if !registry_sources.is_empty() { - let from_registry: Vec = registry_sources - .into_iter() - .filter_map(|s| { - let toolkit = s.toolkit?; - let connection_id = s.connection_id?; - get_composio_sync_provider(&toolkit).map(|_| SyncTarget { - toolkit, - connection_id, - }) - }) - .collect(); - if !from_registry.is_empty() { - tracing::debug!( - count = from_registry.len(), - "[composio:sync] using memory_sources registry for sync targets" - ); - return Ok(from_registry); - } - // Registry has entries but none yielded a valid target (missing - // fields or unregistered toolkit). Fall through to a fresh scan - // rather than reporting an empty target list — otherwise newly - // connected integrations stay invisible until reconcile runs. - tracing::debug!( - "[composio:sync] registry yielded zero valid targets; falling back to connection scan" - ); - } else { - tracing::debug!( - "[composio:sync] no memory_sources entries; falling back to connection scan" - ); - } - - scan_active_sync_targets(config).await -} - -/// Scan all active Composio connections that have a native memory-sync -/// provider. Always hits Composio directly — does not consult the -/// memory_sources registry. Used by reconciliation to seed the registry. -pub async fn scan_active_sync_targets(config: &Config) -> Result, String> { - init_default_composio_sync_providers(); - - // Mode dispatch lives in the host's `ComposioHost` impl: backend mode - // walks the tinyhumans tenant, direct mode the user's own Composio v3 - // tenant. Either way this side gets one flat list. - let connections = composio_host::list_connections(config).await?; - - Ok(connections - .into_iter() - .filter_map(connection_to_sync_target) - .collect()) -} - -/// Run one provider-backed sync end-to-end in-process. -/// -/// Returns the provider's [`SyncOutcome`] together with the -/// [`ComposioUsage`] tally (billable action count + actual USD cost) -/// accumulated at the `execute` chokepoint during this run, so the -/// sync-audit caller can record Composio API-call cost alongside the LLM -/// summarisation cost (#3111). -pub async fn run_connection_sync( - config: Arc, - connection_id: &str, - reason: SyncReason, -) -> Result<(SyncOutcome, ComposioUsage), (String, ComposioUsage)> { - init_default_composio_sync_providers(); - - let no_usage = |e: String| (e, ComposioUsage::default()); - - let target = list_sync_targets(&*config) - .await - .map_err(no_usage)? - .into_iter() - .find(|target| target.connection_id == connection_id) - .ok_or_else(|| { - no_usage(format!( - "no provider-backed active sync target for connection_id={connection_id}", - )) - })?; - - let provider = get_composio_sync_provider(&target.toolkit).ok_or_else(|| { - no_usage(format!( - "no native memory sync provider registered for toolkit '{}'", - target.toolkit, - )) - })?; - - // Look up the source entry to obtain any user-configured caps. - // Non-fatal: if the registry read fails we proceed uncapped. - let (src_max_items, src_sync_depth_days) = { - let registry_sources = - crate::sources::list_enabled_by_kind(crate::sources::SourceKind::Composio) - .await - .unwrap_or_default(); - registry_sources - .iter() - .find(|s| s.connection_id.as_deref() == Some(&target.connection_id)) - .map(|s| (s.max_items, s.sync_depth_days)) - .unwrap_or((None, None)) - }; - - tracing::debug!( - connection_id = %target.connection_id, - max_items = ?src_max_items, - sync_depth_days = ?src_sync_depth_days, - "[composio:sync] run_connection_sync: caps from registry" - ); - - let _ = provider; - let started_at_ms = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64; - match crate::sync::pipelines::host::run_composio_connection( - &target.toolkit, - &target.connection_id, - &*config, - src_max_items, - src_sync_depth_days, - ) - .await - { - Ok(outcome) => { - let usage = ComposioUsage { - actions_called: outcome.actions_called, - cost_usd: outcome.provider_cost_usd, - }; - Ok(( - SyncOutcome { - toolkit: target.toolkit, - connection_id: Some(target.connection_id), - reason: reason.as_str().to_string(), - items_ingested: outcome.records_ingested as usize, - started_at_ms, - finished_at_ms: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64, - summary: outcome.note.unwrap_or_else(|| "sync completed".to_string()), - details: serde_json::json!({ "more_pending": outcome.more_pending }), - }, - usage, - )) - } - Err(error) => Err(( - error.to_string(), - ComposioUsage { - actions_called: error.actions_called, - cost_usd: error.provider_cost_usd, - }, - )), - } -} - -fn connection_to_sync_target(connection: ComposioConnection) -> Option { - if !connection.is_active() { - return None; - } - let toolkit = connection.normalized_toolkit(); - get_composio_sync_provider(&toolkit).map(|_| SyncTarget { - toolkit, - connection_id: connection.id, - }) -} diff --git a/crates/tinymemory-core/src/sync/composio/periodic.rs b/crates/tinymemory-core/src/sync/composio/periodic.rs deleted file mode 100644 index f0c8c4ed..00000000 --- a/crates/tinymemory-core/src/sync/composio/periodic.rs +++ /dev/null @@ -1,1274 +0,0 @@ -//! Periodic sync scheduler for the Composio domain. -//! -//! Spawned once at startup. The scheduler walks every active Composio -//! connection on a fixed tick, looks up the matching native provider, -//! and dispatches the matching tinycortex pipeline if enough time -//! has elapsed since that connection's last sync (per the provider's -//! `sync_interval_secs`). -//! -//! ## Direct mode (`[composio-direct]`) -//! -//! As of #1710 Wave 1, the scheduler is **mode-aware**: it resolves the -//! client via `create_composio_client` each tick so a direct-mode -//! user's personal Composio v3 tenant gets walked (via -//! `direct_list_connections`) instead of returning an empty list from -//! the tinyhumans tenant. The per-connection sync calls go through -//! `ProviderContext::execute` which is itself mode-aware. -//! -//! Real-time trigger webhooks (`composio:trigger` socket.io events -//! fanned out from `wss://api.tinyhumans.ai`) still do not reach the -//! core when `config.composio().mode == "direct"`, because the backend -//! HMAC-verifies the Composio webhook and pushes it down a per-user -//! socket — direct-mode users see synchronous tool execution and -//! periodic poll-based sync, but not async trigger pushes in this -//! release. See the `composio.direct_mode_triggers_gap` capability -//! entry in `about_app/catalog.rs` for the user-visible status. -//! -//! Design notes: -//! -//! * One global tick (5min) drives every provider — we don't spawn a -//! task per connection, because the number of connections per user -//! is small and a single tick keeps the bookkeeping trivial. -//! * Per-connection state (last sync timestamp) lives in a -//! process-global `Arc>` keyed by `(toolkit, -//! connection_id)`. The map is shared with event-driven sync paths -//! (bus subscribers, `on_connection_created`) via -//! [`record_sync_success`] so a recent non-periodic sync prevents -//! the scheduler from redundantly re-firing. The map is rebuilt on -//! restart; to keep a user-configured cadence (e.g. "Sync every 24h", -//! #3302) from re-firing on every cold start, the due-check falls back -//! to the **persisted** sync-audit timestamp (`read_audit_log`) when -//! the in-memory record is absent — see `persisted_since_last_sync`. -//! * Errors are logged and swallowed; the scheduler must never panic -//! out of its loop or periodic sync stops silently for the rest of -//! the process lifetime. - -use std::collections::HashMap; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, OnceLock}; -use std::time::{Duration, Instant}; - -use tokio::time::interval; - -use crate::config_loader as config_rpc; -use crate::scheduler_gate::PauseReason; -use crate::scheduler_gate::{current_policy, resume_notify}; -use crate::sources::{memory_sync_defaults_for_toolkit, MemorySourceEntry, SourceKind}; -use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; - -use super::providers::{get_provider, ComposioUsage}; -use crate::composio_host; -use crate::sync::audit::{append_audit_entry, read_audit_log, SyncAuditEntry}; -use chrono::{DateTime, Utc}; - -/// How often the scheduler wakes up to look for due syncs. Independent -/// from per-provider `sync_interval_secs` — this just bounds how long -/// past a provider's interval we might fire. -/// -/// 20 min trades a little staleness for noticeably less foreground load: -/// each tick triggers an HTTP fetch + DB write per due connection, and -/// for users with several connected providers the old 60s cadence kept -/// the laptop visibly busy. Per-provider `sync_interval_secs` still -/// caps the *minimum* delay between actual syncs — this only loosens -/// the upper bound. -const TICK_SECONDS: u64 = 1200; - -/// Process-wide guard so the scheduler is only started once even -/// when both `start_channels` and `bootstrap_core_runtime` call into -/// us during startup. Without this we'd end up with two parallel tick -/// loops competing for the same connections. -static SCHEDULER_STARTED: OnceLock<()> = OnceLock::new(); - -/// Process-wide map of `(toolkit, connection_id) → last successful sync -/// instant`. Shared between the periodic scheduler loop and event-driven -/// sync paths (e.g. `ComposioConnectionCreatedSubscriber`, -/// `on_connection_created`) so that a recent non-periodic sync prevents -/// the scheduler from firing immediately on the next tick. -type SyncTimestampMap = Arc>>; - -static LAST_SYNC_AT: OnceLock = OnceLock::new(); - -/// Get (or lazily initialise) the shared last-sync-at map. -fn last_sync_map() -> SyncTimestampMap { - LAST_SYNC_AT - .get_or_init(|| Arc::new(Mutex::new(HashMap::new()))) - .clone() -} - -/// Record a successful sync for the given `(toolkit, connection_id)` key. -/// Called by the periodic scheduler after a successful sync and by -/// event-driven paths (bus subscribers, `on_connection_created`) so the -/// periodic ticker respects recent non-periodic syncs. -pub fn record_sync_success(toolkit: &str, connection_id: &str) { - if let Ok(mut map) = last_sync_map().lock() { - map.insert( - (toolkit.to_string(), connection_id.to_string()), - Instant::now(), - ); - } -} - -/// Resolve the effective periodic sync interval (seconds) for one connection, -/// combining the provider's own default with the user's global -/// memory-sync cadence ([`Config::memory_sync_interval_secs`], #3302). -/// -/// - `global == Some(0)` → `None`: "Manual only" — the scheduler skips this -/// source entirely (manual sync still works). -/// - `global == Some(n)` → `Some(max(n, provider_default))`: the user's -/// cadence overrides the provider default but is floored at it, so we never -/// sync *more* often than the provider intended. -/// - `global == None` → `Some(max(DEFAULT, provider_default))`: no explicit -/// user choice, so fall back to the 24h default cadence (also floored at the -/// provider default). -pub(crate) fn effective_interval_secs(provider_default: u64, global: Option) -> Option { - match global { - Some(0) => None, - Some(n) => Some(n.max(provider_default)), - None => Some(DEFAULT_MEMORY_SYNC_INTERVAL_SECS.max(provider_default)), - } -} - -/// Decide whether a connection is due for a periodic sync right now, given the -/// effective interval and how long ago it last synced this run. -/// -/// `since_last_sync == None` means we have no record of a sync this process -/// lifetime, so we fire immediately (the restart-recovery path). Kept pure so -/// the due-check can be simulated without driving the real `Instant` clock. -pub(crate) fn connection_is_due(interval_secs: u64, since_last_sync: Option) -> bool { - match since_last_sync { - Some(elapsed) => elapsed >= Duration::from_secs(interval_secs), - None => true, - } -} - -/// Build an index of `connection_id → most recent successful Composio sync -/// timestamp` from the persisted sync audit log (#3302). -/// -/// The periodic loop writes audit entries with `source_id = connection_id` and -/// `scope = "{toolkit}:{connection_id}"`; we accept either shape and key by the -/// connection id. Only successful syncs count — matching the in-memory -/// [`record_sync_success`] semantics, which never records a failed tick so the -/// next tick retries. This is the wall-clock record that lets the cadence -/// survive restarts (the in-memory monotonic map cannot). -fn index_last_success_by_connection(entries: &[SyncAuditEntry]) -> HashMap> { - let mut idx: HashMap> = HashMap::new(); - for e in entries { - if e.source_kind != "composio" || !e.success { - continue; - } - let connection_id = e - .scope - .rsplit_once(':') - .map(|(_, c)| c.to_string()) - .filter(|c| !c.is_empty()) - .unwrap_or_else(|| e.source_id.clone()); - idx.entry(connection_id) - .and_modify(|t| { - if e.timestamp > *t { - *t = e.timestamp; - } - }) - .or_insert(e.timestamp); - } - idx -} - -/// Wall-clock elapsed since a connection's last persisted successful sync, if -/// any. Saturates at zero for a future timestamp (clock skew), so a skewed -/// record never reads as "wildly overdue". Returns `None` when the connection -/// has no persisted sync — letting the caller treat it as never-synced. -fn persisted_since_last_sync( - idx: &HashMap>, - connection_id: &str, - now: DateTime, -) -> Option { - idx.get(connection_id).map(|ts| { - let secs = (now - *ts).num_seconds().max(0) as u64; - Duration::from_secs(secs) - }) -} - -/// Outcome of consulting the per-source registry for one Composio connection -/// during a periodic tick (#2831). -#[derive(Debug, PartialEq, Eq)] -enum PeriodicSourceDecision { - /// The user toggled this source **off** — skip background sync entirely. - /// Manual `memory_sources_sync` still works (it has its own `enabled` - /// guard); only the automatic loop honours this here. - Skip, - /// Sync this connection with the given caps (`None` = uncapped for that - /// dimension). - Sync { - max_items: Option, - sync_depth_days: Option, - }, -} - -/// Decide whether — and with what caps — to periodically sync one connection, -/// honouring the per-source `enabled` toggle (#2831). Pure so the three -/// branches can be unit-tested without async/registry I/O. -/// -/// - **disabled row** → [`PeriodicSourceDecision::Skip`]: the background loop -/// must not sync a source the user switched off (this was the row-2 leak — -/// previously the loop only read the registry for caps and synced disabled -/// sources anyway, uncapped). -/// - **enabled row** → `Sync` with the row's caps. -/// - **no row yet** → `Sync` with conservative per-toolkit defaults -/// ([`memory_sync_defaults_for_toolkit`]). `reconcile` normally backfills an -/// enabled, capped row for every connection; this covers the brief -/// pre-reconcile window. The no-match path defaults to *sync-bounded*, never -/// *skip* and never *uncapped*, so a missing/mismatched row degrades safely -/// (data keeps flowing, just capped) instead of silently going dark. -fn decide_periodic_source( - source: Option<&MemorySourceEntry>, - toolkit: &str, -) -> PeriodicSourceDecision { - match source { - Some(s) if !s.enabled => PeriodicSourceDecision::Skip, - Some(s) => PeriodicSourceDecision::Sync { - max_items: s.max_items, - sync_depth_days: s.sync_depth_days, - }, - None => { - let (max_items, sync_depth_days) = memory_sync_defaults_for_toolkit(toolkit); - PeriodicSourceDecision::Sync { - max_items, - sync_depth_days, - } - } - } -} - -/// Spawn the periodic sync background task. Idempotent: only the -/// first call actually spawns the loop, every subsequent call is a -/// cheap no-op (logged at `debug` so it's visible during startup -/// tracing without spamming `info`). -pub fn start_periodic_sync() { - if SCHEDULER_STARTED.get().is_some() { - tracing::debug!("[composio:periodic] scheduler already running, skipping start"); - return; - } - // Race-safe: only the thread that wins `set` runs the spawn body. - if SCHEDULER_STARTED.set(()).is_err() { - tracing::debug!("[composio:periodic] scheduler already running (race), skipping start"); - return; - } - - tokio::spawn(async move { - tracing::info!( - tick_seconds = TICK_SECONDS, - "[composio:periodic] scheduler starting" - ); - run_loop().await; - // run_loop only returns on a fatal error in the bus — log it - // so the silent stop is at least visible in the trace. - tracing::error!("[composio:periodic] scheduler loop exited"); - }); -} - -/// Inner loop, broken out so it's easy to mock-replace in tests if we -/// ever want to drive ticks deterministically. -/// -/// Each iteration waits on whichever comes first (#2831): -/// * the 20-min `ticker` — the steady-state cadence, or -/// * the scheduler-gate **resume** notify — fired when the user toggles -/// Memory Tree back on or signs back in. -/// -/// On a resume wake we run a tick **immediately** (so sync restarts within -/// seconds, not at the next ≤20-min boundary) and `reset()` the ticker so the -/// *next* scheduled tick is a full `TICK_SECONDS` out. The reset is what stops -/// rapid off-on toggling from bunch-firing: many wakes collapse into at most -/// one extra tick (the `Notify` stores a single permit), and the cadence -/// re-bases from the last actual tick. -async fn run_loop() { - let mut ticker = interval(Duration::from_secs(TICK_SECONDS)); - let resume = resume_notify(); - // Skip the immediate-fire tick so startup isn't slammed before the - // user even has time to sign in. - ticker.tick().await; - - loop { - tokio::select! { - _ = ticker.tick() => {} - _ = resume.notified() => { - // Woke early on a resume transition. Re-base the cadence so the - // next scheduled tick is TICK_SECONDS from now, then fall - // through and run the tick immediately. - ticker.reset(); - } - } - if let Err(e) = run_one_tick().await { - tracing::warn!( - error = %e, - "[composio:periodic] tick failed (continuing)" - ); - } - } -} - -/// Inspect the scheduler-gate policy and decide whether this tick should -/// fire at all. Returns `Some(reason)` for paused states so the caller can -/// log a single, attributable line instead of doing the work and discovering -/// per-LLM-call later that everything's gated. -/// -/// Covers two reasons the memory subsystem treats as "do no background -/// work": -/// - [`PauseReason::UserDisabled`] — user flipped the Memory Tree toggle off -/// in Settings (#1856 Part 1). The 20-min Composio fetch loop honouring -/// this flag is the explicit follow-up listed in the #2719 PR body. -/// - [`PauseReason::SignedOut`] — no live session; periodic work would just -/// 401-loop against the backend. -/// -/// Other [`PauseReason`] variants: -/// - `OnBattery` / `CpuPressure` (future, per #1073) — intentionally **not** -/// gated here; periodic Composio fetch is network-light, so battery / CPU -/// pressure shouldn't stop the user's data flowing in. Those signals -/// already throttle LLM-bound work through the regular gate. -/// - `Unknown` — documented in `scheduler_gate::policy` as a safe fallback; -/// `Policy::pause_reason()` returns it only when the gate state is in a -/// transitional / not-yet-resolved condition. Letting the tick proceed -/// here keeps periodic sync running through brief transitions instead of -/// pausing on stale unresolved state. -pub(crate) fn periodic_pause_reason() -> Option { - // Delegate the `Policy::Paused { .. }` → `PauseReason` extraction to - // the existing `Policy::pause_reason()` helper (avoids re-implementing - // the same destructure twice). The allow-list below is the only thing - // this site has to own — future `PauseReason` variants stay opt-in. - let reason = current_policy().pause_reason()?; - matches!(reason, PauseReason::UserDisabled | PauseReason::SignedOut).then_some(reason) -} - -/// Process-level "was the last tick paused?" tracker for transition logging. -/// -/// We want `info!` *once* when the periodic loop crosses the pause boundary -/// (so fleet operators investigating "why is Composio not syncing?" see a -/// breadcrumb at default log level), without spamming `info` every 20 min -/// while the user has the toggle off. `Relaxed` ordering is fine because -/// the only consumer is the inside of `run_one_tick`, which is serialised -/// by the singleton scheduler loop. -static LAST_TICK_WAS_PAUSED: AtomicBool = AtomicBool::new(false); - -/// Run a single scheduler tick. Public-ish (`pub(crate)`) so the test -/// module can drive ticks without spinning up the real `interval`. -pub(crate) async fn run_one_tick() -> Result<(), String> { - // Step 0: scheduler-gate check. When the user has paused Memory Tree - // via the Settings toggle, every subsequent tick should be a cheap - // no-op — no `list_connections` call, no provider walk, no API budget - // burn. The check runs **before** config load + auth-client build so - // a paused session never even resolves the API token. - // - // Transition logging: emit `info!` once when the loop crosses the - // pause boundary in either direction; stay at `debug!` for the - // already-paused / already-running steady state. Without this, fleet - // operators investigating "why is Composio not syncing?" see nothing - // at default log level. - if let Some(reason) = periodic_pause_reason() { - let was_paused = LAST_TICK_WAS_PAUSED.swap(true, Ordering::Relaxed); - if was_paused { - tracing::debug!( - reason = reason.as_str(), - "[composio:periodic] scheduler-gate paused — skipping tick" - ); - } else { - tracing::info!( - reason = reason.as_str(), - "[composio:periodic] scheduler-gate paused — pausing periodic Composio sync" - ); - } - return Ok(()); - } else { - let was_paused = LAST_TICK_WAS_PAUSED.swap(false, Ordering::Relaxed); - if was_paused { - tracing::info!( - "[composio:periodic] scheduler-gate resumed — periodic Composio sync re-enabled" - ); - } - } - - // Step 1: load config (also gives us the auth token via the - // shared integrations client builder). - let config = config_rpc::load_config_with_timeout() - .await - .map_err(|e| format!("load_config: {e}"))?; - - // Step 2: list active connections — mode-aware. Backend mode walks - // the tinyhumans tenant; direct mode walks the user's personal - // Composio v3 tenant. Mirrors `ops::composio_list_connections` so - // direct-mode users get periodic sync against their own connections - // instead of seeing an empty list (#1710). - // Mode dispatch lives in the host's `ComposioHost` impl, and so does the - // 401 classification that used to happen here: the direct-mode v3 - // `/connected_accounts` 401 shape is a property of the client, not of this - // loop, and the host reports it through the same observability classifier - // the UI poll uses. A failure here means "not signed in / no direct key" as - // often as it means a real fault, so the tick skips rather than erroring. - let connections = match composio_host::list_connections(&*config).await { - Ok(connections) => connections, - Err(e) => { - tracing::debug!( - error = %e, - "[composio:periodic] no connections (not signed in? no direct key?), skipping tick" - ); - return Ok(()); - } - }; - - let sync_map = last_sync_map(); - - // Global, user-configurable memory-sync cadence (#3302). Applied to every - // opted-in source as a floor/override over the provider's own default; a - // value of `Some(0)` disables periodic auto-sync ("Manual only"). - let global_interval = config.memory_sync_interval_secs(); - - // Persisted last-sync fallback (#3302). The in-memory `LAST_SYNC_AT` map is - // rebuilt empty on every launch, so without this a cold start would re-fire - // every connection on the first tick — silently breaking the configured - // "Sync every 24h" gap across app restarts. We index the persisted sync - // audit log (wall-clock timestamps that survive restarts) and use it as the - // due-check fallback whenever the in-memory monotonic record is absent. - let (audit_index, audit_available) = - composio_audit_state(read_audit_log(config.workspace_dir())); - if !audit_available { - tracing::warn!( - "[memory_sync:periodic] audit unavailable; sources without in-memory cadence will be skipped" - ); - } - let now = Utc::now(); - - // Per-source registry snapshot (#2831). The periodic loop gates on the - // per-source `enabled` toggle so a source the user switched off stops - // syncing in the background — matching the manual paths - // (`memory_sources::sync_source`, `memory_sources_sync_all`), which already - // early-return on `!enabled`. Index every Composio source (enabled and - // disabled) by connection id; the per-connection branch below resolves - // skip/caps via `decide_periodic_source`. - // - // Built from the **already-loaded** `config` snapshot (Step 1), not a second - // `list_sources()` read. A separate read whose error we swallowed to an - // empty map would make every disabled source fall through to the - // `decide_periodic_source(None, ..)` default-caps path — silently - // re-enabling background sync for sources the user switched off on a - // transient config-read failure. Reusing the tick's snapshot is fail-closed - // (a disabled row stays disabled) and avoids the extra read entirely. - let composio_sources: HashMap = - crate::sources::decode_memory_sources(&*config) - .iter() - .filter(|s| s.kind == SourceKind::Composio) - .filter_map(|s| s.connection_id.clone().map(|id| (id, s.clone()))) - .collect(); - - let mut considered = 0usize; - let mut fired = 0usize; - for conn in connections { - considered += 1; - - // Skip connections that aren't actually live yet. - if !conn.is_active() { - continue; - } - - let toolkit = conn.normalized_toolkit(); - let Some(provider) = get_provider(&toolkit) else { - // No provider registered for this toolkit — that's fine, - // we just don't have native code for it. Tools still work - // through `composio_execute`. - continue; - }; - - let Some(provider_default) = provider.sync_interval_secs() else { - // Provider opted out of periodic sync entirely. - continue; - }; - - let Some(interval_secs) = effective_interval_secs(provider_default, global_interval) else { - // User selected "Manual only" — skip auto-sync for this source. - // Manual `memory_sources_sync` still works. - tracing::debug!( - toolkit = %toolkit, - connection_id = %conn.id, - "[composio:periodic] manual-only mode — skipping periodic sync" - ); - continue; - }; - - let key = (toolkit.clone(), conn.id.clone()); - // Prefer the in-memory monotonic record (most accurate within this run); - // fall back to the persisted audit timestamp so the configured cadence - // is honoured across restarts instead of re-firing on every cold start. - let in_memory_since = { - let map = sync_map.lock().unwrap_or_else(|e| e.into_inner()); - map.get(&key).map(|when| when.elapsed()) - }; - let Some(since_last_sync) = cadence_from_audit( - in_memory_since, - audit_available, - persisted_since_last_sync(&audit_index, &conn.id, now), - ) else { - tracing::debug!( - toolkit = %toolkit, - "[composio:periodic] source has unknown cadence while audit is unavailable; skipping" - ); - continue; - }; - if !connection_is_due(interval_secs, since_last_sync) { - continue; - } - - // Per-source gate + caps from the memory_sources registry (#2831). - // A disabled source is skipped here (the background-sync half of the - // toggle); enabled sources sync with their caps; a connection with no - // registry row yet syncs with conservative per-toolkit defaults. - let (src_max_items, src_sync_depth_days) = - match decide_periodic_source(composio_sources.get(&conn.id), &toolkit) { - PeriodicSourceDecision::Skip => { - tracing::debug!( - toolkit = %toolkit, - connection_id = %conn.id, - "[composio:periodic] source disabled — skipping periodic sync" - ); - continue; - } - PeriodicSourceDecision::Sync { - max_items, - sync_depth_days, - } => (max_items, sync_depth_days), - }; - - tracing::debug!( - toolkit = %toolkit, - connection_id = %conn.id, - max_items = ?src_max_items, - sync_depth_days = ?src_sync_depth_days, - "[composio:periodic] caps from registry" - ); - - let mut source = composio_sources - .get(&conn.id) - .cloned() - .unwrap_or_else(|| periodic_source(&toolkit, &conn.id)); - source.max_items = src_max_items; - source.sync_depth_days = src_sync_depth_days; - - tracing::debug!( - toolkit = %conn.toolkit, - connection_id = %conn.id, - interval_secs, - "[composio:periodic] firing sync" - ); - let sync_started = Instant::now(); - let result = crate::sync::pipelines::host::run_composio_connection_with_caps( - &toolkit, - &conn.id, - &*config, - crate::sync::pipelines::host::SourceCaps::from_source(&source), - ) - .await; - let duration_ms = sync_started.elapsed().as_millis() as u64; - - match result { - Ok(outcome) => { - let usage = ComposioUsage { - actions_called: outcome.actions_called, - cost_usd: outcome.provider_cost_usd, - }; - tracing::debug!( - toolkit = %conn.toolkit, - connection_id = %conn.id, - items = outcome.records_ingested, - composio_actions = usage.actions_called, - "[composio:periodic] sync ok" - ); - let entry = build_periodic_audit_entry( - &toolkit, - &conn.id, - &usage, - outcome.records_ingested as usize, - duration_ms, - None, - ); - if let Err(error) = append_audit_entry(config.workspace_dir(), &entry) { - tracing::warn!(%error, "[memory_sync:audit] append failed"); - } - record_sync_success(&conn.toolkit, &conn.id); - fired += 1; - } - Err(e) => { - let usage = ComposioUsage { - actions_called: e.actions_called, - cost_usd: e.provider_cost_usd, - }; - tracing::warn!( - toolkit = %conn.toolkit, - connection_id = %conn.id, - error = %e, - "[composio:periodic] sync failed (will retry next tick)" - ); - // A failed tick may still have fired billable fetch actions - // before erroring — audit the partial cost so it isn't lost. - let entry = build_periodic_audit_entry( - &toolkit, - &conn.id, - &usage, - 0, - duration_ms, - Some(e.to_string()), - ); - if let Err(error) = append_audit_entry(config.workspace_dir(), &entry) { - tracing::warn!(%error, "[memory_sync:audit] append failed"); - } - // Intentionally do NOT update last_sync_at on failure - // so the next tick retries immediately. - } - } - } - - tracing::debug!(considered, fired, "[composio:periodic] tick complete"); - Ok(()) -} - -fn composio_audit_state( - read: anyhow::Result>, -) -> (HashMap>, bool) { - match read { - Ok(entries) => (index_last_success_by_connection(&entries), true), - Err(error) => { - tracing::warn!(%error, "[memory_sync:periodic] audit read failed"); - (HashMap::new(), false) - } - } -} - -fn cadence_from_audit( - in_memory_since: Option, - audit_available: bool, - persisted_since: Option, -) -> Option> { - match in_memory_since { - Some(since) => Some(Some(since)), - None if audit_available => Some(persisted_since), - None => None, - } -} - -fn periodic_source(toolkit: &str, connection_id: &str) -> MemorySourceEntry { - MemorySourceEntry { - id: format!("composio:{connection_id}"), - kind: SourceKind::Composio, - label: toolkit.to_string(), - enabled: true, - toolkit: Some(toolkit.to_string()), - connection_id: Some(connection_id.to_string()), - path: None, - glob: None, - url: None, - branch: None, - paths: Vec::new(), - max_commits: None, - max_issues: None, - max_prs: None, - query: None, - since_days: None, - max_items: None, - selector: None, - max_tokens_per_sync: None, - max_cost_per_sync_usd: None, - sync_depth_days: None, - } -} - -/// Build a [`SyncAuditEntry`] for one periodic Composio sync tick (#3111 -/// follow-up). -/// -/// Periodic syncs only fetch + ingest; summarisation runs later in the async -/// job worker, so the LLM-cost columns (tokens, estimated / actual charge) -/// are zero here. The meaningful spend is the Composio billable actions the -/// fetch fired, carried in `usage`. `scope` is `{toolkit}:{connection_id}` to -/// match the owner shape the per-source memory-tree ingest uses, and -/// `source_kind` is `"composio"` so the Sync History panel groups periodic -/// rows alongside the manual-sync rows the dispatcher already writes. -fn build_periodic_audit_entry( - toolkit: &str, - connection_id: &str, - usage: &ComposioUsage, - items_ingested: usize, - duration_ms: u64, - error: Option, -) -> SyncAuditEntry { - SyncAuditEntry { - timestamp: chrono::Utc::now(), - source_id: connection_id.to_string(), - source_kind: "composio".to_string(), - scope: format!("{toolkit}:{connection_id}"), - items_fetched: items_ingested as u32, - batches: 0, - input_tokens: 0, - output_tokens: 0, - estimated_cost_usd: 0.0, - composio_actions_called: usage.actions_called, - composio_cost_usd: usage.cost_usd, - actual_charged_usd: None, - duration_ms, - success: error.is_none(), - error, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_env_lock::TEST_ENV_LOCK as ENV_LOCK; - use tempfile::tempdir; - - #[test] - fn tick_seconds_is_sane_default() { - // Sanity check: don't accidentally ship a 1-second tick. - const _: () = assert!(TICK_SECONDS >= 30); - const _: () = assert!(TICK_SECONDS <= 3600); - } - - #[test] - fn effective_interval_none_falls_back_to_default() { - // No user choice → 24h default, floored at the provider default. - assert_eq!( - effective_interval_secs(15 * 60, None), - Some(DEFAULT_MEMORY_SYNC_INTERVAL_SECS) - ); - } - - #[test] - fn effective_interval_manual_disables_sync() { - // Some(0) is the "Manual only" sentinel — periodic sync is skipped. - assert_eq!(effective_interval_secs(15 * 60, Some(0)), None); - } - - #[test] - fn effective_interval_override_is_floored_at_provider_default() { - // A user cadence longer than the provider default is honoured as-is. - assert_eq!( - effective_interval_secs(15 * 60, Some(4 * 3600)), - Some(4 * 3600) - ); - // A user cadence shorter than the provider default is clamped up to it - // so we never sync more often than the provider intends. - assert_eq!(effective_interval_secs(30 * 60, Some(60)), Some(30 * 60)); - // Exactly equal stays equal. - assert_eq!(effective_interval_secs(1800, Some(1800)), Some(1800)); - } - - #[test] - fn effective_interval_default_is_floored_at_a_longer_provider_default() { - // If a provider ever defaults to longer than 24h, that wins under None. - let long = DEFAULT_MEMORY_SYNC_INTERVAL_SECS + 3600; - assert_eq!(effective_interval_secs(long, None), Some(long)); - } - - #[test] - fn connection_is_due_compares_elapsed_against_interval() { - let interval = 4 * 3600; - // Never synced this run → always due. - assert!(connection_is_due(interval, None)); - // Synced more recently than the interval → not due. - assert!(!connection_is_due( - interval, - Some(Duration::from_secs(3600)) - )); - // Synced exactly at the interval boundary → due. - assert!(connection_is_due( - interval, - Some(Duration::from_secs(interval)) - )); - // Synced longer ago than the interval → due. - assert!(connection_is_due( - interval, - Some(Duration::from_secs(interval + 1)) - )); - } - - /// Build a minimal Composio `MemorySourceEntry` for the per-source gate - /// tests — only the fields `decide_periodic_source` reads are meaningful. - fn composio_source( - enabled: bool, - max_items: Option, - sync_depth_days: Option, - ) -> MemorySourceEntry { - MemorySourceEntry { - id: "src_test".to_string(), - kind: SourceKind::Composio, - label: "test".to_string(), - enabled, - toolkit: Some("gmail".to_string()), - connection_id: Some("cmp-1".to_string()), - path: None, - glob: None, - url: None, - branch: None, - paths: Vec::new(), - max_commits: None, - max_issues: None, - max_prs: None, - query: None, - since_days: None, - max_items, - selector: None, - max_tokens_per_sync: None, - max_cost_per_sync_usd: None, - sync_depth_days, - } - } - - /// #2831 row 2: a source explicitly toggled **off** must be skipped by the - /// background loop — this is the leak the gate closes. - #[test] - fn decide_periodic_source_skips_disabled_source() { - let src = composio_source(false, Some(100), Some(30)); - assert_eq!( - decide_periodic_source(Some(&src), "gmail"), - PeriodicSourceDecision::Skip - ); - } - - /// An enabled source syncs with exactly its configured caps (no defaulting). - #[test] - fn decide_periodic_source_uses_enabled_source_caps() { - let src = composio_source(true, Some(42), Some(7)); - assert_eq!( - decide_periodic_source(Some(&src), "gmail"), - PeriodicSourceDecision::Sync { - max_items: Some(42), - sync_depth_days: Some(7), - } - ); - } - - /// A connection with no registry row yet (pre-reconcile window) syncs with - /// the conservative per-toolkit defaults — **bounded**, never uncapped, and - /// never skipped. This is the safe-direction fallback for a missing match. - #[test] - fn decide_periodic_source_defaults_caps_when_no_row() { - let (want_items, want_depth) = memory_sync_defaults_for_toolkit("gmail"); - assert_eq!( - decide_periodic_source(None, "gmail"), - PeriodicSourceDecision::Sync { - max_items: want_items, - sync_depth_days: want_depth, - } - ); - // The defaults are bounded for a known toolkit (regression guard against - // an accidental return to uncapped background fetches). - assert!(want_items.is_some()); - } - - /// Multi-account regression (#3443 added multiple account connections per - /// toolkit): two live connections of the *same* toolkit must be gated - /// **independently** by their own per-`connection_id` source rows. This - /// pins the loop's connection-id keying — re-keying the lookup by toolkit - /// would collapse the two accounts and is the regression this guards. - #[test] - fn per_connection_gate_is_independent_across_accounts_of_same_toolkit() { - // gmail account A: enabled with caps; gmail account B: disabled. - let mut a = composio_source(true, Some(10), Some(5)); - a.connection_id = Some("conn-A".to_string()); - a.toolkit = Some("gmail".to_string()); - let mut b = composio_source(false, Some(99), Some(99)); - b.connection_id = Some("conn-B".to_string()); - b.toolkit = Some("gmail".to_string()); - - // Build the same connection_id → entry index the live tick builds. - let index: HashMap = [a, b] - .into_iter() - .filter_map(|s| s.connection_id.clone().map(|id| (id, s))) - .collect(); - - // Account A (enabled) syncs with its own caps... - assert_eq!( - decide_periodic_source(index.get("conn-A"), "gmail"), - PeriodicSourceDecision::Sync { - max_items: Some(10), - sync_depth_days: Some(5), - } - ); - // ...account B (disabled) is skipped, even though it shares the toolkit. - assert_eq!( - decide_periodic_source(index.get("conn-B"), "gmail"), - PeriodicSourceDecision::Skip - ); - // A third, not-yet-registered account of the same toolkit falls back to - // bounded defaults (never skipped, never uncapped). - let (def_items, def_depth) = memory_sync_defaults_for_toolkit("gmail"); - assert_eq!( - decide_periodic_source(index.get("conn-C"), "gmail"), - PeriodicSourceDecision::Sync { - max_items: def_items, - sync_depth_days: def_depth, - } - ); - } - - /// End-to-end simulation of the scheduler's per-connection decision: prove - /// that **changing the global setting changes when the next sync fires** - /// (issue #3302 acceptance criterion). We drive the same two pure helpers - /// the live tick uses (`effective_interval_secs` → `connection_is_due`) - /// across realistic last-sync ages, so no clock or network is needed. - #[test] - fn scheduler_decision_honors_the_global_setting() { - // A chatty provider that natively wants to sync every 15 minutes. - let provider_default = 15 * 60; - - // Helper mirroring the live loop: returns whether the connection would - // fire right now, or `None` for "Manual only" (skipped entirely). - let decide = |global: Option, since: Option| -> Option { - effective_interval_secs(provider_default, global) - .map(|interval| connection_is_due(interval, since)) - }; - - let one_hour_ago = Some(Duration::from_secs(3600)); - let five_hours_ago = Some(Duration::from_secs(5 * 3600)); - - // Baseline (no global override): with only the 15m provider default, a - // connection synced an hour ago is already overdue and WOULD fire. - // (This is the behavior the feature is reining in.) - assert!(connection_is_due(provider_default, one_hour_ago)); - - // User picks "every 4h": now that same hour-old connection must NOT - // fire — the global cadence (not the 15m default) governs the gap… - assert_eq!(decide(Some(4 * 3600), one_hour_ago), Some(false)); - // …but once 5h have passed it fires again. - assert_eq!(decide(Some(4 * 3600), five_hours_ago), Some(true)); - - // User picks "Manual only" (0): never auto-fires, no matter how stale. - assert_eq!(decide(Some(0), five_hours_ago), None); - assert_eq!(decide(Some(0), None), None); - - // Unset (None) → 24h default: the hour-old connection is not yet due, - // confirming the default is far more conservative than the 15m native - // cadence. - assert_eq!(decide(None, one_hour_ago), Some(false)); - assert_eq!( - decide(None, Some(Duration::from_secs(25 * 3600))), - Some(true) - ); - - // A never-synced connection fires on any non-manual setting (the - // restart-recovery path). - assert_eq!(decide(Some(4 * 3600), None), Some(true)); - } - - fn audit_entry( - connection_id: &str, - scope: &str, - success: bool, - ts: DateTime, - ) -> SyncAuditEntry { - SyncAuditEntry { - timestamp: ts, - source_id: connection_id.to_string(), - source_kind: "composio".to_string(), - scope: scope.to_string(), - items_fetched: 1, - batches: 0, - input_tokens: 0, - output_tokens: 0, - estimated_cost_usd: 0.0, - composio_actions_called: 1, - composio_cost_usd: 0.0, - actual_charged_usd: None, - duration_ms: 10, - success, - error: None, - } - } - - #[test] - fn index_last_success_keeps_latest_success_and_ignores_failures() { - let now = Utc::now(); - let older = now - chrono::Duration::hours(6); - let newer = now - chrono::Duration::hours(1); - let entries = vec![ - audit_entry("cmp-1", "gmail:cmp-1", true, older), - audit_entry("cmp-1", "gmail:cmp-1", true, newer), // newer success wins - audit_entry("cmp-1", "gmail:cmp-1", false, now), // failure ignored - audit_entry("cmp-2", "slack:cmp-2", false, now), // only-failure → absent - ]; - let idx = index_last_success_by_connection(&entries); - assert_eq!(idx.get("cmp-1"), Some(&newer)); - assert!( - !idx.contains_key("cmp-2"), - "a connection with only failed syncs is not indexed" - ); - } - - #[test] - fn index_last_success_falls_back_to_source_id_without_scope_suffix() { - let now = Utc::now(); - // A non-composio kind is skipped entirely. - let entries = vec![ - SyncAuditEntry { - source_kind: "github_repo".to_string(), - ..audit_entry("ignored", "github:org/repo", true, now) - }, - // Composio entry whose scope has no ':' → key by source_id. - audit_entry("cmp-3", "noscope", true, now), - ]; - let idx = index_last_success_by_connection(&entries); - assert!(idx.contains_key("cmp-3")); - assert!(!idx.contains_key("ignored")); - } - - #[test] - fn persisted_since_last_sync_computes_and_saturates() { - let now = Utc::now(); - let mut idx = HashMap::new(); - idx.insert("cmp-1".to_string(), now - chrono::Duration::hours(3)); - idx.insert("future".to_string(), now + chrono::Duration::hours(2)); - - let elapsed = persisted_since_last_sync(&idx, "cmp-1", now).unwrap(); - // ~3h, allow a small window for test execution time. - assert!(elapsed >= Duration::from_secs(3 * 3600 - 5)); - assert!(elapsed <= Duration::from_secs(3 * 3600 + 5)); - // Clock skew (future timestamp) saturates to zero, not a huge value. - assert_eq!( - persisted_since_last_sync(&idx, "future", now), - Some(Duration::ZERO) - ); - // Unknown connection → None (treated as never synced). - assert_eq!(persisted_since_last_sync(&idx, "unknown", now), None); - } - - /// The cadence must survive a restart: with the in-memory map cold, the - /// persisted audit timestamp drives the due-check so a connection synced - /// 1h ago does NOT re-fire under a 4h setting, but one synced 5h ago does. - #[test] - fn cadence_survives_restart_via_persisted_audit() { - let now = Utc::now(); - let mut idx = HashMap::new(); - idx.insert("cmp-1".to_string(), now - chrono::Duration::hours(1)); - idx.insert("cmp-2".to_string(), now - chrono::Duration::hours(5)); - - let interval = effective_interval_secs(15 * 60, Some(4 * 3600)).unwrap(); - - // cmp-1 (synced 1h ago) — in-memory cold, persisted fallback says NOT due. - let cmp1 = None.or_else(|| persisted_since_last_sync(&idx, "cmp-1", now)); - assert!(!connection_is_due(interval, cmp1)); - - // cmp-2 (synced 5h ago) — persisted fallback says due. - let cmp2 = None.or_else(|| persisted_since_last_sync(&idx, "cmp-2", now)); - assert!(connection_is_due(interval, cmp2)); - - // A connection with no persisted record still fires (truly fresh). - let fresh = None.or_else(|| persisted_since_last_sync(&idx, "cmp-new", now)); - assert!(connection_is_due(interval, fresh)); - } - - #[test] - fn audit_failure_is_unavailable_and_unknown_cadence_is_skipped() { - let (index, available) = - composio_audit_state(Err(anyhow::anyhow!("simulated audit I/O failure"))); - assert!(index.is_empty()); - assert!(!available); - assert_eq!(cadence_from_audit(None, available, None), None); - - let known = Duration::from_secs(60); - assert_eq!( - cadence_from_audit(Some(known), available, None), - Some(Some(known)) - ); - } - - #[test] - fn readable_empty_audit_preserves_first_sync_behavior() { - let (index, available) = composio_audit_state(Ok(Vec::new())); - assert!(index.is_empty()); - assert!(available); - - let cadence = cadence_from_audit(None, available, None) - .expect("readable empty audit keeps the source eligible"); - assert!(connection_is_due(3600, cadence)); - } - - /// A successful periodic tick produces a Composio-kind audit entry that - /// carries the billable-action tally + cost and zeroes the LLM-cost - /// columns (summarisation happens later in the job worker). Pins the - /// shape the Sync History panel reads (#3111 follow-up). - #[test] - fn periodic_audit_entry_records_composio_cost_on_success() { - let usage = ComposioUsage { - actions_called: 3, - cost_usd: 0.042, - }; - let entry = build_periodic_audit_entry("gmail", "cmp-123", &usage, 17, 1234, None); - - assert_eq!(entry.source_kind, "composio"); - assert_eq!(entry.source_id, "cmp-123"); - assert_eq!(entry.scope, "gmail:cmp-123"); - assert_eq!(entry.items_fetched, 17); - assert_eq!(entry.composio_actions_called, 3); - assert!((entry.composio_cost_usd - 0.042).abs() < f64::EPSILON); - assert!(entry.success); - assert!(entry.error.is_none()); - // Periodic fetch does no summarisation — LLM cost columns stay zero, - // and the Composio spend is the whole combined cost. - assert_eq!(entry.input_tokens, 0); - assert_eq!(entry.estimated_cost_usd, 0.0); - assert!((entry.combined_cost_usd() - 0.042).abs() < f64::EPSILON); - } - - /// A failed periodic tick still records the partial billable cost it - /// incurred before erroring (the fetch may have fired actions), with - /// `success = false` and the error message preserved. - #[test] - fn periodic_audit_entry_preserves_partial_cost_on_failure() { - let usage = ComposioUsage { - actions_called: 1, - cost_usd: 0.01, - }; - let entry = build_periodic_audit_entry( - "notion", - "cmp-9", - &usage, - 0, - 500, - Some("fetch timed out".to_string()), - ); - - assert!(!entry.success); - assert_eq!(entry.error.as_deref(), Some("fetch timed out")); - assert_eq!(entry.items_fetched, 0); - // The billable action it managed to fire before failing is still - // recorded so cost isn't under-reported on failures. - assert_eq!(entry.composio_actions_called, 1); - assert!((entry.composio_cost_usd - 0.01).abs() < f64::EPSILON); - } - - #[test] - fn record_sync_success_stores_timestamp_keyed_by_toolkit_and_connection() { - // Use unique keys so this test doesn't collide with other tests - // writing into the process-wide map. - let toolkit = "test_periodic_toolkit_a"; - let conn = "test-conn-a"; - record_sync_success(toolkit, conn); - let map = last_sync_map(); - let guard = map.lock().expect("lock"); - let ts = guard - .get(&(toolkit.to_string(), conn.to_string())) - .expect("entry recorded"); - // Just-recorded timestamps should be very recent. - assert!(ts.elapsed() < Duration::from_secs(5)); - } - - #[test] - fn record_sync_success_overwrites_previous_timestamp() { - let toolkit = "test_periodic_toolkit_b"; - let conn = "test-conn-b"; - record_sync_success(toolkit, conn); - let first = last_sync_map() - .lock() - .expect("lock") - .get(&(toolkit.to_string(), conn.to_string())) - .copied() - .expect("first entry"); - // Second call must replace (not keep the older) timestamp. - std::thread::sleep(Duration::from_millis(5)); - record_sync_success(toolkit, conn); - let second = last_sync_map() - .lock() - .expect("lock") - .get(&(toolkit.to_string(), conn.to_string())) - .copied() - .expect("second entry"); - assert!( - second >= first, - "record_sync_success should advance the stored Instant" - ); - } - - // The `_guard` below is held deliberately across the `run_one_tick().await` - // in this test: it's a std::sync::Mutex used purely as a test-isolation - // gate around the process-global `OPENHUMAN_WORKSPACE` env var, not an - // async resource lock guarding shared runtime state. Dropping it before - // the await would let a sibling test mutate the env var mid-tick, - // defeating the isolation this guard exists to provide. - #[allow(clippy::await_holding_lock)] - #[tokio::test] - async fn run_one_tick_returns_ok_when_no_client() { - // Isolate the workspace/env so config loading doesn't contend with - // sibling tests mutating OPENHUMAN_WORKSPACE in parallel. - let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - let tmp = tempdir().expect("tempdir"); - unsafe { - std::env::set_var("OPENHUMAN_WORKSPACE", tmp.path()); - } - - // With no session stored in the isolated workspace, - // `build_composio_client` returns None and the tick should - // silently skip (returning Ok). This covers the early-return - // path that's otherwise only hit in production. - let inner = tokio::time::timeout(Duration::from_secs(5), run_one_tick()) - .await - .expect("run_one_tick should not hang indefinitely during tests"); - assert!( - inner.is_ok(), - "run_one_tick should return Ok when no client is available: {inner:?}" - ); - - unsafe { - std::env::remove_var("OPENHUMAN_WORKSPACE"); - } - } - - #[tokio::test] - async fn start_periodic_sync_is_idempotent() { - // First call installs the scheduler via the OnceLock; subsequent - // calls must be cheap no-ops without panicking. `tokio::spawn` - // needs an ambient runtime, so this test runs under `tokio::test`. - start_periodic_sync(); - start_periodic_sync(); - assert!(SCHEDULER_STARTED.get().is_some()); - } - - #[test] - fn record_sync_success_distinguishes_connections() { - let toolkit = "test_periodic_toolkit_c"; - record_sync_success(toolkit, "conn-1"); - record_sync_success(toolkit, "conn-2"); - let map = last_sync_map(); - let guard = map.lock().expect("lock"); - assert!(guard - .get(&(toolkit.to_string(), "conn-1".to_string())) - .is_some()); - assert!(guard - .get(&(toolkit.to_string(), "conn-2".to_string())) - .is_some()); - // Unrelated key should be absent. - assert!(guard - .get(&(toolkit.to_string(), "conn-3".to_string())) - .is_none()); - } - - /// In unit tests `scheduler_gate::STATE` is never initialised, so - /// `current_policy()` returns `Policy::Normal` and the helper must - /// return `None` — i.e. the tick is allowed to proceed. This pins the - /// happy-path wiring; an accidental "always pause" regression in the - /// helper would break every `run_one_tick`-driven test that follows it. - /// - /// (The redundant "does-not-short-circuit" tick-level test that was - /// here in the first review pass was dropped per @oxoxDev's - /// [#2825 review](https://github.com/tinyhumansai/openhuman/pull/2825): - /// it duplicated `run_one_tick_returns_ok_when_no_client` because - /// both exited at the same `create_composio_client` no-client branch, - /// so neither actually proved the new gate-check arm fired in the - /// right direction. Asserting log-line absence via `tracing-test` - /// would prove it but adds a new dev-dependency for one assertion — - /// the helper-level test below already pins the wiring.) - #[test] - fn periodic_pause_reason_returns_none_when_gate_not_initialised() { - // Calling without `scheduler_gate::init_global(...)` exercises the - // OnceLock-uninitialised branch in `current_policy`, which is the - // realistic test-environment state. - assert!( - periodic_pause_reason().is_none(), - "expected None (i.e. tick proceeds) when scheduler_gate is in default Normal state, \ - got {:?}", - periodic_pause_reason() - ); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/catalogs.rs b/crates/tinymemory-core/src/sync/composio/providers/catalogs.rs deleted file mode 100644 index dfd0d817..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/catalogs.rs +++ /dev/null @@ -1,38 +0,0 @@ -//! Curated catalogs for Composio toolkits that don't (yet) have a -//! native [`super::ComposioProvider`] implementation. -//! -//! These slices are consulted by [`super::catalog_for_toolkit`] alongside -//! provider-supplied catalogs (gmail, notion, github), so the meta-tool -//! layer applies the same whitelist + scope filtering. -//! -//! Slugs sourced from `https://docs.composio.dev/toolkits/.md` — -//! best-effort. Slugs that don't exist on the backend simply never -//! appear in `composio_list_tools`, so extras are harmless. -//! -//! Data is split into category submodules: -//! - `catalogs_messaging` — Slack, Discord, Telegram, WhatsApp, MS Teams -//! - `catalogs_google` — GoogleCalendar, GoogleDrive, GoogleDocs, GoogleSheets -//! - `catalogs_microsoft` — OneDrive, Excel -//! - `catalogs_productivity` — Outlook, Linear, Jira, Trello, Asana, Dropbox, Todoist -//! - `catalogs_social_media` — Twitter, Spotify, YouTube -//! - `catalogs_business` — Shopify, Stripe, HubSpot, Salesforce, Airtable, Figma - -pub use super::catalogs_business::{ - AIRTABLE_CURATED, FIGMA_CURATED, HUBSPOT_CURATED, SALESFORCE_CURATED, SHOPIFY_CURATED, - STRIPE_CURATED, -}; -pub use super::catalogs_google::{ - GOOGLECALENDAR_CURATED, GOOGLEDOCS_CURATED, GOOGLEDRIVE_CURATED, GOOGLESHEETS_CURATED, -}; -pub use super::catalogs_messaging::{ - DISCORD_CURATED, MICROSOFT_TEAMS_CURATED, SLACK_CURATED, TELEGRAM_CURATED, WHATSAPP_CURATED, -}; -pub use super::catalogs_microsoft::{EXCEL_CURATED, ONE_DRIVE_CURATED}; -pub use super::catalogs_productivity::{ - ASANA_CURATED, DROPBOX_CURATED, JIRA_CURATED, OUTLOOK_CURATED, TODOIST_CURATED, TRELLO_CURATED, -}; -// `LINEAR_CURATED` moved into `super::linear::LINEAR_CURATED` alongside -// the native LinearProvider impl. `catalog_for_toolkit("linear")` now -// routes there directly. Removing the re-export keeps a single source -// of truth and matches how `gmail` / `notion` / `clickup` are wired. -pub use super::catalogs_social_media::{SPOTIFY_CURATED, TWITTER_CURATED, YOUTUBE_CURATED}; diff --git a/crates/tinymemory-core/src/sync/composio/providers/catalogs_business.rs b/crates/tinymemory-core/src/sync/composio/providers/catalogs_business.rs deleted file mode 100644 index 0033aec8..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/catalogs_business.rs +++ /dev/null @@ -1,524 +0,0 @@ -//! Curated catalogs — business toolkits: Shopify, Stripe, HubSpot, -//! Salesforce, Airtable, Figma. - -use super::tool_scope::{CuratedTool, ToolScope}; - -// ── shopify ───────────────────────────────────────────────────────── -pub const SHOPIFY_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "SHOPIFY_BULK_QUERY_OPERATION", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SHOPIFY_COUNT_PRODUCTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SHOPIFY_COUNT_ORDERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SHOPIFY_COUNT_FULFILLMENTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SHOPIFY_COUNT_CUSTOMERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SHOPIFY_CREATE_ORDER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SHOPIFY_CREATE_PRODUCT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SHOPIFY_CREATE_DRAFT_ORDER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SHOPIFY_CREATE_FULFILLMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SHOPIFY_CREATE_CUSTOMER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SHOPIFY_CREATE_PRICE_RULE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SHOPIFY_ADJUST_INVENTORY_LEVEL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SHOPIFY_CREATE_DISCOUNT_CODE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SHOPIFY_UPDATE_PRODUCT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SHOPIFY_CREATE_CUSTOM_COLLECTION", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SHOPIFY_CANCEL_ORDER", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SHOPIFY_CANCEL_FULFILLMENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SHOPIFY_DELETE_PRODUCT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SHOPIFY_BULK_DELETE_CUSTOMER_ADDRESSES", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SHOPIFY_BULK_DELETE_METAFIELDS", - scope: ToolScope::Admin, - }, -]; - -// ── stripe ────────────────────────────────────────────────────────── -pub const STRIPE_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "STRIPE_GET_PAYMENT_INTENT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "STRIPE_LIST_INVOICES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "STRIPE_GET_CUSTOMER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "STRIPE_LIST_CHARGES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "STRIPE_GET_SUBSCRIPTION", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "STRIPE_CREATE_PAYMENT_INTENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "STRIPE_CREATE_INVOICE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "STRIPE_CREATE_CUSTOMER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "STRIPE_CREATE_CUSTOMER_SUBSCRIPTION", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "STRIPE_CREATE_CHECKOUT_SESSION", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "STRIPE_CONFIRM_PAYMENT_INTENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "STRIPE_CAPTURE_PAYMENT_INTENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "STRIPE_ATTACH_PAYMENT_METHOD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "STRIPE_CANCEL_SUBSCRIPTION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "STRIPE_CANCEL_PAYMENT_INTENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "STRIPE_CREATE_CHARGE_REFUND", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "STRIPE_CLOSE_DISPUTE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "STRIPE_CANCEL_SETUP_INTENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "STRIPE_ARCHIVE_BILLING_ALERT", - scope: ToolScope::Admin, - }, -]; - -// ── hubspot ───────────────────────────────────────────────────────── -pub const HUBSPOT_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "HUBSPOT_GET_CONTACTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "HUBSPOT_SEARCH_CONTACTS_BY_CRITERIA", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "HUBSPOT_LIST_CONTACTS_PAGE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "HUBSPOT_GET_COMPANIES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "HUBSPOT_GET_DEALS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "HUBSPOT_GET_CRM_OBJECT_BY_ID", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "HUBSPOT_BATCH_READ_COMPANIES_BY_PROPERTIES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "HUBSPOT_CREATE_CONTACT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "HUBSPOT_CREATE_COMPANY", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "HUBSPOT_CREATE_DEAL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "HUBSPOT_CREATE_CONTACTS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "HUBSPOT_UPDATE_CONTACT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "HUBSPOT_UPDATE_COMPANY", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "HUBSPOT_CREATE_OBJECT_ASSOCIATION", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "HUBSPOT_CREATE_A_NEW_MARKETING_EMAIL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "HUBSPOT_CREATE_BATCH_OF_OBJECTS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "HUBSPOT_BATCH_UPDATE_QUOTES", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "HUBSPOT_ARCHIVE_CONTACT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "HUBSPOT_ARCHIVE_COMPANY", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "HUBSPOT_ARCHIVE_DEAL", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "HUBSPOT_ARCHIVE_CONTACTS", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "HUBSPOT_ARCHIVE_COMPANIES", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "HUBSPOT_ARCHIVE_DEALS", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "HUBSPOT_ARCHIVE_CRM_OBJECT_BY_ID", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "HUBSPOT_ARCHIVE_PROPERTY_BY_OBJECT_TYPE_AND_NAME", - scope: ToolScope::Admin, - }, -]; - -// ── salesforce ────────────────────────────────────────────────────── -pub const SALESFORCE_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "SALESFORCE_RUN_SOQL_QUERY", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SALESFORCE_EXECUTE_SOSL_SEARCH", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SALESFORCE_GET_ACCOUNT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SALESFORCE_GET_CAMPAIGN", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SALESFORCE_GET_ALL_FIELDS_FOR_OBJECT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SALESFORCE_GET_ALL_CUSTOM_OBJECTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SALESFORCE_CREATE_ACCOUNT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_CREATE_CONTACT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_CREATE_LEAD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_CREATE_OPPORTUNITY", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_CREATE_CAMPAIGN", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_CREATE_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_UPDATE_ACCOUNT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_UPDATE_CONTACT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_UPDATE_OPPORTUNITY", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_ADD_OPPORTUNITY_LINE_ITEM", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_ADD_CONTACT_TO_CAMPAIGN", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_ADD_LEAD_TO_CAMPAIGN", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_ASSOCIATE_CONTACT_TO_ACCOUNT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_CLONE_OPPORTUNITY_WITH_PRODUCTS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SALESFORCE_DELETE_ACCOUNT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SALESFORCE_DELETE_CONTACT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SALESFORCE_DELETE_LEAD", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SALESFORCE_DELETE_OPPORTUNITY", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SALESFORCE_DELETE_CAMPAIGN", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SALESFORCE_DELETE_SOBJECT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SALESFORCE_DELETE_SOBJECT_COLLECTIONS", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SALESFORCE_CREATE_CUSTOM_FIELD", - scope: ToolScope::Admin, - }, -]; - -// ── airtable ──────────────────────────────────────────────────────── -pub const AIRTABLE_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "AIRTABLE_LIST_RECORDS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "AIRTABLE_GET_RECORD", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "AIRTABLE_GET_BASE_SCHEMA", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "AIRTABLE_LIST_BASES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "AIRTABLE_LIST_COMMENTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "AIRTABLE_CREATE_RECORDS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "AIRTABLE_UPDATE_RECORD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "AIRTABLE_UPDATE_MULTIPLE_RECORDS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "AIRTABLE_CREATE_FIELD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "AIRTABLE_CREATE_TABLE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "AIRTABLE_CREATE_COMMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "AIRTABLE_UPLOAD_ATTACHMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "AIRTABLE_UPDATE_FIELD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "AIRTABLE_UPDATE_TABLE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "AIRTABLE_DELETE_RECORD", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "AIRTABLE_DELETE_MULTIPLE_RECORDS", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "AIRTABLE_DELETE_COMMENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "AIRTABLE_CREATE_BASE", - scope: ToolScope::Admin, - }, -]; - -// ── figma ─────────────────────────────────────────────────────────── -pub const FIGMA_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "FIGMA_GET_FILE_JSON", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "FIGMA_GET_FILE_NODES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "FIGMA_GET_COMMENTS_IN_A_FILE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "FIGMA_GET_CURRENT_USER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "FIGMA_DISCOVER_FIGMA_RESOURCES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "FIGMA_GET_FILE_COMPONENTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "FIGMA_GET_LOCAL_VARIABLES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "FIGMA_EXTRACT_DESIGN_TOKENS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "FIGMA_ADD_A_COMMENT_TO_A_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "FIGMA_CREATE_DEV_RESOURCES", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "FIGMA_CREATE_MODIFY_DELETE_VARIABLES", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "FIGMA_DELETE_A_COMMENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "FIGMA_DELETE_A_WEBHOOK", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "FIGMA_DELETE_DEV_RESOURCE", - scope: ToolScope::Admin, - }, -]; diff --git a/crates/tinymemory-core/src/sync/composio/providers/catalogs_google.rs b/crates/tinymemory-core/src/sync/composio/providers/catalogs_google.rs deleted file mode 100644 index be25c83d..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/catalogs_google.rs +++ /dev/null @@ -1,360 +0,0 @@ -//! Curated catalogs — Google toolkits: GoogleCalendar, GoogleDrive, -//! GoogleDocs, GoogleSheets. - -use super::tool_scope::{CuratedTool, ToolScope}; - -// ── googlecalendar ────────────────────────────────────────────────── -pub const GOOGLECALENDAR_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "GOOGLECALENDAR_EVENTS_LIST", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLECALENDAR_FIND_EVENT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLECALENDAR_LIST_CALENDARS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLECALENDAR_EVENTS_GET", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLECALENDAR_FIND_FREE_SLOTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLECALENDAR_GET_CALENDAR", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLECALENDAR_EVENTS_LIST_ALL_CALENDARS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLECALENDAR_CREATE_EVENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLECALENDAR_UPDATE_EVENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLECALENDAR_PATCH_EVENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLECALENDAR_QUICK_ADD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLECALENDAR_EVENTS_MOVE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLECALENDAR_REMOVE_ATTENDEE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLECALENDAR_EVENTS_IMPORT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLECALENDAR_DELETE_EVENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLECALENDAR_CLEAR_CALENDAR", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLECALENDAR_CALENDARS_DELETE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLECALENDAR_DUPLICATE_CALENDAR", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLECALENDAR_PATCH_CALENDAR", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLECALENDAR_ACL_INSERT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLECALENDAR_ACL_DELETE", - scope: ToolScope::Admin, - }, -]; - -// ── googledrive ───────────────────────────────────────────────────── -pub const GOOGLEDRIVE_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "GOOGLEDRIVE_FIND_FILE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLEDRIVE_LIST_FILES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLEDRIVE_GET_FILE_METADATA", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLEDRIVE_DOWNLOAD_FILE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLEDRIVE_LIST_PERMISSIONS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLEDRIVE_FIND_FOLDER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLEDRIVE_GET_ABOUT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLEDRIVE_CREATE_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDRIVE_CREATE_FOLDER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDRIVE_UPLOAD_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDRIVE_CREATE_FILE_FROM_TEXT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDRIVE_COPY_FILE_ADVANCED", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDRIVE_MOVE_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDRIVE_EDIT_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDRIVE_RENAME_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDRIVE_CREATE_PERMISSION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLEDRIVE_DELETE_PERMISSION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLEDRIVE_UPDATE_PERMISSION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLEDRIVE_DELETE_FILE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLEDRIVE_GOOGLE_DRIVE_DELETE_FOLDER_OR_FILE_ACTION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLEDRIVE_EMPTY_TRASH", - scope: ToolScope::Admin, - }, -]; - -// ── googledocs ────────────────────────────────────────────────────── -pub const GOOGLEDOCS_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "GOOGLEDOCS_GET_DOCUMENT_BY_ID", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLEDOCS_GET_DOCUMENT_PLAINTEXT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLEDOCS_SEARCH_DOCUMENTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLEDOCS_CREATE_DOCUMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_CREATE_DOCUMENT_MARKDOWN", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_INSERT_TEXT_ACTION", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_INSERT_TABLE_ACTION", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_INSERT_INLINE_IMAGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_UPDATE_EXISTING_DOCUMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_UPDATE_DOCUMENT_MARKDOWN", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_UPDATE_DOCUMENT_SECTION_MARKDOWN", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_REPLACE_ALL_TEXT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_COPY_DOCUMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_CREATE_HEADER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_CREATE_FOOTER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLEDOCS_DELETE_CONTENT_RANGE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLEDOCS_DELETE_HEADER", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLEDOCS_DELETE_FOOTER", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLEDOCS_DELETE_NAMED_RANGE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLEDOCS_DELETE_TABLE_ROW", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLEDOCS_DELETE_TABLE_COLUMN", - scope: ToolScope::Admin, - }, -]; - -// ── googlesheets ──────────────────────────────────────────────────── -pub const GOOGLESHEETS_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "GOOGLESHEETS_BATCH_GET", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLESHEETS_VALUES_GET", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLESHEETS_LOOKUP_SPREADSHEET_ROW", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLESHEETS_GET_SPREADSHEET_INFO", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLESHEETS_GET_SHEET_NAMES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLESHEETS_SEARCH_SPREADSHEETS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GOOGLESHEETS_VALUES_UPDATE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLESHEETS_UPDATE_VALUES_BATCH", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLESHEETS_SPREADSHEETS_VALUES_APPEND", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLESHEETS_UPSERT_ROWS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLESHEETS_CREATE_GOOGLE_SHEET1", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLESHEETS_ADD_SHEET", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLESHEETS_CREATE_SPREADSHEET_ROW", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLESHEETS_CREATE_SPREADSHEET_COLUMN", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLESHEETS_FIND_REPLACE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLESHEETS_FORMAT_CELL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLESHEETS_SET_DATA_VALIDATION_RULE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GOOGLESHEETS_SPREADSHEETS_VALUES_BATCH_CLEAR", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLESHEETS_DELETE_SHEET", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLESHEETS_DELETE_DIMENSION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLESHEETS_UPDATE_SHEET_PROPERTIES", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GOOGLESHEETS_UPDATE_SPREADSHEET_PROPERTIES", - scope: ToolScope::Admin, - }, -]; diff --git a/crates/tinymemory-core/src/sync/composio/providers/catalogs_messaging.rs b/crates/tinymemory-core/src/sync/composio/providers/catalogs_messaging.rs deleted file mode 100644 index 33d2975e..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/catalogs_messaging.rs +++ /dev/null @@ -1,386 +0,0 @@ -//! Curated catalogs — messaging toolkits: Slack, Discord, Telegram, -//! WhatsApp, Microsoft Teams. - -use super::tool_scope::{CuratedTool, ToolScope}; - -// ── slack ─────────────────────────────────────────────────────────── -pub const SLACK_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "SLACK_FIND_CHANNELS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SLACK_FIND_USERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SLACK_FETCH_CONVERSATION_HISTORY", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SLACK_FETCH_MESSAGE_THREAD_FROM_A_CONVERSATION", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SLACK_LIST_ALL_CHANNELS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SLACK_LIST_ALL_USERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SLACK_LIST_CONVERSATIONS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SLACK_FETCH_TEAM_INFO", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SLACK_GET_USER_PRESENCE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SLACK_ASSISTANT_SEARCH_CONTEXT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SLACK_SEND_MESSAGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SLACK_POST_MESSAGE_TO_CHANNEL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SLACK_SEND_MESSAGE_TO_CHANNEL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SLACK_CREATE_CHANNEL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SLACK_INVITE_USERS_TO_A_SLACK_CHANNEL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SLACK_ADD_REACTION_TO_AN_ITEM", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SLACK_UPLOAD_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SLACK_CREATE_A_REMINDER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SLACK_CREATE_USER_GROUP", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SLACK_DELETE_CHANNEL", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SLACK_ARCHIVE_CONVERSATION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SLACK_DELETE_FILE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SLACK_DELETES_A_MESSAGE_FROM_A_CHAT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SLACK_DELETE_REMINDER", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SLACK_LEAVE_CONVERSATION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SLACK_INVITE_USER_TO_WORKSPACE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SLACK_CONVERT_CHANNEL_TO_PRIVATE", - scope: ToolScope::Admin, - }, -]; - -// ── discord ───────────────────────────────────────────────────────── -pub const DISCORD_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "DISCORD_GET_MY_USER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DISCORD_GET_USER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DISCORD_LIST_MY_GUILDS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DISCORD_GET_MY_GUILD_MEMBER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DISCORD_INVITE_RESOLVE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DISCORD_GET_GUILD_WIDGET", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DISCORD_LIST_MY_CONNECTIONS", - scope: ToolScope::Read, - }, - // NOTE: guild-channel and channel-message actions are intentionally NOT - // listed here. Composio's `discord` toolkit is OAuth2 / user-scoped and does - // not expose them (its full action set is user/account-scoped: my user, my - // guilds, my member, invites, …). `DISCORD_LIST_GUILD_CHANNELS`, - // `DISCORD_GET_CHANNEL`, `DISCORD_SEND_MESSAGE`, and `DISCORD_CREATE_MESSAGE` - // were whitelisted here (#3085/#3144) but no such slugs exist on this - // toolkit, so Composio never returned them — the whitelist entries were - // inert and misleadingly implied channel access was possible over OAuth. - // Guild-channel / message reads live in Composio's SEPARATE `discordbot` - // toolkit (bot-token auth, `DISCORDBOT_*` slugs, e.g. - // `DISCORDBOT_FETCH_MESSAGES_FROM_CHANNEL`). Those pass the visibility - // filter via `classify_unknown` once a `discordbot` connection exists; do - // NOT hand-list `DISCORDBOT_*` slugs here from guesses — a wrong slug makes - // `find_curated` drop the real tool (worse than the pass-through default). -]; - -// ── telegram ──────────────────────────────────────────────────────── -pub const TELEGRAM_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "TELEGRAM_GET_UPDATES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TELEGRAM_GET_CHAT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TELEGRAM_GET_CHAT_HISTORY", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TELEGRAM_GET_CHAT_MEMBER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TELEGRAM_GET_CHAT_MEMBERS_COUNT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TELEGRAM_GET_CHAT_ADMINISTRATORS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TELEGRAM_GET_ME", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TELEGRAM_SEND_MESSAGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TELEGRAM_SEND_PHOTO", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TELEGRAM_SEND_DOCUMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TELEGRAM_SEND_LOCATION", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TELEGRAM_SEND_POLL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TELEGRAM_FORWARD_MESSAGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TELEGRAM_EDIT_MESSAGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TELEGRAM_ANSWER_CALLBACK_QUERY", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TELEGRAM_DELETE_MESSAGE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TELEGRAM_CREATE_CHAT_INVITE_LINK", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TELEGRAM_SET_MY_COMMANDS", - scope: ToolScope::Admin, - }, -]; - -// ── whatsapp ──────────────────────────────────────────────────────── -pub const WHATSAPP_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "WHATSAPP_GET_PHONE_NUMBERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "WHATSAPP_GET_MESSAGE_TEMPLATES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "WHATSAPP_GET_PHONE_NUMBER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "WHATSAPP_GET_BUSINESS_PROFILE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "WHATSAPP_GET_TEMPLATE_STATUS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "WHATSAPP_GET_MEDIA_INFO", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "WHATSAPP_SEND_MESSAGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "WHATSAPP_SEND_TEMPLATE_MESSAGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "WHATSAPP_SEND_MEDIA", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "WHATSAPP_SEND_MEDIA_BY_ID", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "WHATSAPP_SEND_INTERACTIVE_BUTTONS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "WHATSAPP_SEND_INTERACTIVE_LIST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "WHATSAPP_UPLOAD_MEDIA", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "WHATSAPP_CREATE_MESSAGE_TEMPLATE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "WHATSAPP_DELETE_MESSAGE_TEMPLATE", - scope: ToolScope::Admin, - }, -]; - -// ── microsoft_teams ───────────────────────────────────────────────── -pub const MICROSOFT_TEAMS_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "MICROSOFT_TEAMS_GET_CHAT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_GET_CHANNEL", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_GET_TEAM_FROM_GROUP", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_CHATS_GET_ALL_CHATS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_GET_PRESENCE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_GET_ONLINE_MEETING", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_GET_SCHEDULE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_CREATE_CHANNEL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_CREATE_TEAM", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_CREATE_MEETING", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_ADD_TEAM_MEMBER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_ADD_CHAT_MEMBER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_CREATE_SHIFT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_CREATE_TIME_OFF_REQUEST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_DELETE_TEAM", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_DELETE_CHANNEL", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_ARCHIVE_TEAM", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_ARCHIVE_CHANNEL", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_DELETE_TAB", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "MICROSOFT_TEAMS_DELETE_TIME_OFF", - scope: ToolScope::Admin, - }, -]; diff --git a/crates/tinymemory-core/src/sync/composio/providers/catalogs_microsoft.rs b/crates/tinymemory-core/src/sync/composio/providers/catalogs_microsoft.rs deleted file mode 100644 index cd54ede9..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/catalogs_microsoft.rs +++ /dev/null @@ -1,207 +0,0 @@ -//! Curated catalogs — Microsoft personal-productivity toolkits: -//! OneDrive (files) and Excel (spreadsheets). -//! -//! These toolkits are catalog-only: they don't ship a native -//! [`super::ComposioProvider`] implementation, so they have no -//! user-profile fetch, no initial/periodic sync, no trigger webhooks, -//! and no memory ingestion. Connecting them via the UI lets the agent -//! invoke the listed actions through Composio's API, but their data -//! is not pre-ingested into OpenHuman's memory tree. -//! -//! Action slugs are sourced best-effort from -//! `https://docs.composio.dev/toolkits/.md`. Slugs that don't -//! exist on the backend simply never appear in `composio_list_tools`, -//! so over-shooting is harmless. - -use super::tool_scope::{CuratedTool, ToolScope}; - -// ── onedrive ──────────────────────────────────────────────────────── -pub const ONE_DRIVE_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "ONE_DRIVE_GET_FILE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ONE_DRIVE_GET_FILE_METADATA", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ONE_DRIVE_LIST_FILES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ONE_DRIVE_LIST_CHILDREN", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ONE_DRIVE_SEARCH_FILES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ONE_DRIVE_DOWNLOAD_FILE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ONE_DRIVE_GET_DRIVE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ONE_DRIVE_UPLOAD_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ONE_DRIVE_CREATE_FOLDER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ONE_DRIVE_COPY_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ONE_DRIVE_MOVE_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ONE_DRIVE_UPDATE_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ONE_DRIVE_CREATE_SHARE_LINK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ONE_DRIVE_DELETE_FILE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "ONE_DRIVE_DELETE_FOLDER", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "ONE_DRIVE_RESTORE_FILE", - scope: ToolScope::Admin, - }, -]; - -// ── excel ─────────────────────────────────────────────────────────── -pub const EXCEL_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "EXCEL_GET_WORKBOOK", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "EXCEL_LIST_WORKSHEETS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "EXCEL_GET_WORKSHEET", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "EXCEL_GET_RANGE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "EXCEL_GET_USED_RANGE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "EXCEL_LIST_TABLES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "EXCEL_GET_TABLE_ROWS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "EXCEL_CREATE_WORKSHEET", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "EXCEL_UPDATE_RANGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "EXCEL_APPEND_ROWS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "EXCEL_INSERT_TABLE_ROW", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "EXCEL_UPDATE_TABLE_ROW", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "EXCEL_CREATE_TABLE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "EXCEL_FORMAT_RANGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "EXCEL_DELETE_WORKSHEET", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "EXCEL_DELETE_TABLE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "EXCEL_DELETE_TABLE_ROW", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "EXCEL_CLEAR_RANGE", - scope: ToolScope::Admin, - }, -]; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn one_drive_catalog_is_non_empty_and_unique() { - assert!(!ONE_DRIVE_CURATED.is_empty()); - let mut slugs: Vec<&'static str> = ONE_DRIVE_CURATED.iter().map(|t| t.slug).collect(); - slugs.sort_unstable(); - slugs.dedup(); - assert_eq!(slugs.len(), ONE_DRIVE_CURATED.len()); - for tool in ONE_DRIVE_CURATED { - assert!(tool.slug.starts_with("ONE_DRIVE_")); - } - } - - #[test] - fn excel_catalog_is_non_empty_and_unique() { - assert!(!EXCEL_CURATED.is_empty()); - let mut slugs: Vec<&'static str> = EXCEL_CURATED.iter().map(|t| t.slug).collect(); - slugs.sort_unstable(); - slugs.dedup(); - assert_eq!(slugs.len(), EXCEL_CURATED.len()); - for tool in EXCEL_CURATED { - assert!(tool.slug.starts_with("EXCEL_")); - } - } - - #[test] - fn one_drive_catalog_covers_all_three_scopes() { - assert!(ONE_DRIVE_CURATED.iter().any(|t| t.scope == ToolScope::Read)); - assert!(ONE_DRIVE_CURATED - .iter() - .any(|t| t.scope == ToolScope::Write)); - assert!(ONE_DRIVE_CURATED - .iter() - .any(|t| t.scope == ToolScope::Admin)); - } - - #[test] - fn excel_catalog_covers_all_three_scopes() { - assert!(EXCEL_CURATED.iter().any(|t| t.scope == ToolScope::Read)); - assert!(EXCEL_CURATED.iter().any(|t| t.scope == ToolScope::Write)); - assert!(EXCEL_CURATED.iter().any(|t| t.scope == ToolScope::Admin)); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/catalogs_productivity.rs b/crates/tinymemory-core/src/sync/composio/providers/catalogs_productivity.rs deleted file mode 100644 index 424efde9..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/catalogs_productivity.rs +++ /dev/null @@ -1,600 +0,0 @@ -//! Curated catalogs — productivity toolkits: Outlook, Linear, Jira, -//! Trello, Asana, Dropbox, Todoist. -//! -//! Catalog-only toolkits (Linear, Jira, Trello, Asana, Dropbox, -//! Todoist) don't ship a native [`super::ComposioProvider`] — they -//! have no user-profile fetch, no initial/periodic sync, no trigger -//! webhooks, and no memory ingestion. The agent invokes their actions -//! through Composio's API, but their data is not pre-ingested into -//! OpenHuman's memory tree. - -use super::tool_scope::{CuratedTool, ToolScope}; - -// ── outlook ───────────────────────────────────────────────────────── -pub const OUTLOOK_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "OUTLOOK_GET_MESSAGE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "OUTLOOK_LIST_MESSAGES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "OUTLOOK_SEARCH_MESSAGES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "OUTLOOK_LIST_CALENDARS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "OUTLOOK_LIST_CALENDAR_EVENTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "OUTLOOK_GET_CALENDAR_EVENT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "OUTLOOK_LIST_CONTACTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "OUTLOOK_LIST_MAIL_FOLDERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "OUTLOOK_SEND_EMAIL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "OUTLOOK_CREATE_DRAFT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "OUTLOOK_SEND_DRAFT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "OUTLOOK_CREATE_DRAFT_REPLY", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "OUTLOOK_CREATE_ME_FORWARD_DRAFT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "OUTLOOK_CALENDAR_CREATE_EVENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "OUTLOOK_CREATE_CONTACT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "OUTLOOK_CREATE_MAIL_FOLDER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "OUTLOOK_DELETE_MESSAGE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "OUTLOOK_BATCH_MOVE_MESSAGES", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "OUTLOOK_BATCH_UPDATE_MESSAGES", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "OUTLOOK_ACCEPT_EVENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "OUTLOOK_CANCEL_EVENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "OUTLOOK_CREATE_ME_CALENDAR_PERMISSION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "OUTLOOK_CREATE_EMAIL_RULE", - scope: ToolScope::Admin, - }, -]; - -// ── linear ────────────────────────────────────────────────────────── -// -// `LINEAR_CURATED` lives in `super::linear::tools` alongside the native -// `LinearProvider` impl (per-issue #2400). `catalog_for_toolkit("linear")` -// in `super::mod` routes through that constant directly. Removing the -// catalog-only declaration here keeps a single source of truth and -// matches how `gmail` / `notion` / `clickup` are wired. - -// ── jira ──────────────────────────────────────────────────────────── -pub const JIRA_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "JIRA_GET_ISSUE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "JIRA_GET_ALL_PROJECTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "JIRA_FETCH_BULK_ISSUES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "JIRA_GET_ISSUE_TYPES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "JIRA_GET_PROJECT_ROLES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "JIRA_FIND_USERS2", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "JIRA_GET_FIELDS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "JIRA_GET_ISSUE_EDIT_METADATA", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "JIRA_GET_PROJECT_VERSIONS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "JIRA_CREATE_ISSUE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "JIRA_BULK_CREATE_ISSUE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "JIRA_EDIT_ISSUE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "JIRA_ADD_COMMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "JIRA_ASSIGN_ISSUE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "JIRA_ADD_ATTACHMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "JIRA_CREATE_ISSUE_LINK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "JIRA_ADD_WORKLOG", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "JIRA_TRANSITION_ISSUE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "JIRA_DELETE_ISSUE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "JIRA_DELETE_COMMENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "JIRA_DELETE_VERSION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "JIRA_DELETE_WORKLOG", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "JIRA_CREATE_PROJECT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "JIRA_ADD_USERS_TO_PROJECT_ROLE", - scope: ToolScope::Admin, - }, -]; - -// ── trello ────────────────────────────────────────────────────────── -pub const TRELLO_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "TRELLO_GET_BOARDS_BY_ID_BOARD", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TRELLO_GET_ACTIONS_BY_ID_ACTION", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TRELLO_GET_BATCH", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TRELLO_GET_BOARDS_ACTIONS_BY_ID_BOARD", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TRELLO_GET_MEMBERS_BOARDS_BY_ID_MEMBER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TRELLO_ADD_CARDS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TRELLO_ADD_BOARDS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TRELLO_ADD_LISTS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TRELLO_ADD_CARDS_ACTIONS_COMMENTS_BY_ID_CARD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TRELLO_ADD_MEMBER_TO_CARD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TRELLO_CREATE_CARD_LABEL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TRELLO_ADD_CARDS_ATTACHMENTS_BY_ID_CARD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TRELLO_ADD_CARDS_CHECKLISTS_BY_ID_CARD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TRELLO_CREATE_WEBHOOK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TRELLO_DELETE_CARDS_BY_ID_CARD", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TRELLO_DELETE_BOARD", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TRELLO_DELETE_CHECKLISTS_BY_ID_CHECKLIST", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TRELLO_ARCHIVE_ALL_LIST_CARDS", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TRELLO_DELETE_CARD_COMMENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TRELLO_DELETE_LABELS_BY_ID_LABEL", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TRELLO_DELETE_ORGANIZATIONS_BY_ID_ORG", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TRELLO_DELETE_WEBHOOKS_BY_ID_WEBHOOK", - scope: ToolScope::Admin, - }, -]; - -// ── asana ─────────────────────────────────────────────────────────── -pub const ASANA_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "ASANA_GET_A_TASK", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ASANA_GET_A_PROJECT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ASANA_GET_MULTIPLE_TASKS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ASANA_GET_MULTIPLE_PROJECTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ASANA_GET_CURRENT_USER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ASANA_GET_MULTIPLE_WORKSPACES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ASANA_GET_PORTFOLIO", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ASANA_GET_GOALS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ASANA_GET_CUSTOM_FIELDS_FOR_WORKSPACE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "ASANA_CREATE_A_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ASANA_CREATE_A_PROJECT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ASANA_CREATE_SUBTASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ASANA_CREATE_TASK_COMMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ASANA_UPDATE_A_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ASANA_ADD_FOLLOWERS_TO_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ASANA_ADD_TAG_TO_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ASANA_ADD_PROJECT_FOR_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ASANA_ADD_TASK_DEPENDENCIES", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ASANA_CREATE_ATTACHMENT_FOR_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "ASANA_DELETE_TASK", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "ASANA_DELETE_PROJECT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "ASANA_DELETE_SECTION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "ASANA_DELETE_TAG", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "ASANA_DELETE_CUSTOM_FIELD", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "ASANA_DELETE_MEMBERSHIP", - scope: ToolScope::Admin, - }, -]; - -// ── dropbox ───────────────────────────────────────────────────────── -pub const DROPBOX_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "DROPBOX_GET_METADATA", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DROPBOX_FILES_SEARCH", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DROPBOX_LIST_FILE_MEMBERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DROPBOX_GET_SHARED_LINK_METADATA", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DROPBOX_GET_ABOUT_ME", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DROPBOX_GET_SPACE_USAGE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "DROPBOX_ALPHA_UPLOAD_FILE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "DROPBOX_CREATE_FOLDER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "DROPBOX_COPY_FILE_OR_FOLDER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "DROPBOX_CREATE_SHARED_LINK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "DROPBOX_ADD_FILE_MEMBER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "DROPBOX_DELETE_FILE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "DROPBOX_DELETE_BATCH", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "DROPBOX_ADD_TEAM_MEMBERS", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "DROPBOX_CREATE_TEAM_FOLDER", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "DROPBOX_ARCHIVE_TEAM_FOLDER", - scope: ToolScope::Admin, - }, -]; - -// ── todoist ───────────────────────────────────────────────────────── -pub const TODOIST_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "TODOIST_GET_TASK", - scope: ToolScope::Read, - }, - CuratedTool { - // Composio's catalog has no `TODOIST_GET_ACTIVE_TASKS`; the real - // incomplete-tasks slug is `TODOIST_GET_ALL_TASKS` (docs.composio.dev/ - // toolkits/todoist). The old slug was rejected as an unknown action. - slug: "TODOIST_GET_ALL_TASKS", - scope: ToolScope::Read, - }, - CuratedTool { - // Real completed-tasks slug; `TODOIST_GET_COMPLETED_TASKS` does not - // exist in Composio's catalog. - slug: "TODOIST_LIST_COMPLETED_TASKS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TODOIST_GET_PROJECTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TODOIST_GET_PROJECT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TODOIST_GET_SECTIONS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TODOIST_GET_LABELS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TODOIST_GET_COMMENTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TODOIST_CREATE_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TODOIST_UPDATE_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TODOIST_CLOSE_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TODOIST_REOPEN_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TODOIST_CREATE_PROJECT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TODOIST_UPDATE_PROJECT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TODOIST_CREATE_SECTION", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TODOIST_CREATE_LABEL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TODOIST_CREATE_COMMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TODOIST_DELETE_TASK", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TODOIST_DELETE_PROJECT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TODOIST_DELETE_SECTION", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TODOIST_DELETE_LABEL", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TODOIST_DELETE_COMMENT", - scope: ToolScope::Admin, - }, -]; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn todoist_catalog_is_non_empty_and_unique() { - assert!(!TODOIST_CURATED.is_empty()); - let mut slugs: Vec<&'static str> = TODOIST_CURATED.iter().map(|t| t.slug).collect(); - slugs.sort_unstable(); - slugs.dedup(); - assert_eq!(slugs.len(), TODOIST_CURATED.len()); - for tool in TODOIST_CURATED { - assert!(tool.slug.starts_with("TODOIST_")); - } - } - - #[test] - fn todoist_catalog_covers_all_three_scopes() { - assert!(TODOIST_CURATED.iter().any(|t| t.scope == ToolScope::Read)); - assert!(TODOIST_CURATED.iter().any(|t| t.scope == ToolScope::Write)); - assert!(TODOIST_CURATED.iter().any(|t| t.scope == ToolScope::Admin)); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/catalogs_social_media.rs b/crates/tinymemory-core/src/sync/composio/providers/catalogs_social_media.rs deleted file mode 100644 index a05c2b40..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/catalogs_social_media.rs +++ /dev/null @@ -1,248 +0,0 @@ -//! Curated catalogs — social media / entertainment toolkits: Twitter, -//! Spotify, YouTube. - -use super::tool_scope::{CuratedTool, ToolScope}; - -// ── twitter ───────────────────────────────────────────────────────── -pub const TWITTER_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "TWITTER_RECENT_SEARCH", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TWITTER_GET_USER_BY_ID", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TWITTER_POST_LOOKUP_BY_POST_ID", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TWITTER_FOLLOWERS_BY_USER_ID", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TWITTER_FOLLOWING_BY_USER_ID", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TWITTER_BOOKMARKS_BY_USER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TWITTER_GET_LIST_MEMBERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TWITTER_FULL_ARCHIVE_SEARCH", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "TWITTER_CREATION_OF_A_POST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TWITTER_RETWEET_POST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TWITTER_ADD_POST_TO_BOOKMARKS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TWITTER_FOLLOW_USER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TWITTER_MUTE_USER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TWITTER_CREATE_DM_CONVERSATION", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TWITTER_CREATE_LIST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TWITTER_ADD_LIST_MEMBER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "TWITTER_POST_DELETE_BY_POST_ID", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TWITTER_DELETE_LIST", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TWITTER_REMOVE_LIST_MEMBER", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TWITTER_DELETE_DM", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "TWITTER_REMOVE_POST_FROM_BOOKMARKS", - scope: ToolScope::Admin, - }, -]; - -// ── spotify ───────────────────────────────────────────────────────── -pub const SPOTIFY_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "SPOTIFY_GET_CURRENT_USER_S_PROFILE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SPOTIFY_GET_USER_S_TOP_TRACKS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SPOTIFY_GET_PLAYLIST", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SPOTIFY_GET_PLAYLIST_ITEMS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SPOTIFY_GET_RECENTLY_PLAYED_TRACKS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SPOTIFY_GET_USER_S_SAVED_TRACKS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SPOTIFY_SEARCH_FOR_ITEM", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SPOTIFY_GET_AVAILABLE_DEVICES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "SPOTIFY_ADD_ITEMS_TO_PLAYLIST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SPOTIFY_CREATE_PLAYLIST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SPOTIFY_SAVE_TRACKS_FOR_CURRENT_USER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SPOTIFY_PAUSE_PLAYBACK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SPOTIFY_ADD_ITEM_TO_PLAYBACK_QUEUE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SPOTIFY_CHANGE_PLAYLIST_DETAILS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "SPOTIFY_REMOVE_PLAYLIST_ITEMS", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SPOTIFY_REMOVE_USER_S_SAVED_TRACKS", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SPOTIFY_UNFOLLOW_ARTISTS_OR_USERS", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "SPOTIFY_REMOVE_USER_S_SAVED_ALBUMS", - scope: ToolScope::Admin, - }, -]; - -// ── youtube ───────────────────────────────────────────────────────── -pub const YOUTUBE_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "YOUTUBE_SEARCH_YOU_TUBE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "YOUTUBE_LIST_CHANNEL_VIDEOS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "YOUTUBE_GET_CHANNEL_STATISTICS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "YOUTUBE_LIST_COMMENT_THREADS2", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "YOUTUBE_LIST_COMMENTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "YOUTUBE_GET_VIDEO_DETAILS_BATCH", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "YOUTUBE_LIST_USER_PLAYLISTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "YOUTUBE_LIST_PLAYLIST_ITEMS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "YOUTUBE_UPLOAD_VIDEO", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "YOUTUBE_UPDATE_VIDEO", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "YOUTUBE_CREATE_PLAYLIST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "YOUTUBE_ADD_VIDEO_TO_PLAYLIST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "YOUTUBE_POST_COMMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "YOUTUBE_RATE_VIDEO", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "YOUTUBE_UPDATE_PLAYLIST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "YOUTUBE_DELETE_VIDEO", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "YOUTUBE_DELETE_PLAYLIST", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "YOUTUBE_DELETE_COMMENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "YOUTUBE_DELETE_PLAYLIST_ITEM", - scope: ToolScope::Admin, - }, -]; diff --git a/crates/tinymemory-core/src/sync/composio/providers/clickup/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/clickup/mod.rs deleted file mode 100644 index 7ca8115c..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/clickup/mod.rs +++ /dev/null @@ -1,27 +0,0 @@ -//! ClickUp Composio provider — incremental Memory Tree ingest for -//! tasks owned by (or assigned to) the connected user. -//! -//! Mirrors the [`crate::sync::composio::providers::notion`] layout -//! so anyone familiar with Notion/Slack ingestion can read this without -//! re-learning a new shape: -//! -//! - `provider.rs` — `impl ComposioProvider for ClickUpProvider` -//! - `normalization` — payload-shape helpers, owned by `tinymemory-sync` -//! (issue #18 §B3) -//! - `ingest.rs` — memory_tree document ingest (issue #2885) -//! - `tools.rs` — `CLICKUP_CURATED` whitelist of Composio actions -//! - `tests.rs` — unit tests for the helpers + trait metadata -//! -//! Issue: #2288 (introduction); #2885 (memory_tree migration). - -// The payload normalisers moved to tinycortex (they are pure Value -// transforms, i.e. driver-side). Aliased under the old module name so -// every `normalization::extract_*` call site below stays unchanged. -use tinymemory_sync::clickup as normalization; -mod provider; -#[cfg(test)] -mod tests; -pub mod tools; - -pub use provider::ClickUpProvider; -pub use tools::CLICKUP_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/clickup/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/clickup/provider.rs deleted file mode 100644 index 916caaa3..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/clickup/provider.rs +++ /dev/null @@ -1,271 +0,0 @@ -//! ClickUp provider — incremental sync of tasks assigned to the -//! authenticated user, with per-item persistence into the Memory Tree. -//! -//! On each sync pass: -//! -//! 1. Load persistent [`SyncState`] from the KV store. -//! 2. Check the daily request budget — bail early if exhausted. -//! 3. If we don't yet know the user's numeric ID, call -//! `CLICKUP_GET_AUTHORIZED_USER` and cache the result in memory -//! (it doesn't change for the lifetime of the connection). -//! 4. If we don't yet know which workspaces (teams) the connection -//! can see, call `CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES` and -//! cache the list. -//! 5. For each workspace, page through -//! `CLICKUP_GET_FILTERED_TEAM_TASKS` filtered to the user as -//! assignee, sorted by `date_updated` descending. Stop a workspace -//! early once we hit tasks older than the cursor. -//! 6. For each task, persist as a single memory document if it's new -//! *or* edited since the last sync. -//! 7. Advance the cursor to the newest `date_updated` seen and save. -//! -//! Privacy posture: we only pull tasks the user is assigned to, never -//! the whole workspace's task graph. This mirrors the -//! "fetch-what-the-user-sees" model `gmail` / `notion` already follow -//! and avoids accidentally ingesting other teammates' private tasks. - -use async_trait::async_trait; -use serde_json::json; - -use super::normalization; -use crate::sync::composio::providers::{ - first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, - CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind, -}; - -pub(crate) const ACTION_GET_AUTHORIZED_USER: &str = "CLICKUP_GET_AUTHORIZED_USER"; -pub(crate) const ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES: &str = - "CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES"; -pub(crate) const ACTION_GET_FILTERED_TEAM_TASKS: &str = "CLICKUP_GET_FILTERED_TEAM_TASKS"; - -/// Paths for extracting a task's unique ID. Composio sometimes wraps -/// the upstream payload under `data`, so we check both shapes. -pub(super) const TASK_ID_PATHS: &[&str] = &["id", "data.id", "task_id", "data.task_id"]; - -pub struct ClickUpProvider; - -impl ClickUpProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for ClickUpProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for ClickUpProvider { - fn toolkit_slug(&self) -> &'static str { - "clickup" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(super::tools::CLICKUP_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - // 30 minutes — same cadence as Notion. ClickUp tasks change - // more slowly than chat but faster than email, so this is in - // the middle. - Some(resolve_sync_interval_secs("clickup", 30 * 60)) - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:clickup] fetch_user_profile via {ACTION_GET_AUTHORIZED_USER}" - ); - - let resp = ctx - .execute(ACTION_GET_AUTHORIZED_USER, Some(json!({}))) - .await - .map_err(|e| { - format!("[composio:clickup] {ACTION_GET_AUTHORIZED_USER} failed: {e:#}") - })?; - - if !resp.successful { - let err = resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!( - "[composio:clickup] {ACTION_GET_AUTHORIZED_USER}: {err}" - )); - } - - // Composio's wrapping puts ClickUp's `{user: {…}}` payload at - // `data` or `data.user`. We probe both — `pick_str` walks dotted - // paths so `user.username` and `data.user.username` both work. - let data = &resp.data; - let display_name = pick_str(data, &["user.username", "data.user.username", "username"]); - let email = pick_str(data, &["user.email", "data.user.email", "email"]); - let username = normalization::extract_user_id(data); - let avatar_url = pick_str( - data, - &[ - "user.profilePicture", - "data.user.profilePicture", - "profilePicture", - ], - ); - let profile_url = None; - - Ok(ProviderUserProfile { - toolkit: "clickup".to_string(), - connection_id: ctx.connection_id.clone(), - display_name, - email, - username, - avatar_url, - profile_url, - extras: data.clone(), - }) - } - - /// Incremental sync via the generic - /// `orchestrator`: - /// user/workspace resolution, the per-workspace page loop, dedup, the - /// `max_items` cap, the epoch-ms `sync_depth_days` window, and cursor - /// handling live in `run_sync`; the ClickUp-specific primitives live in - /// `super::source`. - async fn fetch_tasks( - &self, - ctx: &ProviderContext, - filter: &TaskFetchFilter, - ) -> Result, String> { - let max = filter.effective_max(); - tracing::debug!( - connection_id = ?ctx.connection_id, - max, - team_id = ?filter.team_id, - assignee_is_me = filter.assignee_is_me, - "[composio:clickup] fetch_tasks" - ); - - // Resolve which workspaces (teams) to query. An explicit - // `team_id` from the filter wins; otherwise enumerate every - // workspace the connection can see. - let workspaces = match &filter.team_id { - Some(team) if !team.trim().is_empty() => vec![team.trim().to_string()], - _ => { - let resp = ctx - .execute(ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES, Some(json!({}))) - .await - .map_err(|e| { - format!( - "[composio:clickup] {ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES}: {e:#}" - ) - })?; - if !resp.successful { - return Err(format!( - "[composio:clickup] {ACTION_GET_AUTHORIZED_TEAMS_WORKSPACES}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - normalization::extract_workspace_ids(&resp.data) - } - }; - - // Resolve the current user id only when the filter scopes to - // "assigned to me". - let assignees: Vec = if filter.assignee_is_me { - let resp = ctx - .execute(ACTION_GET_AUTHORIZED_USER, Some(json!({}))) - .await - .map_err(|e| format!("[composio:clickup] {ACTION_GET_AUTHORIZED_USER}: {e:#}"))?; - // Fail closed: if we can't resolve the user, error rather than - // silently dropping the assignee filter and fetching the whole - // workspace's tasks. - if !resp.successful { - return Err(format!( - "[composio:clickup] {ACTION_GET_AUTHORIZED_USER}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - let id = normalization::extract_user_id(&resp.data).ok_or_else(|| { - "[composio:clickup] CLICKUP_GET_AUTHORIZED_USER returned no user.id".to_string() - })?; - vec![id] - } else { - Vec::new() - }; - - let mut out: Vec = Vec::new(); - 'workspaces: for workspace_id in &workspaces { - let mut args = json!({ - "team_id": workspace_id, - "order_by": "updated", - "reverse": true, - "page": 0, - "page_size": max.min(100) as u32, - "subtasks": true, - }); - if !assignees.is_empty() { - args["assignees"] = json!(assignees); - } - if let Some(list_id) = filter.list_id.as_deref().filter(|s| !s.trim().is_empty()) { - args["list_ids"] = json!([list_id]); - } - merge_extra(&mut args, &filter.extra); - - let resp = ctx - .execute(ACTION_GET_FILTERED_TEAM_TASKS, Some(args)) - .await - .map_err(|e| { - format!("[composio:clickup] {ACTION_GET_FILTERED_TEAM_TASKS} ws={workspace_id}: {e:#}") - })?; - if !resp.successful { - return Err(format!( - "[composio:clickup] {ACTION_GET_FILTERED_TEAM_TASKS} ws={workspace_id}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - - for task in normalization::extract_tasks(&resp.data) { - if out.len() >= max { - break 'workspaces; - } - if let Some(nt) = normalize_clickup_task(&task) { - out.push(nt); - } - } - } - - tracing::debug!(count = out.len(), "[composio:clickup] fetch_tasks complete"); - Ok(out) - } -} - -/// Map a raw ClickUp task payload into a [`NormalizedTask`]. Returns -/// `None` only when the task has no extractable id (unroutable). -fn normalize_clickup_task(task: &serde_json::Value) -> Option { - let external_id = pick_str(task, TASK_ID_PATHS)?; - let title = normalization::extract_task_name(task) - .unwrap_or_else(|| format!("ClickUp task {external_id}")); - Some(NormalizedTask { - external_id, - source_id: String::new(), - provider: "clickup".to_string(), - kind: TaskKind::Generic, - title, - body: pick_str(task, &["description", "data.description", "text_content"]), - url: pick_str(task, &["url", "data.url"]), - status: pick_str(task, &["status.status", "data.status.status", "status"]), - assignee: first_array_str( - task, - &["assignees", "data.assignees"], - &["username", "email"], - ), - due: pick_str(task, &["due_date", "data.due_date"]), - labels: Vec::new(), - priority: pick_str(task, &["priority.priority", "data.priority.priority"]), - updated_at: normalization::extract_task_updated(task), - raw: task.clone(), - }) -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/clickup/tests.rs b/crates/tinymemory-core/src/sync/composio/providers/clickup/tests.rs deleted file mode 100644 index 5294aa08..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/clickup/tests.rs +++ /dev/null @@ -1,155 +0,0 @@ -//! Unit tests for the ClickUp provider. - -use super::normalization::{ - extract_task_name, extract_task_updated, extract_tasks, extract_user_id, extract_workspace_ids, -}; -use super::ClickUpProvider; -use crate::sync::composio::providers::ComposioProvider; -use serde_json::json; - -#[test] -fn extract_tasks_walks_common_shapes() { - let v1 = json!({ "data": { "tasks": [{"id": "t1"}] } }); - let v2 = json!({ "tasks": [{"id": "t2"}, {"id": "t3"}] }); - let v3 = json!({ "data": {} }); - assert_eq!(extract_tasks(&v1).len(), 1); - assert_eq!(extract_tasks(&v2).len(), 2); - assert_eq!(extract_tasks(&v3).len(), 0); -} - -#[test] -fn extract_task_name_finds_name_field() { - let task = json!({ "id": "abc", "name": "Build feature X" }); - assert_eq!(extract_task_name(&task), Some("Build feature X".into())); -} - -#[test] -fn extract_task_name_falls_back_to_wrapped_data() { - let task = json!({ "data": { "name": "Wrapped" } }); - assert_eq!(extract_task_name(&task), Some("Wrapped".into())); -} - -#[test] -fn extract_task_name_returns_none_when_missing() { - let task = json!({ "id": "abc" }); - assert!(extract_task_name(&task).is_none()); -} - -#[test] -fn extract_task_updated_handles_string_form() { - let task = json!({ "date_updated": "1733412345678" }); - assert_eq!( - extract_task_updated(&task), - Some("1733412345678".to_string()) - ); -} - -#[test] -fn extract_task_updated_handles_nested_data() { - let task = json!({ "data": { "dateUpdated": "1700000000000" } }); - assert_eq!( - extract_task_updated(&task), - Some("1700000000000".to_string()) - ); -} - -#[test] -fn extract_task_updated_returns_none_when_missing() { - let task = json!({ "id": "abc" }); - assert!(extract_task_updated(&task).is_none()); -} - -#[test] -fn extract_user_id_handles_numeric_id() { - let data = json!({ "user": { "id": 12345 } }); - assert_eq!(extract_user_id(&data), Some("12345".to_string())); -} - -#[test] -fn extract_user_id_handles_wrapped_payload() { - let data = json!({ "data": { "user": { "id": "777" } } }); - assert_eq!(extract_user_id(&data), Some("777".to_string())); -} - -#[test] -fn extract_user_id_none_when_missing() { - let data = json!({ "foo": "bar" }); - assert!(extract_user_id(&data).is_none()); -} - -#[test] -fn extract_workspace_ids_from_teams_array() { - let data = json!({ - "teams": [ - { "id": "ws1", "name": "Personal" }, - { "id": "ws2", "name": "Acme" }, - ] - }); - assert_eq!(extract_workspace_ids(&data), vec!["ws1", "ws2"]); -} - -#[test] -fn extract_workspace_ids_handles_wrapped_payload() { - let data = json!({ - "data": { - "teams": [ - { "id": "ws1" }, - { "id": "ws2" }, - { "id": "ws3" }, - ] - } - }); - assert_eq!(extract_workspace_ids(&data), vec!["ws1", "ws2", "ws3"]); -} - -#[test] -fn extract_workspace_ids_empty_when_no_teams() { - let data = json!({ "foo": "bar" }); - assert!(extract_workspace_ids(&data).is_empty()); -} - -#[test] -fn extract_workspace_ids_skips_entries_without_id() { - let data = json!({ - "teams": [ - { "name": "Anonymous" }, - { "id": "ws1", "name": "Real" }, - ] - }); - assert_eq!(extract_workspace_ids(&data), vec!["ws1"]); -} - -#[test] -fn provider_metadata_is_stable() { - let p = ClickUpProvider::new(); - assert_eq!(p.toolkit_slug(), "clickup"); - assert_eq!(p.sync_interval_secs(), Some(30 * 60)); - assert!(p.curated_tools().is_some()); -} - -#[test] -fn curated_tools_contains_core_read_surface() { - let p = ClickUpProvider::new(); - let curated = p.curated_tools().expect("CLICKUP_CURATED is registered"); - let slugs: Vec<&str> = curated.iter().map(|t| t.slug).collect(); - // The three actions the sync path depends on must be advertised. - assert!(slugs.contains(&"CLICKUP_GET_AUTHORIZED_USER")); - assert!(slugs.contains(&"CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES")); - assert!(slugs.contains(&"CLICKUP_GET_FILTERED_TEAM_TASKS")); -} - -#[test] -fn default_impl_matches_new() { - // `ClickUpProvider` is a unit struct, so we compare observable - // trait surface instead of deriving `PartialEq`. This catches a - // future regression where `new()` and `default()` drift apart - // (e.g. one is given an extra field but the other is forgotten). - let a = ClickUpProvider::new(); - let b = ::default(); - assert_eq!(a.toolkit_slug(), b.toolkit_slug()); - assert_eq!(a.sync_interval_secs(), b.sync_interval_secs()); - assert_eq!( - a.curated_tools().map(<[_]>::len), - b.curated_tools().map(<[_]>::len), - ); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/clickup/tools.rs b/crates/tinymemory-core/src/sync/composio/providers/clickup/tools.rs deleted file mode 100644 index d0c89fe4..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/clickup/tools.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! Curated catalog of ClickUp Composio actions exposed to the agent. -//! -//! Slugs match Composio's naming convention (`_`) for -//! the ClickUp REST surface. See -//! for the canonical action list; the entries here are the read-oriented -//! subset the periodic Memory Tree sync relies on, plus the most common -//! task-write surface the agent already uses through generic tool-calling. - -use crate::sync::composio::providers::tool_scope::{CuratedTool, ToolScope}; - -pub const CLICKUP_CURATED: &[CuratedTool] = &[ - // ── Read: identity ───────────────────────────────────────────── - CuratedTool { - slug: "CLICKUP_GET_AUTHORIZED_USER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES", - scope: ToolScope::Read, - }, - // ── Read: structure (workspace → space → folder → list) ────── - CuratedTool { - slug: "CLICKUP_GET_SPACES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_FOLDERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_LISTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_FOLDERLESS_LISTS", - scope: ToolScope::Read, - }, - // ── Read: tasks (the main memory ingest surface) ────────────── - CuratedTool { - slug: "CLICKUP_GET_FILTERED_TEAM_TASKS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_TASKS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_TASK", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_TASK_COMMENTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_LIST_COMMENTS", - scope: ToolScope::Read, - }, - // ── Read: docs / views / time tracking ──────────────────────── - CuratedTool { - slug: "CLICKUP_SEARCH_DOCS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_DOC_PAGES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_VIEW_TASKS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_TIME_ENTRIES_WITHIN_A_DATE_RANGE", - scope: ToolScope::Read, - }, - // ── Read: members ───────────────────────────────────────────── - CuratedTool { - slug: "CLICKUP_GET_WORKSPACE_MEMBERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "CLICKUP_GET_TASK_MEMBERS", - scope: ToolScope::Read, - }, - // ── Write: create / update tasks ────────────────────────────── - CuratedTool { - slug: "CLICKUP_CREATE_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "CLICKUP_UPDATE_TASK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "CLICKUP_CREATE_TASK_COMMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "CLICKUP_UPDATE_COMMENT", - scope: ToolScope::Write, - }, - // ── Write: structure ────────────────────────────────────────── - CuratedTool { - slug: "CLICKUP_CREATE_LIST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "CLICKUP_UPDATE_LIST", - scope: ToolScope::Write, - }, - // ── Admin: destructive ──────────────────────────────────────── - CuratedTool { - slug: "CLICKUP_DELETE_TASK", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "CLICKUP_DELETE_COMMENT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "CLICKUP_DELETE_LIST", - scope: ToolScope::Admin, - }, -]; diff --git a/crates/tinymemory-core/src/sync/composio/providers/descriptions.rs b/crates/tinymemory-core/src/sync/composio/providers/descriptions.rs deleted file mode 100644 index 09468447..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/descriptions.rs +++ /dev/null @@ -1,61 +0,0 @@ -//! Human-readable capability summaries for Composio toolkit slugs. - -/// Human-readable capability summary for a Composio toolkit slug. -/// -/// Used by the prompt renderer to tell the orchestrator what each connected -/// integration can do. Covers the most common toolkits; unknown slugs get -/// a generic fallback so newly connected services still appear. -pub fn toolkit_description(slug: &str) -> &'static str { - match slug { - "gmail" => { - "Send, read, draft, reply, forward, and search emails; manage labels and threads" - } - "notion" => "Create, read, update, and search notion pages and notion databases", - "github" => { - "Manage repositories, issues, and pull requests on GitHub; sync \ - assigned issues into Memory Tree" - } - "slack" => "Send messages, read channels, manage threads, and post updates in Slack", - "discord" => "Send messages, manage channels, and interact with Discord servers", - "google_calendar" => "Create, update, and query calendar events; check availability", - "google_drive" => "Upload, download, search, and share files in Google Drive", - "google_docs" => "Create, read, and edit Google Docs documents", - "google_sheets" => "Read, write, and manage Google Sheets spreadsheets", - "outlook" => "Send, read, and manage emails in Microsoft Outlook", - "microsoft_teams" => "Send messages and manage channels in Microsoft Teams", - "larksuite" => { - "Connect Lark / Feishu workspace chat, docs, wiki, and meetings via Composio" - } - "linear" => { - "Create, read, and manage issues, projects, and cycles in Linear; sync \ - assigned issues into Memory Tree" - } - "jira" => "Create and manage issues, projects, and sprints in Jira", - "trello" => "Create and manage cards, lists, and boards in Trello", - "asana" => "Create and manage tasks, projects, and sections in Asana", - "clickup" => { - "Create, read, and manage tasks, lists, and docs in ClickUp; sync \ - assigned tasks into Memory Tree" - } - "dropbox" => "Upload, download, and share files in Dropbox", - "twitter" => "Post tweets, read timelines, and manage Twitter interactions", - "spotify" => "Control playback, search music, and manage playlists on Spotify", - "telegram" => "Send and receive messages via Telegram", - "whatsapp" => "Send and receive messages via WhatsApp", - "twilio" => "Send SMS, make calls, and manage communications via Twilio", - "shopify" => "Manage products, orders, and customers in Shopify", - "stripe" => "Manage payments, subscriptions, and customers in Stripe", - "hubspot" => "Manage contacts, deals, and marketing in HubSpot", - "salesforce" => "Manage contacts, leads, and opportunities in Salesforce", - "airtable" => "Read and write records in Airtable bases", - "figma" => "Access and manage Figma design files and components", - "youtube" => "Search videos, manage playlists, and interact with YouTube", - "calendar" => "Create, update, and query calendar events", - "one_drive" | "onedrive" => { - "Upload, download, search, and share files in Microsoft OneDrive" - } - "excel" => "Read, write, and manage workbooks, worksheets, and tables in Microsoft Excel", - "todoist" => "Create and manage tasks, projects, sections, and labels in Todoist", - _ => "Interact with this connected service via its available actions", - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/github/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/github/mod.rs deleted file mode 100644 index 89dbd5b4..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/github/mod.rs +++ /dev/null @@ -1,26 +0,0 @@ -//! GitHub Composio provider — incremental Memory Tree ingest for issues and -//! pull requests involving the connected user. -//! -//! Mirrors the [`crate::sync::composio::providers::clickup`] layout so -//! anyone familiar with ClickUp/Notion ingestion can read this without -//! re-learning a new shape: -//! -//! - `provider.rs` — `impl ComposioProvider for GitHubProvider` -//! - `normalization` — payload-shape helpers, now reached through -//! `tinymemory-sync` (issue #18 §B3) -//! - `tools.rs` — `GITHUB_CURATED` whitelist of Composio actions -//! - `tests.rs` — unit tests for the helpers + trait metadata -//! -//! Issue: #2408. - -// The payload normalisers moved to tinycortex (they are pure Value -// transforms, i.e. driver-side). Aliased under the old module name so -// every `normalization::extract_*` call site below stays unchanged. -use tinymemory_sync::github as normalization; -mod provider; -#[cfg(test)] -mod tests; -pub mod tools; - -pub use provider::GitHubProvider; -pub use tools::GITHUB_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/github/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/github/provider.rs deleted file mode 100644 index e01992a6..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/github/provider.rs +++ /dev/null @@ -1,547 +0,0 @@ -//! GitHub provider — incremental sync of issues and pull requests involving -//! the authenticated user, with per-item persistence into the Memory Tree. -//! -//! On each sync pass: -//! -//! 1. Load persistent [`SyncState`] from the KV store. -//! 2. Check the daily request budget — bail early if exhausted. -//! 3. Resolve the authenticated user's GitHub login (used in the search -//! query); cached cheaply across re-fetches. -//! 4. Search for issues and PRs involving the user via -//! `GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS` with `involves:{login}`, filtered to items -//! updated since the cursor (when available). -//! 5. For each result, persist as a single memory document if it's new -//! *or* edited since the last sync. -//! 6. Advance the cursor to the newest `updated_at` seen and save. -//! -//! Privacy posture: the `involves:` search qualifier returns only items the -//! user created, was assigned to, mentioned in, or commented on — it never -//! surfaces private repos the user can't access. This mirrors the -//! "fetch-what-the-user-sees" model gmail / notion already follow. - -use async_trait::async_trait; -use serde_json::{json, Value}; -use std::time::Duration; - -use super::normalization; -use crate::sync::composio::providers::{ - merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, - GithubFetchMode, NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, - TaskKind, -}; - -pub(crate) const ACTION_GET_AUTHENTICATED_USER: &str = "GITHUB_GET_THE_AUTHENTICATED_USER"; -pub(crate) const ACTION_SEARCH_ISSUES: &str = "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS"; - -const GH_CLI_TIMEOUT: Duration = Duration::from_secs(30); -const GITHUB_TASK_SEARCH_TIMEOUT: Duration = Duration::from_secs(20); - -pub struct GitHubProvider; - -impl GitHubProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for GitHubProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for GitHubProvider { - fn toolkit_slug(&self) -> &'static str { - "github" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(super::tools::GITHUB_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - // 30 minutes — GitHub issues change less frequently than Slack - // messages, so a half-hour cadence keeps the memory fresh without - // hammering the search API. - Some(resolve_sync_interval_secs("github", 30 * 60)) - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:github] fetch_user_profile via {ACTION_GET_AUTHENTICATED_USER}" - ); - - let resp = ctx - .execute(ACTION_GET_AUTHENTICATED_USER, Some(json!({}))) - .await - .map_err(|e| { - format!("[composio:github] {ACTION_GET_AUTHENTICATED_USER} failed: {e:#}") - })?; - - if !resp.successful { - let err = resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!( - "[composio:github] {ACTION_GET_AUTHENTICATED_USER}: {err}" - )); - } - - let data = &resp.data; - let login = normalization::extract_user_login(data); - let display_name = pick_str(data, &["name", "data.name"]).or_else(|| login.clone()); - let email = pick_str(data, &["email", "data.email"]); - let avatar_url = pick_str(data, &["avatar_url", "data.avatar_url"]); - let profile_url = pick_str(data, &["html_url", "data.html_url"]); - - Ok(ProviderUserProfile { - toolkit: "github".to_string(), - connection_id: ctx.connection_id.clone(), - display_name, - email, - username: login, - avatar_url, - profile_url, - extras: data.clone(), - }) - } - - /// Incremental sync via the generic - /// `orchestrator`: - /// login resolution, pagination, dedup, the `max_items` cap, and cursor - /// handling live in `run_sync`; the GitHub-specific primitives — including - /// the **server-side** `sync_depth_days` window — live in `super::source`. - async fn fetch_tasks( - &self, - ctx: &ProviderContext, - filter: &TaskFetchFilter, - ) -> Result, String> { - let max = filter.effective_max(); - let query = build_fetch_query(filter); - tracing::debug!( - connection_id = ?ctx.connection_id, - max, - mode = ?filter.github_fetch_mode, - query = %query, - "[composio:github] fetch_tasks" - ); - - // Select the data source by the user-configured fetch mode. `Auto` - // (the default) keeps the shipped Composio path as primary and treats - // local `gh`/REST as a true fallback — only used when the Composio - // round-trip errors or is unavailable. `Composio` / `Local` force one - // path. Normalization happens ONCE below regardless of source. - let data = match filter.github_fetch_mode { - GithubFetchMode::Composio => { - fetch_github_tasks_composio(ctx, &query, max, &filter.extra).await? - } - GithubFetchMode::Local => fetch_github_tasks_local(&query, max, &filter.extra).await?, - GithubFetchMode::Auto => { - match fetch_github_tasks_composio(ctx, &query, max, &filter.extra).await { - Ok(d) => d, - Err(e) => { - tracing::info!( - error = %e, - "[composio:github] Composio fetch unavailable; falling back to local gh/REST" - ); - fetch_github_tasks_local(&query, max, &filter.extra).await? - } - } - } - }; - - let mut out: Vec = Vec::new(); - for issue in normalization::extract_issues(&data) { - if out.len() >= max { - break; - } - if let Some(nt) = normalize_github_issue(&issue) { - out.push(nt); - } - } - tracing::debug!(count = out.len(), "[composio:github] fetch_tasks complete"); - Ok(out) - } -} - -/// Fetch GitHub issues/PRs through the connected Composio account. -/// -/// This is the original shipped `fetch_tasks` data path: it builds the -/// `GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS` search args, merges any advanced -/// `extra` query fragment, fires the action through the mode-aware -/// `ctx.execute` chokepoint, and returns the raw response `data` for the -/// shared normalization loop. Kept as a sibling of -/// [`fetch_github_tasks_local`] so `fetch_tasks` can select between them by -/// [`GithubFetchMode`]. -async fn fetch_github_tasks_composio( - ctx: &ProviderContext, - query: &str, - max: usize, - extra: &Value, -) -> Result { - let mut args = json!({ - "q": query, - "sort": "updated", - "order": "desc", - "per_page": max.min(100) as u32, - "page": 1, - }); - merge_extra(&mut args, extra); - - let resp = ctx - .execute(ACTION_SEARCH_ISSUES, Some(args)) - .await - .map_err(|e| format!("[composio:github] {ACTION_SEARCH_ISSUES}: {e:#}"))?; - if !resp.successful { - return Err(format!( - "[composio:github] {ACTION_SEARCH_ISSUES}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - Ok(resp.data) -} - -async fn fetch_github_tasks_local(query: &str, max: usize, extra: &Value) -> Result { - let mut args = json!({ - "q": query, - "sort": "updated", - "order": "desc", - "per_page": max.min(100) as u32, - "page": 1, - }); - merge_extra(&mut args, extra); - expand_me_in_github_search_args(&mut args).await; - - match gh_search_issues(&args).await { - Ok(data) => Ok(data), - Err(gh_err) => { - tracing::debug!( - error = %gh_err, - "[task_sources:github] gh api search failed, falling back to REST" - ); - rest_search_issues(&args).await.map_err(|rest_err| { - format!("[task_sources:github] local GitHub search failed: gh: {gh_err}; REST: {rest_err}") - }) - } - } -} - -async fn gh_search_issues(args: &Value) -> Result { - let mut cmd = tokio::process::Command::new("gh"); - cmd.arg("api") - .arg("--method") - .arg("GET") - .arg("search/issues"); - for (key, value) in github_search_arg_pairs(args)? { - cmd.arg("-f").arg(format!("{key}={value}")); - } - - let output = tokio::time::timeout(GH_CLI_TIMEOUT, cmd.output()) - .await - .map_err(|_| format!("gh command timed out after {}s", GH_CLI_TIMEOUT.as_secs()))? - .map_err(|e| format!("gh command failed: {e}"))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - return Err(format!("gh exited {}: {stderr}", output.status)); - } - - let stdout = - String::from_utf8(output.stdout).map_err(|e| format!("gh output not utf8: {e}"))?; - serde_json::from_str(&stdout).map_err(|e| format!("parse gh search response: {e}")) -} - -async fn rest_search_issues(args: &Value) -> Result { - let client = reqwest::Client::builder() - .timeout(GITHUB_TASK_SEARCH_TIMEOUT) - .build() - .map_err(|e| format!("failed to build GitHub client: {e}"))?; - - let mut request = client - .get("https://api.github.com/search/issues") - .header("User-Agent", "openhuman") - .header("Accept", "application/vnd.github+json"); - - if let Some(token) = github_env_token() { - request = request.header("Authorization", format!("Bearer {token}")); - } - - let pairs = github_search_arg_pairs(args)?; - let resp = request - .query(&pairs) - .send() - .await - .map_err(|e| format!("GitHub API request failed: {e}"))?; - - if !resp.status().is_success() { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - return Err(format!("GitHub API returned {status}: {body}")); - } - - resp.json::() - .await - .map_err(|e| format!("parse GitHub API response: {e}")) -} - -pub(super) fn github_env_token() -> Option { - std::env::var("GH_TOKEN") - .or_else(|_| std::env::var("GITHUB_TOKEN")) - .ok() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) -} - -async fn expand_me_in_github_search_args(args: &mut Value) { - let Some(query) = args.get("q").and_then(Value::as_str).map(str::to_string) else { - return; - }; - if !query.contains("@me") { - return; - } - let Some(login) = resolve_github_login().await else { - return; - }; - if let Some(obj) = args.as_object_mut() { - obj.insert("q".to_string(), Value::String(query.replace("@me", &login))); - } -} - -async fn resolve_github_login() -> Option { - if let Some(login) = resolve_github_login_with_gh().await { - return Some(login); - } - resolve_github_login_with_rest().await -} - -async fn resolve_github_login_with_gh() -> Option { - let output = tokio::time::timeout( - GH_CLI_TIMEOUT, - tokio::process::Command::new("gh") - .arg("api") - .arg("user") - .arg("--jq") - .arg(".login") - .output(), - ) - .await - .ok()? - .ok()?; - if !output.status.success() { - return None; - } - String::from_utf8(output.stdout) - .ok() - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) -} - -async fn resolve_github_login_with_rest() -> Option { - let token = github_env_token()?; - let client = reqwest::Client::builder() - .timeout(GITHUB_TASK_SEARCH_TIMEOUT) - .build() - .ok()?; - let resp = client - .get("https://api.github.com/user") - .header("User-Agent", "openhuman") - .header("Accept", "application/vnd.github+json") - .header("Authorization", format!("Bearer {token}")) - .send() - .await - .ok()?; - if !resp.status().is_success() { - return None; - } - resp.json::() - .await - .ok() - .and_then(|value| { - value - .get("login") - .and_then(Value::as_str) - .map(str::to_string) - }) - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) -} - -pub(super) fn github_search_arg_pairs(args: &Value) -> Result, String> { - let obj = args - .as_object() - .ok_or_else(|| "GitHub search args must be a JSON object".to_string())?; - let mut out = Vec::with_capacity(obj.len()); - for (key, value) in obj { - let rendered = match value { - Value::String(s) => s.trim().to_string(), - Value::Number(n) => n.to_string(), - Value::Bool(b) => b.to_string(), - Value::Null => continue, - other => other.to_string(), - }; - if !rendered.is_empty() { - out.push((key.clone(), rendered)); - } - } - Ok(out) -} - -/// Build a GitHub Search-Issues query from a [`TaskFetchFilter`]. -/// -/// Combines repo / label / state / assignee qualifiers. When the filter -/// carries no scoping constraints at all we fall back to `involves:@me` so a -/// task source never accidentally pulls the entire public issue universe. -/// -/// State bias: when the filter sets no explicit `state`, we append `is:open` -/// so closed issues and merged/closed PRs aren't fetched in the first place -/// (the unconditional skip in `normalize_github_issue` is the hard guarantee; -/// this is the fetch-side optimization). An explicit `state` is respected and -/// `is:open` is not double-added. -pub(super) fn build_fetch_query(filter: &TaskFetchFilter) -> String { - let mut parts: Vec = Vec::new(); - if let Some(repo) = filter - .repo - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(normalize_github_repo_filter) - { - parts.push(format!("repo:{repo}")); - } - for label in filter - .labels - .iter() - .map(|l| l.trim()) - .filter(|l| !l.is_empty()) - { - parts.push(format!("label:\"{label}\"")); - } - if filter.assignee_is_me { - parts.push("assignee:@me".to_string()); - } - // If no repo/label/assignee scoping was supplied, fall back to - // `involves:@me` (plus the open bias) rather than the whole issue universe. - if parts.is_empty() { - parts.push("involves:@me".to_string()); - } - let explicit_state = filter - .state - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - match explicit_state { - // Caller pinned a state — respect it verbatim, don't add `is:open`. - Some(state) => parts.push(format!("state:{state}")), - // No explicit state — bias the fetch toward open items. - None => parts.push("is:open".to_string()), - } - parts.join(" ") -} - -pub(super) fn normalize_github_repo_filter(raw: &str) -> String { - let trimmed = raw.trim(); - let without_scheme = trimmed - .strip_prefix("https://github.com/") - .or_else(|| trimmed.strip_prefix("http://github.com/")) - .or_else(|| trimmed.strip_prefix("git@github.com:")) - .unwrap_or(trimmed); - let cleaned = without_scheme - .trim_start_matches('/') - .trim_end_matches('/') - .trim_end_matches(".git"); - let mut parts = cleaned.split('/').filter(|part| !part.is_empty()); - match (parts.next(), parts.next()) { - (Some(owner), Some(repo)) => { - let repo = repo.trim_end_matches(".git"); - if owner.is_empty() || repo.is_empty() { - trimmed.to_string() - } else { - format!("{owner}/{repo}") - } - } - _ => trimmed.to_string(), - } -} - -/// Map a raw GitHub issue/PR payload into a [`NormalizedTask`]. -/// -/// GitHub's search-issues-and-PRs endpoint returns both shapes; a hit is a -/// pull request iff it carries a `pull_request` object. We tag the kind here -/// so enrichment can phrase the objective as "review" vs "resolve". -/// -/// Returns `None` when the item's state is `"closed"` — a merged/closed PR -/// and a closed issue both report `state == "closed"`, and there is no point -/// ingesting work that is already done. This skip is unconditional (it does -/// not depend on the fetch query), so even if a `closed` item slips through -/// the query bias it is dropped here. -pub(super) fn normalize_github_issue(issue: &serde_json::Value) -> Option { - let external_id = normalization::extract_issue_id(issue)?; - let status = pick_str(issue, &["state", "data.state"]); - if status - .as_deref() - .map(|s| s.eq_ignore_ascii_case("closed")) - .unwrap_or(false) - { - tracing::debug!( - external_id = %external_id, - "[composio:github] normalize_github_issue: skipping closed item (merged PR / closed issue)" - ); - return None; - } - let title = normalization::extract_issue_title(issue) - .unwrap_or_else(|| format!("GitHub issue {external_id}")); - let kind = if is_pull_request(issue) { - TaskKind::PullRequest - } else { - TaskKind::Issue - }; - Some(NormalizedTask { - external_id, - source_id: String::new(), - provider: "github".to_string(), - kind, - title, - body: pick_str(issue, &["body", "data.body"]), - url: pick_str(issue, &["html_url", "data.html_url"]), - status, - assignee: pick_str(issue, &["assignee.login", "data.assignee.login"]), - due: None, - labels: extract_github_labels(issue), - priority: None, - updated_at: normalization::extract_issue_updated_at(issue), - raw: issue.clone(), - }) -} - -/// A GitHub search hit is a pull request iff it carries a non-null -/// `pull_request` object (issues never do). Tolerant of the Composio `data` -/// wrapper. -fn is_pull_request(issue: &serde_json::Value) -> bool { - let pr = issue - .get("pull_request") - .or_else(|| issue.get("data").and_then(|d| d.get("pull_request"))); - matches!(pr, Some(v) if !v.is_null()) -} - -/// Extract label names from a GitHub issue payload (`labels` is an array -/// of `{ name }` objects). Tolerant of the Composio `data` wrapper. -fn extract_github_labels(issue: &serde_json::Value) -> Vec { - let arr = issue - .get("labels") - .or_else(|| issue.get("data").and_then(|d| d.get("labels"))) - .and_then(|v| v.as_array()); - match arr { - Some(items) => items - .iter() - .filter_map(|l| l.get("name").and_then(|n| n.as_str())) - .map(|s| s.to_string()) - .collect(), - None => Vec::new(), - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/github/tests.rs b/crates/tinymemory-core/src/sync/composio/providers/github/tests.rs deleted file mode 100644 index 10ee9625..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/github/tests.rs +++ /dev/null @@ -1,611 +0,0 @@ -//! Unit tests for the GitHub Composio provider. - -use super::normalization::{ - extract_issue_id, extract_issue_title, extract_issue_updated_at, extract_issues, - extract_user_login, -}; -use super::provider::github_env_token; -use super::provider::{ - build_fetch_query, github_search_arg_pairs, normalize_github_issue, - normalize_github_repo_filter, ACTION_GET_AUTHENTICATED_USER, ACTION_SEARCH_ISSUES, -}; -use super::tools::GITHUB_CURATED; -use super::GitHubProvider; -use crate::sync::composio::providers::ComposioProvider; -use crate::sync::composio::providers::{GithubFetchMode, TaskFetchFilter, TaskKind}; -use serde_json::json; - -// ── extract_issues ─────────────────────────────────────────────────────────── - -#[test] -fn extract_issues_walks_data_items_shape() { - let data = json!({ "data": { "items": [{"id": 1u64}] } }); - assert_eq!(extract_issues(&data).len(), 1); -} - -#[test] -fn extract_issues_walks_top_level_items_shape() { - let data = json!({ "items": [{"id": 1u64}, {"id": 2u64}] }); - assert_eq!(extract_issues(&data).len(), 2); -} - -#[test] -fn extract_issues_returns_empty_when_no_items_key() { - let data = json!({ "foo": "bar" }); - assert!(extract_issues(&data).is_empty()); -} - -#[test] -fn extract_issues_handles_data_data_nesting() { - let data = json!({ "data": { "data": { "items": [{"id": 9u64}] } } }); - assert_eq!(extract_issues(&data).len(), 1); -} - -// ── extract_issue_id ───────────────────────────────────────────────────────── - -#[test] -fn extract_issue_id_from_numeric_id() { - let issue = json!({ "id": 123456789u64, "title": "Fix race" }); - assert_eq!(extract_issue_id(&issue), Some("123456789".to_string())); -} - -#[test] -fn extract_issue_id_from_wrapped_data() { - let issue = json!({ "data": { "id": 42u64 } }); - assert_eq!(extract_issue_id(&issue), Some("42".to_string())); -} - -#[test] -fn extract_issue_id_falls_back_to_html_url_path() { - let issue = json!({ - "html_url": "https://github.com/owner/repo/issues/7" - }); - assert_eq!(extract_issue_id(&issue), Some("owner/repo#7".to_string())); -} - -#[test] -fn extract_issue_id_none_when_no_id_or_url() { - let issue = json!({ "title": "orphan" }); - assert!(extract_issue_id(&issue).is_none()); -} - -// ── extract_issue_title ────────────────────────────────────────────────────── - -#[test] -fn extract_issue_title_builds_prefixed_title() { - let issue = json!({ - "id": 1u64, - "title": "Fix race condition", - "html_url": "https://github.com/acme/core/issues/99" - }); - assert_eq!( - extract_issue_title(&issue), - Some("GitHub: acme/core#99: Fix race condition".to_string()) - ); -} - -#[test] -fn extract_issue_title_pr_url_also_works() { - let issue = json!({ - "id": 2u64, - "title": "Add feature", - "html_url": "https://github.com/org/repo/pull/101" - }); - assert_eq!( - extract_issue_title(&issue), - Some("GitHub: org/repo#101: Add feature".to_string()) - ); -} - -#[test] -fn extract_issue_title_returns_raw_title_when_no_url() { - let issue = json!({ "title": "Bare title" }); - assert_eq!(extract_issue_title(&issue), Some("Bare title".to_string())); -} - -#[test] -fn extract_issue_title_none_when_no_title() { - let issue = json!({ "id": 1u64 }); - assert!(extract_issue_title(&issue).is_none()); -} - -// ── extract_issue_updated_at ───────────────────────────────────────────────── - -#[test] -fn extract_issue_updated_at_from_top_level() { - let issue = json!({ "updated_at": "2024-05-21T15:30:00Z" }); - assert_eq!( - extract_issue_updated_at(&issue), - Some("2024-05-21T15:30:00Z".to_string()) - ); -} - -#[test] -fn extract_issue_updated_at_from_data_wrapper() { - let issue = json!({ "data": { "updated_at": "2023-01-01T00:00:00Z" } }); - assert_eq!( - extract_issue_updated_at(&issue), - Some("2023-01-01T00:00:00Z".to_string()) - ); -} - -#[test] -fn extract_issue_updated_at_none_when_missing() { - let issue = json!({ "id": 1u64 }); - assert!(extract_issue_updated_at(&issue).is_none()); -} - -// ── extract_user_login ─────────────────────────────────────────────────────── - -#[test] -fn extract_user_login_from_top_level() { - let data = json!({ "login": "octocat" }); - assert_eq!(extract_user_login(&data), Some("octocat".to_string())); -} - -#[test] -fn extract_user_login_from_data_wrapper() { - let data = json!({ "data": { "login": "monalisa" } }); - assert_eq!(extract_user_login(&data), Some("monalisa".to_string())); -} - -#[test] -fn extract_user_login_none_when_missing() { - let data = json!({ "id": 1u64 }); - assert!(extract_user_login(&data).is_none()); -} - -// ── provider metadata ──────────────────────────────────────────────────────── - -#[test] -fn provider_metadata_is_stable() { - let p = GitHubProvider::new(); - assert_eq!(p.toolkit_slug(), "github"); - assert_eq!(p.sync_interval_secs(), Some(30 * 60)); - assert!(p.curated_tools().is_some()); -} - -#[test] -fn curated_tools_contains_core_actions() { - let p = GitHubProvider::new(); - let curated = p.curated_tools().expect("GITHUB_CURATED is registered"); - let slugs: Vec<&str> = curated.iter().map(|t| t.slug).collect(); - assert!(slugs.contains(&"GITHUB_GET_THE_AUTHENTICATED_USER")); - assert!(slugs.contains(&"GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS")); - assert!(slugs.contains(&"GITHUB_LIST_REPOSITORY_ISSUES")); - assert!(slugs.contains(&"GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER")); - assert!(slugs.contains(&"GITHUB_CREATE_A_REPOSITORY_FOR_THE_AUTHENTICATED_USER")); - // DELETE_A_REFERENCE replaces DELETE_A_BRANCH (Composio v3 rename). - assert!(slugs.contains(&"GITHUB_DELETE_A_REFERENCE")); - // CLOSE_AN_ISSUE was removed — callers must use UPDATE_AN_ISSUE with state:"closed". - assert!( - !slugs.contains(&"GITHUB_CLOSE_AN_ISSUE"), - "GITHUB_CLOSE_AN_ISSUE was removed — use GITHUB_UPDATE_AN_ISSUE with state:closed" - ); -} - -#[test] -fn default_impl_matches_new() { - let a = GitHubProvider::new(); - let b = ::default(); - assert_eq!(a.toolkit_slug(), b.toolkit_slug()); - assert_eq!(a.sync_interval_secs(), b.sync_interval_secs()); - assert_eq!( - a.curated_tools().map(<[_]>::len), - b.curated_tools().map(<[_]>::len), - ); -} - -#[test] -fn build_fetch_query_scopes_repo_labels_state_and_assignee() { - let query = build_fetch_query(&TaskFetchFilter { - repo: Some("tinyhumansai/openhuman".to_string()), - labels: vec!["bug".to_string(), "agent harness".to_string()], - state: Some("open".to_string()), - assignee_is_me: true, - ..Default::default() - }); - - assert_eq!( - query, - "repo:tinyhumansai/openhuman label:\"bug\" label:\"agent harness\" assignee:@me state:open" - ); -} - -#[test] -fn build_fetch_query_normalizes_github_repo_urls() { - let query = build_fetch_query(&TaskFetchFilter { - repo: Some("https://github.com/tinyhumansai/openhuman/pull/3267".to_string()), - state: Some("open".to_string()), - ..Default::default() - }); - - assert_eq!(query, "repo:tinyhumansai/openhuman state:open"); -} - -#[test] -fn normalize_github_repo_filter_accepts_common_repo_inputs() { - assert_eq!( - normalize_github_repo_filter("tinyhumansai/openhuman"), - "tinyhumansai/openhuman" - ); - assert_eq!( - normalize_github_repo_filter("https://github.com/tinyhumansai/openhuman.git"), - "tinyhumansai/openhuman" - ); - assert_eq!( - normalize_github_repo_filter("git@github.com:tinyhumansai/openhuman.git"), - "tinyhumansai/openhuman" - ); -} - -#[test] -fn build_fetch_query_falls_back_to_involves_me_when_unscoped() { - // No scoping and no explicit state: fall back to `involves:@me` and bias - // toward open items so closed issues / merged PRs aren't even fetched. - assert_eq!( - build_fetch_query(&TaskFetchFilter::default()), - "involves:@me is:open" - ); -} - -#[test] -fn build_fetch_query_appends_is_open_when_no_explicit_state() { - // Scoped by repo but no explicit state — `is:open` is appended. - let query = build_fetch_query(&TaskFetchFilter { - repo: Some("tinyhumansai/openhuman".to_string()), - ..Default::default() - }); - assert_eq!(query, "repo:tinyhumansai/openhuman is:open"); -} - -#[test] -fn build_fetch_query_respects_explicit_state_without_double_open() { - // Explicit `state` is respected verbatim and `is:open` is NOT added. - let query = build_fetch_query(&TaskFetchFilter { - repo: Some("tinyhumansai/openhuman".to_string()), - state: Some("closed".to_string()), - ..Default::default() - }); - assert_eq!(query, "repo:tinyhumansai/openhuman state:closed"); - assert!(!query.contains("is:open")); -} - -#[test] -fn github_search_arg_pairs_render_cli_and_rest_params() { - let args = json!({ - "q": "repo:tinyhumansai/openhuman state:open", - "sort": "updated", - "order": "desc", - "per_page": 25, - "page": 1, - "include_prs": true, - "skip": null - }); - - let pairs = github_search_arg_pairs(&args).expect("pairs"); - assert!(pairs.contains(&( - "q".to_string(), - "repo:tinyhumansai/openhuman state:open".to_string() - ))); - assert!(pairs.contains(&("per_page".to_string(), "25".to_string()))); - assert!(pairs.contains(&("include_prs".to_string(), "true".to_string()))); - assert!(!pairs.iter().any(|(key, _)| key == "skip")); -} - -// ── slug regression tests (#2768) ─────────────────────────────────────────── -// -// Guard the current Composio action slug values used by the GitHub provider. -// Outdated slugs (e.g. GITHUB_USERS_GET_AUTHENTICATED, GITHUB_LIST_REPOS, -// GITHUB_LIST_ISSUES) were previously scattered across tests; these assertions -// pin the correct values in one place so a slug rename is caught immediately. - -#[test] -fn action_get_authenticated_user_slug_is_current() { - // The Composio v3 slug is GITHUB_GET_THE_AUTHENTICATED_USER. - // Regression: was mistakenly referenced as GITHUB_USERS_GET_AUTHENTICATED - // in tests (see issue #2768). - assert_eq!( - ACTION_GET_AUTHENTICATED_USER, "GITHUB_GET_THE_AUTHENTICATED_USER", - "slug must match Composio v3 catalog; old slug GITHUB_USERS_GET_AUTHENTICATED is retired" - ); -} - -#[test] -fn action_search_issues_slug_is_current() { - assert_eq!( - ACTION_SEARCH_ISSUES, "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS", - "slug must match Composio v3 catalog" - ); -} - -#[test] -fn curated_list_does_not_contain_retired_slugs() { - // Guard against re-introducing removed slugs that no longer exist in the - // Composio v3 GitHub app catalog. - const RETIRED: &[&str] = &[ - "GITHUB_USERS_GET_AUTHENTICATED", // replaced by GITHUB_GET_THE_AUTHENTICATED_USER - "GITHUB_LIST_REPOS", // replaced by GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER - "GITHUB_LIST_ISSUES", // replaced by GITHUB_LIST_REPOSITORY_ISSUES - "GITHUB_COMMIT_MULTIPLE_FILES", // removed from Composio catalog - "GITHUB_CLOSE_AN_ISSUE", // removed; use GITHUB_UPDATE_AN_ISSUE with state=closed - "GITHUB_DELETE_A_BRANCH", // removed; use GITHUB_DELETE_A_REFERENCE - ]; - - let slugs: Vec<&str> = GITHUB_CURATED.iter().map(|t| t.slug).collect(); - for retired in RETIRED { - assert!( - !slugs.contains(retired), - "curated list must not contain retired slug {retired} (see #2768)" - ); - } -} - -#[test] -fn curated_list_contains_current_read_slugs() { - // Verify that the primary read-tier actions are present with their correct - // v3 slug names (not the old v1/v2 names). - let slugs: Vec<&str> = GITHUB_CURATED.iter().map(|t| t.slug).collect(); - let required = [ - "GITHUB_GET_THE_AUTHENTICATED_USER", - "GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER", - "GITHUB_LIST_REPOSITORY_ISSUES", - "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS", - "GITHUB_LIST_PULL_REQUESTS", - "GITHUB_GET_A_PULL_REQUEST", - ]; - for slug in required { - assert!( - slugs.contains(&slug), - "curated list must contain current slug {slug} (see #2768)" - ); - } -} - -#[test] -fn curated_list_contains_current_write_slugs() { - let slugs: Vec<&str> = GITHUB_CURATED.iter().map(|t| t.slug).collect(); - let required = [ - "GITHUB_CREATE_AN_ISSUE", - "GITHUB_UPDATE_AN_ISSUE", - "GITHUB_CREATE_A_PULL_REQUEST", - "GITHUB_MERGE_A_PULL_REQUEST", - ]; - for slug in required { - assert!( - slugs.contains(&slug), - "curated list must contain current write slug {slug} (see #2768)" - ); - } -} - -// ── GithubFetchMode (#3279) ───────────────────────────────────────────────── -// -// The fetch-mode selector makes the local `gh`/REST path a *true fallback* -// (default `Auto`) instead of a hard Composio replacement. These tests pin the -// default and the serde wire contract the UI persists into a source's -// `FilterSpec::Github { fetch_mode }`. - -#[test] -fn github_fetch_mode_defaults_to_auto() { - // `Auto` must be the default so shipped Composio users keep working and - // local/dev setups still get the fallback — neither side regresses. - assert_eq!(GithubFetchMode::default(), GithubFetchMode::Auto); -} - -#[test] -fn task_fetch_filter_default_uses_auto_fetch_mode() { - // A filter built with no explicit mode (the common path) carries `Auto`. - let filter = TaskFetchFilter::default(); - assert_eq!(filter.github_fetch_mode, GithubFetchMode::Auto); -} - -#[test] -fn github_fetch_mode_serializes_snake_case() { - assert_eq!( - serde_json::to_value(GithubFetchMode::Auto).expect("ser auto"), - json!("auto") - ); - assert_eq!( - serde_json::to_value(GithubFetchMode::Composio).expect("ser composio"), - json!("composio") - ); - assert_eq!( - serde_json::to_value(GithubFetchMode::Local).expect("ser local"), - json!("local") - ); -} - -#[test] -fn github_fetch_mode_deserializes_each_variant() { - let auto: GithubFetchMode = serde_json::from_value(json!("auto")).expect("de auto"); - let composio: GithubFetchMode = serde_json::from_value(json!("composio")).expect("de composio"); - let local: GithubFetchMode = serde_json::from_value(json!("local")).expect("de local"); - assert_eq!(auto, GithubFetchMode::Auto); - assert_eq!(composio, GithubFetchMode::Composio); - assert_eq!(local, GithubFetchMode::Local); -} - -#[test] -fn github_fetch_mode_round_trips_through_json() { - for mode in [ - GithubFetchMode::Auto, - GithubFetchMode::Composio, - GithubFetchMode::Local, - ] { - let json = serde_json::to_string(&mode).expect("ser"); - let back: GithubFetchMode = serde_json::from_str(&json).expect("de"); - assert_eq!(back, mode, "round-trip must preserve {mode:?}"); - } -} - -#[test] -fn github_fetch_mode_rejects_unknown_variant() { - let parsed: Result = serde_json::from_value(json!("remote")); - assert!(parsed.is_err(), "unknown mode strings must fail to parse"); -} - -// ── github_search_arg_pairs edge cases (#3279) ────────────────────────────── - -#[test] -fn github_search_arg_pairs_skips_null_and_empty_string_values() { - // Null values are dropped entirely; whitespace-only / empty strings are - // trimmed to empty and also dropped, so they never reach the gh CLI / REST - // query as blank params. - let args = json!({ - "q": "involves:@me", - "empty": "", - "blank": " ", - "missing": null, - "page": 1, - }); - let pairs = github_search_arg_pairs(&args).expect("pairs"); - assert!(pairs.contains(&("q".to_string(), "involves:@me".to_string()))); - assert!(pairs.contains(&("page".to_string(), "1".to_string()))); - assert!(!pairs.iter().any(|(k, _)| k == "empty")); - assert!(!pairs.iter().any(|(k, _)| k == "blank")); - assert!(!pairs.iter().any(|(k, _)| k == "missing")); -} - -#[test] -fn github_search_arg_pairs_errors_when_not_an_object() { - // A non-object value (array, scalar) is a programmer error — surface it - // rather than silently producing an empty arg set. - let err = github_search_arg_pairs(&json!(["not", "an", "object"])) - .expect_err("array args must error"); - assert!(err.contains("JSON object"), "got: {err}"); -} - -// ── github_env_token (#3279) ──────────────────────────────────────────────── - -#[test] -fn github_env_token_reads_env_and_is_none_when_unset() { - // Env-mutation test: this whole suite is the only reader of GH_TOKEN / - // GITHUB_TOKEN, but cargo runs tests in parallel threads sharing the - // process env. Hold a process-wide lock so concurrent token reads don't - // race, and restore the original values on exit. - use std::sync::Mutex; - static ENV_LOCK: Mutex<()> = Mutex::new(()); - let _guard = ENV_LOCK.lock().unwrap_or_else(|e| e.into_inner()); - - let prev_gh = std::env::var("GH_TOKEN").ok(); - let prev_github = std::env::var("GITHUB_TOKEN").ok(); - - // Neither set → None. - std::env::remove_var("GH_TOKEN"); - std::env::remove_var("GITHUB_TOKEN"); - assert_eq!(github_env_token(), None, "no token vars → None"); - - // GH_TOKEN takes precedence; surrounding whitespace is trimmed. - std::env::set_var("GH_TOKEN", " gh-pat-123 "); - assert_eq!(github_env_token().as_deref(), Some("gh-pat-123")); - - // Falls back to GITHUB_TOKEN when GH_TOKEN is absent. - std::env::remove_var("GH_TOKEN"); - std::env::set_var("GITHUB_TOKEN", "github-pat-456"); - assert_eq!(github_env_token().as_deref(), Some("github-pat-456")); - - // A blank token is treated as unset. - std::env::set_var("GH_TOKEN", " "); - std::env::remove_var("GITHUB_TOKEN"); - assert_eq!(github_env_token(), None, "blank token → None"); - - // Restore original env. - match prev_gh { - Some(v) => std::env::set_var("GH_TOKEN", v), - None => std::env::remove_var("GH_TOKEN"), - } - match prev_github { - Some(v) => std::env::set_var("GITHUB_TOKEN", v), - None => std::env::remove_var("GITHUB_TOKEN"), - } -} - -// ── issue vs pull-request kind detection ───────────────────────────────────── - -#[test] -fn normalize_tags_pull_request_when_pull_request_object_present() { - // GitHub's issues-and-PRs search marks a PR hit with a `pull_request` object. - let pr = json!({ - "id": 42, - "title": "Add retry to fetch", - "state": "open", - "html_url": "https://github.com/o/r/pull/42", - "pull_request": { "url": "https://api.github.com/repos/o/r/pulls/42" } - }); - let nt = normalize_github_issue(&pr).expect("normalizes"); - assert_eq!(nt.kind, TaskKind::PullRequest); -} - -#[test] -fn normalize_tags_issue_when_no_pull_request_object() { - let issue = json!({ - "id": 7, - "title": "Login throws on empty password", - "state": "open", - "html_url": "https://github.com/o/r/issues/7" - }); - let nt = normalize_github_issue(&issue).expect("normalizes"); - assert_eq!(nt.kind, TaskKind::Issue); -} - -#[test] -fn normalize_tags_issue_when_pull_request_is_null() { - // The REST issue payload carries `pull_request: null` for plain issues. - let issue = json!({ - "id": 8, - "title": "Docs typo", - "state": "open", - "html_url": "https://github.com/o/r/issues/8", - "pull_request": null - }); - let nt = normalize_github_issue(&issue).expect("normalizes"); - assert_eq!(nt.kind, TaskKind::Issue); -} - -// ── skip merged PRs / closed issues ────────────────────────────────────────── - -#[test] -fn normalize_skips_closed_issue() { - // A closed issue is already-done work — drop it. - let issue = json!({ - "id": 100, - "title": "Old bug", - "state": "closed", - "html_url": "https://github.com/o/r/issues/100" - }); - assert!( - normalize_github_issue(&issue).is_none(), - "closed issue must be skipped" - ); -} - -#[test] -fn normalize_skips_merged_or_closed_pull_request() { - // A merged/closed PR also reports `state == "closed"` — drop it too. - let pr = json!({ - "id": 101, - "title": "Shipped feature", - "state": "closed", - "html_url": "https://github.com/o/r/pull/101", - "pull_request": { "url": "https://api.github.com/repos/o/r/pulls/101" } - }); - assert!( - normalize_github_issue(&pr).is_none(), - "merged/closed PR must be skipped" - ); -} - -#[test] -fn normalize_keeps_open_item() { - // An open item is kept and tagged with its kind. - let issue = json!({ - "id": 102, - "title": "Active work", - "state": "open", - "html_url": "https://github.com/o/r/issues/102" - }); - let nt = normalize_github_issue(&issue).expect("open item is kept"); - assert_eq!(nt.kind, TaskKind::Issue); - assert_eq!(nt.status.as_deref(), Some("open")); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/github/tools.rs b/crates/tinymemory-core/src/sync/composio/providers/github/tools.rs deleted file mode 100644 index 40be9137..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/github/tools.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! Curated catalog of GitHub Composio actions exposed to the agent. -//! -//! Composio publishes hundreds of GitHub actions; this hand-tuned slice -//! covers the day-to-day operations an AI assistant actually performs -//! (browsing repos, reading/writing issues + PRs, code search, basic -//! workflow control) and hides the long tail of admin endpoints. - -use crate::sync::composio::providers::tool_scope::{CuratedTool, ToolScope}; - -pub const GITHUB_CURATED: &[CuratedTool] = &[ - // ── Read: user / repos ────────────────────────────────────────── - CuratedTool { - slug: "GITHUB_GET_THE_AUTHENTICATED_USER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_LIST_REPOSITORIES_FOR_THE_AUTHENTICATED_USER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_GET_A_REPOSITORY", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_LIST_REPOSITORY_COLLABORATORS", - scope: ToolScope::Read, - }, - // ── Read: search ──────────────────────────────────────────────── - CuratedTool { - slug: "GITHUB_SEARCH_REPOSITORIES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_SEARCH_CODE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_SEARCH_USERS", - scope: ToolScope::Read, - }, - // ── Read: issues ──────────────────────────────────────────────── - CuratedTool { - slug: "GITHUB_LIST_REPOSITORY_ISSUES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_GET_AN_ISSUE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_LIST_ISSUE_COMMENTS", - scope: ToolScope::Read, - }, - // ── Read: pull requests ───────────────────────────────────────── - CuratedTool { - slug: "GITHUB_LIST_PULL_REQUESTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_GET_A_PULL_REQUEST", - scope: ToolScope::Read, - }, - // CuratedTool { slug: "GITHUB_CHECK_IF_PULL_REQUEST_HAS_BEEN_MERGED", scope: ToolScope::Read }, - // ── Read: branches / commits ──────────────────────────────────── - CuratedTool { - slug: "GITHUB_LIST_BRANCHES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_GET_A_BRANCH", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_LIST_COMMITS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GITHUB_GET_A_COMMIT", - scope: ToolScope::Read, - }, - // CuratedTool { slug: "GITHUB_COMPARE_TWO_COMMITS", scope: ToolScope::Read }, - // // ── Read: contents / releases / gists ─────────────────────────── - // CuratedTool { slug: "GITHUB_GET_REPOSITORY_CONTENTS", scope: ToolScope::Read }, - // CuratedTool { slug: "GITHUB_LIST_RELEASES", scope: ToolScope::Read }, - // CuratedTool { slug: "GITHUB_LIST_GISTS", scope: ToolScope::Read }, - // // ── Read: workflows ───────────────────────────────────────────── - // CuratedTool { slug: "GITHUB_LIST_WORKFLOWS", scope: ToolScope::Read }, - // CuratedTool { slug: "GITHUB_LIST_WORKFLOW_RUNS", scope: ToolScope::Read }, - // ── Write: repos / contents ───────────────────────────────────── - CuratedTool { - slug: "GITHUB_CREATE_A_REPOSITORY_FOR_THE_AUTHENTICATED_USER", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GITHUB_CREATE_OR_UPDATE_FILE_CONTENTS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GITHUB_CREATE_A_COMMIT", - scope: ToolScope::Write, - }, - // GITHUB_COMMIT_MULTIPLE_FILES removed from Composio catalog - CuratedTool { - slug: "GITHUB_CREATE_A_COMMIT_COMMENT", - scope: ToolScope::Write, - }, - // ── Write: issues ─────────────────────────────────────────────── - CuratedTool { - slug: "GITHUB_CREATE_AN_ISSUE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GITHUB_UPDATE_AN_ISSUE", - scope: ToolScope::Write, - }, - // GITHUB_CLOSE_AN_ISSUE — removed: no dedicated Composio slug. - // Use GITHUB_UPDATE_AN_ISSUE with state:"closed" instead. - CuratedTool { - slug: "GITHUB_CREATE_AN_ISSUE_COMMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GITHUB_ADD_LABELS_TO_AN_ISSUE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GITHUB_ADD_ASSIGNEES_TO_AN_ISSUE", - scope: ToolScope::Write, - }, - // ── Write: pull requests ──────────────────────────────────────── - CuratedTool { - slug: "GITHUB_CREATE_A_PULL_REQUEST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GITHUB_UPDATE_A_PULL_REQUEST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GITHUB_MERGE_A_PULL_REQUEST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GITHUB_CREATE_A_REVIEW_FOR_A_PULL_REQUEST", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GITHUB_CREATE_A_REVIEW_COMMENT_FOR_A_PULL_REQUEST", - scope: ToolScope::Write, - }, - // // ── Write: releases / gists / workflows ───────────────────────── - // CuratedTool { slug: "GITHUB_CREATE_A_RELEASE", scope: ToolScope::Write }, - CuratedTool { - slug: "GITHUB_CREATE_A_GIST", - scope: ToolScope::Write, - }, - // CuratedTool { slug: "GITHUB_CREATE_WORKFLOW_DISPATCH", scope: ToolScope::Write }, - // ── Admin: destructive / permission-changing ──────────────────── - CuratedTool { - slug: "GITHUB_DELETE_A_REPOSITORY", - scope: ToolScope::Admin, - }, - // DELETE_A_REFERENCE maps to DELETE /repos/{owner}/{repo}/git/refs/{ref}. - // The ref must be a full path (e.g. `refs/heads/branch-name` or - // `refs/tags/v1.0`) — passing a bare branch name deletes nothing (404). - // This replaces the old GITHUB_DELETE_A_BRANCH slug (Composio v3 rename); - // it is broader — it can delete tags too — so agents should always specify - // a `refs/heads/` prefix when the intent is branch deletion. - CuratedTool { - slug: "GITHUB_DELETE_A_REFERENCE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GITHUB_DELETE_A_FILE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GITHUB_ADD_A_REPOSITORY_COLLABORATOR", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GITHUB_CANCEL_A_WORKFLOW_RUN", - scope: ToolScope::Admin, - }, -]; diff --git a/crates/tinymemory-core/src/sync/composio/providers/gmail/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/gmail/mod.rs deleted file mode 100644 index df067ec2..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/gmail/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -// The Gmail post-processor moved to tinycortex (a pure Value transform, i.e. -// driver-side). Aliased under the old module name so the single call site in -// `provider.rs` stays unchanged. -use tinymemory_sync::gmail_post_process as post_process; -mod provider; -#[cfg(test)] -mod tests; -pub mod tools; - -pub use provider::GmailProvider; -pub use tools::GMAIL_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/gmail/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/gmail/provider.rs deleted file mode 100644 index 701d56f7..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/gmail/provider.rs +++ /dev/null @@ -1,177 +0,0 @@ -//! Gmail provider — incremental sync into the memory tree. -//! -//! On each sync pass: -//! -//! 1. Load persistent [`SyncState`] from the KV store. -//! 2. Check the daily request budget — bail early if exhausted. -//! 3. Fetch a page of recent messages via `GMAIL_FETCH_EMAILS`, adding -//! a date filter when a cursor exists so only newer mail is returned. -//! 4. Run [`ComposioProvider::post_process_action_result`] (bounded -//! HTML→text, normalise, sanitise) on the page so the LLM-facing chunk -//! content is cleaned, not raw. -//! 5. Delegate incremental filtering and document ingestion to tinycortex. -//! 6. Paginate (up to budget) until no more results or all items in the -//! page are already synced. -//! 7. Advance the cursor and save state. -//! -//! Daily budget (`DEFAULT_DAILY_REQUEST_LIMIT`, default 500) caps the -//! number of `execute_tool` calls per calendar day, preventing runaway -//! API usage during large initial backfills. - -use async_trait::async_trait; -use serde_json::{json, Value}; - -use crate::sync::composio::providers::{ - pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, ProviderContext, - ProviderUserProfile, -}; - -pub(super) const ACTION_GET_PROFILE: &str = "GMAIL_GET_PROFILE"; - -pub struct GmailProvider; - -impl GmailProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for GmailProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for GmailProvider { - fn toolkit_slug(&self) -> &'static str { - "gmail" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(super::tools::GMAIL_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - Some(resolve_sync_interval_secs("gmail", 15 * 60)) - } - - fn post_process_action_result( - &self, - slug: &str, - arguments: Option<&serde_json::Value>, - data: &mut serde_json::Value, - ) { - super::post_process::post_process(slug, arguments, data); - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:gmail] fetch_user_profile via {ACTION_GET_PROFILE}" - ); - - let resp = ctx - .execute(ACTION_GET_PROFILE, Some(json!({}))) - .await - .map_err(|e| format!("[composio:gmail] {ACTION_GET_PROFILE} failed: {e:#}"))?; - - if !resp.successful { - let err = resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!("[composio:gmail] {ACTION_GET_PROFILE}: {err}")); - } - - // `data` is the inner Composio payload — paths here are relative - // to it. (The previous `data.*` paths were dead — `pick_str` - // does dotted-path traversal, so `data.emailAddress` looked for - // a nested `data.data.emailAddress` that never exists.) - let data = &resp.data; - let email = pick_str(data, &["emailAddress", "email", "profile.emailAddress"]); - // Don't fall back to the email when no name is returned — that - // produces duplicated `display_name == email` rows in the - // identity registry (#1365). Gmail's `GMAIL_GET_PROFILE` action - // doesn't return a name today, so this stays None. - let display_name = pick_str(data, &["name", "profile.name", "displayName"]); - let profile_url = pick_str( - data, - &["display_url", "profileUrl", "profile_url", "profile.url"], - ); - - let profile = ProviderUserProfile { - toolkit: "gmail".to_string(), - connection_id: ctx.connection_id.clone(), - display_name, - email, - username: None, - avatar_url: None, - profile_url, - extras: data.clone(), - }; - let has_email = profile.email.is_some(); - let email_domain = profile - .email - .as_deref() - .and_then(|e| e.split('@').nth(1)) - .map(|d| d.to_string()); - tracing::info!( - connection_id = ?profile.connection_id, - has_email, - email_domain = ?email_domain, - "[composio:gmail] fetched user profile" - ); - Ok(profile) - } - - /// Incremental sync via the generic - /// `orchestrator`: - /// pagination, dedup, the `max_items` cap, and cursor handling live in - /// `run_sync`; the Gmail-specific primitives — the account-email preamble, - /// server-side `after:` depth window, adaptive page ceiling, all-synced - /// stop, and batch ingest — live in `super::source`. - async fn on_trigger( - &self, - ctx: &ProviderContext, - trigger: &str, - _payload: &Value, - ) -> Result<(), String> { - tracing::info!( - connection_id = ?ctx.connection_id, - trigger = %trigger, - "[composio:gmail] on_trigger" - ); - - if trigger.eq_ignore_ascii_case("GMAIL_NEW_GMAIL_MESSAGE") - || trigger.eq_ignore_ascii_case("GMAIL_NEW_MESSAGE") - { - let Some(connection_id) = ctx.connection_id.as_deref() else { - return Err("[composio:gmail] trigger missing connection_id".to_string()); - }; - if let Err(e) = crate::sync::pipelines::host::run_composio_connection( - "gmail", - connection_id, - ctx.config.as_ref(), - None, - None, - ) - .await - { - tracing::warn!( - error = %e, - "[composio:gmail] trigger-driven sync failed (non-fatal)" - ); - } - } - Ok(()) - } -} - -// Message fetching (the `GMAIL_FETCH_EMAILS` action, the search query, the -// `max_items` cap math and the `sync_depth_days` `after:` floor) is owned -// by `crate::sync::pipelines::composio::GmailSyncPipeline`. What stays here is the -// host-side provider surface: profile lookup and trigger dispatch. diff --git a/crates/tinymemory-core/src/sync/composio/providers/gmail/tests.rs b/crates/tinymemory-core/src/sync/composio/providers/gmail/tests.rs deleted file mode 100644 index 4c6554b8..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/gmail/tests.rs +++ /dev/null @@ -1,29 +0,0 @@ -//! Host-owned Gmail provider surface tests. -//! -//! Pagination, cursor, envelope parsing, and ingest behavior are owned and -//! tested by `crate::sync::pipelines::composio::GmailSyncPipeline`. - -use super::GmailProvider; -use crate::sync::composio::providers::ComposioProvider; - -#[test] -fn provider_metadata_is_stable() { - let provider = GmailProvider::new(); - assert_eq!(provider.toolkit_slug(), "gmail"); - assert_eq!(provider.sync_interval_secs(), Some(15 * 60)); -} - -#[test] -fn default_impl_matches_new() { - let _new = GmailProvider::new(); - let _default = ::default(); -} - -#[test] -fn provider_source_does_not_restrict_to_inbox() { - let source = include_str!("provider.rs"); - assert!( - !source.contains("\"in:inbox"), - "provider query must not exclude sent mail" - ); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/gmail/tools.rs b/crates/tinymemory-core/src/sync/composio/providers/gmail/tools.rs deleted file mode 100644 index ac14e106..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/gmail/tools.rs +++ /dev/null @@ -1,145 +0,0 @@ -//! Curated catalog of Gmail Composio actions exposed to the agent. -//! -//! Composio publishes 60+ Gmail actions; this hand-tuned slice covers -//! the cases the agent actually plans for (read, compose, manage) and -//! hides the long tail of edge-case admin endpoints. - -use crate::sync::composio::providers::tool_scope::{CuratedTool, ToolScope}; - -pub const GMAIL_CURATED: &[CuratedTool] = &[ - // ── Read: messages & threads ──────────────────────────────────── - CuratedTool { - slug: "GMAIL_FETCH_EMAILS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GMAIL_LIST_MESSAGES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GMAIL_FETCH_MESSAGE_BY_MESSAGE_ID", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GMAIL_FETCH_MESSAGE_BY_THREAD_ID", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GMAIL_LIST_THREADS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GMAIL_GET_ATTACHMENT", - scope: ToolScope::Read, - }, - // ── Read: profile & settings ──────────────────────────────────── - CuratedTool { - slug: "GMAIL_GET_PROFILE", - scope: ToolScope::Read, - }, - // CuratedTool { slug: "GMAIL_GET_LANGUAGE_SETTINGS", scope: ToolScope::Read }, - // CuratedTool { slug: "GMAIL_GET_VACATION_SETTINGS", scope: ToolScope::Read }, - // CuratedTool { slug: "GMAIL_GET_AUTO_FORWARDING", scope: ToolScope::Read }, - // ── Read: contacts & people ───────────────────────────────────── - CuratedTool { - slug: "GMAIL_GET_CONTACTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GMAIL_GET_PEOPLE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GMAIL_SEARCH_PEOPLE", - scope: ToolScope::Read, - }, - // ── Read: drafts & labels ─────────────────────────────────────── - CuratedTool { - slug: "GMAIL_LIST_DRAFTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GMAIL_GET_DRAFT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GMAIL_LIST_LABELS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "GMAIL_GET_LABEL", - scope: ToolScope::Read, - }, - // ── Write: send & compose ─────────────────────────────────────── - CuratedTool { - slug: "GMAIL_SEND_EMAIL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GMAIL_REPLY_TO_THREAD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GMAIL_FORWARD_MESSAGE", - scope: ToolScope::Write, - }, - // ── Write: drafts ─────────────────────────────────────────────── - CuratedTool { - slug: "GMAIL_CREATE_EMAIL_DRAFT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GMAIL_UPDATE_DRAFT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "GMAIL_SEND_DRAFT", - scope: ToolScope::Write, - }, - // ── Write: labels (create/update on user labels) ──────────────── - // CuratedTool { slug: "GMAIL_CREATE_LABEL", scope: ToolScope::Write }, - // CuratedTool { slug: "GMAIL_UPDATE_LABEL", scope: ToolScope::Write }, - // CuratedTool { slug: "GMAIL_PATCH_LABEL", scope: ToolScope::Write }, - CuratedTool { - slug: "GMAIL_ADD_LABEL_TO_EMAIL", - scope: ToolScope::Write, - }, - // ── Admin: destructive & permission-changing ──────────────────── - CuratedTool { - slug: "GMAIL_DELETE_MESSAGE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GMAIL_BATCH_DELETE_MESSAGES", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GMAIL_MOVE_TO_TRASH", - scope: ToolScope::Admin, - }, - // CuratedTool { slug: "GMAIL_UNTRASH_MESSAGE", scope: ToolScope::Admin }, - CuratedTool { - slug: "GMAIL_DELETE_THREAD", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GMAIL_MOVE_THREAD_TO_TRASH", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GMAIL_UNTRASH_THREAD", - scope: ToolScope::Admin, - }, - // CuratedTool { slug: "GMAIL_MODIFY_THREAD_LABELS", scope: ToolScope::Admin }, - // CuratedTool { slug: "GMAIL_BATCH_MODIFY_MESSAGES", scope: ToolScope::Admin }, - CuratedTool { - slug: "GMAIL_DELETE_DRAFT", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "GMAIL_DELETE_LABEL", - scope: ToolScope::Admin, - }, - // CuratedTool { slug: "GMAIL_PATCH_SEND_AS", scope: ToolScope::Admin }, - // CuratedTool { slug: "GMAIL_UPDATE_IMAP_SETTINGS", scope: ToolScope::Admin }, -]; diff --git a/crates/tinymemory-core/src/sync/composio/providers/helpers.rs b/crates/tinymemory-core/src/sync/composio/providers/helpers.rs deleted file mode 100644 index e0358e20..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/helpers.rs +++ /dev/null @@ -1,56 +0,0 @@ -//! Shared helpers for Composio provider implementations. -//! -//! `pick_str` used to live here. It is a provider payload normaliser, so it -//! moved to `tinymemory_sync::helpers` -//! and is re-exported from this module's parent. The helpers that remain are -//! request-building rather than normalisation, and stay host-side. - -use tinymemory_sync::helpers::pick_str; - -/// Shallow-merge an `extra` JSON object into a (mutable) action-args -/// object. Only object-typed extras are merged; non-object `extra` -/// values are ignored. Backs the `task_sources` advanced free-form -/// filter escape hatch — provider `fetch_tasks` impls call this to fold -/// user-supplied provider-native query fragments into their request -/// arguments. -pub(crate) fn merge_extra(args: &mut serde_json::Value, extra: &serde_json::Value) { - if let (Some(args_obj), Some(extra_obj)) = (args.as_object_mut(), extra.as_object()) { - for (k, v) in extra_obj { - args_obj.insert(k.clone(), v.clone()); - } - } -} - -/// Resolve the first array found among `array_paths` (dotted object -/// paths), then return the first non-empty string at one of `fields` -/// on that array's first element. Complements [`pick_str`], which -/// cannot index into arrays. Used to pull e.g. the first assignee's -/// username out of an `assignees` array. -pub(crate) fn first_array_str( - value: &serde_json::Value, - array_paths: &[&str], - fields: &[&str], -) -> Option { - for path in array_paths { - let mut cur = value; - let mut ok = true; - for segment in path.split('.') { - match cur.get(segment) { - Some(next) => cur = next, - None => { - ok = false; - break; - } - } - } - if !ok { - continue; - } - if let Some(first) = cur.as_array().and_then(|a| a.first()) { - if let Some(found) = pick_str(first, fields) { - return Some(found); - } - } - } - None -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/linear/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/linear/mod.rs deleted file mode 100644 index 7ccce87c..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/linear/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Linear Composio provider — incremental Memory Tree ingest for -//! issues assigned to the connected user. -//! -//! Issue: #2400. - -// The payload normalisers moved to tinycortex (they are pure Value -// transforms, i.e. driver-side). Aliased under the old module name so -// every `normalization::extract_*` call site below stays unchanged. -use tinymemory_sync::linear as normalization; -mod provider; -#[cfg(test)] -mod tests; -pub mod tools; - -pub use provider::LinearProvider; -pub use tools::LINEAR_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/linear/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/linear/provider.rs deleted file mode 100644 index 3ee95846..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/linear/provider.rs +++ /dev/null @@ -1,241 +0,0 @@ -//! Linear provider — incremental sync of issues assigned to the -//! authenticated user, with per-issue memory_tree ingest. -//! -//! On each sync pass: -//! -//! 1. Load persistent [`SyncState`] from the KV store. -//! 2. Check the daily request budget — bail early if exhausted. -//! 3. Resolve the viewer ID via `LINEAR_LIST_LINEAR_USERS { isMe: true }`. -//! 4. Page through `LINEAR_LIST_LINEAR_ISSUES` filtered to the viewer as -//! assignee, ordered by `updatedAt` descending. Stop early once we hit -//! issues older than the cursor or a page without a next-page cursor. -//! 5. For each issue, ingest into memory_tree if it's new *or* edited -//! since the last sync. -//! 6. Advance the cursor to the newest `updatedAt` seen and save. -//! -//! Privacy posture: we only pull issues the user is assigned to, never -//! the whole workspace's issue graph. This mirrors the -//! "fetch-what-the-user-sees" model `gmail` / `notion` already follow -//! and avoids accidentally ingesting other teammates' private issues. - -use async_trait::async_trait; -use serde_json::json; - -use super::normalization; -use crate::sync::composio::providers::{ - merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, - NormalizedTask, ProviderContext, ProviderUserProfile, TaskFetchFilter, TaskKind, -}; - -pub(super) const ACTION_LIST_USERS: &str = "LINEAR_LIST_LINEAR_USERS"; -pub(super) const ACTION_LIST_ISSUES: &str = "LINEAR_LIST_LINEAR_ISSUES"; - -/// Paths for extracting a Linear issue's unique ID. -pub(super) const ISSUE_ID_PATHS: &[&str] = &["id", "data.id", "identifier", "data.identifier"]; - -pub struct LinearProvider; - -impl LinearProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for LinearProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for LinearProvider { - fn toolkit_slug(&self) -> &'static str { - "linear" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(super::tools::LINEAR_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - // 30 minutes — same cadence as ClickUp/Notion. Linear issues change - // more slowly than chat but faster than email. - Some(resolve_sync_interval_secs("linear", 30 * 60)) - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:linear] fetch_user_profile via {ACTION_LIST_USERS}" - ); - - let resp = ctx - .execute(ACTION_LIST_USERS, Some(json!({ "isMe": true }))) - .await - .map_err(|e| format!("[composio:linear] {ACTION_LIST_USERS} failed: {e:#}"))?; - - if !resp.successful { - let err = resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!("[composio:linear] {ACTION_LIST_USERS}: {err}")); - } - - let data = &resp.data; - let viewer = normalization::extract_viewer(data); - let viewer_ref = viewer.as_ref().unwrap_or(data); - - let display_name = pick_str(viewer_ref, &["name", "data.name", "displayName"]); - let email = pick_str(viewer_ref, &["email", "data.email"]); - let username = pick_str(viewer_ref, &["id", "data.id"]); - let avatar_url = pick_str(viewer_ref, &["avatarUrl", "data.avatarUrl"]); - let profile_url = pick_str(viewer_ref, &["url", "data.url"]); - - Ok(ProviderUserProfile { - toolkit: "linear".to_string(), - connection_id: ctx.connection_id.clone(), - display_name, - email, - username, - avatar_url, - profile_url, - extras: data.clone(), - }) - } - - /// Incremental sync via the generic - /// `orchestrator`: - /// viewer resolution, pagination, dedup, the `max_items` cap, the - /// `sync_depth_days` window, and cursor handling live in `run_sync`; the - /// Linear-specific primitives live in `super::source`. - async fn fetch_tasks( - &self, - ctx: &ProviderContext, - filter: &TaskFetchFilter, - ) -> Result, String> { - let max = filter.effective_max(); - tracing::debug!( - connection_id = ?ctx.connection_id, - max, - team_id = ?filter.team_id, - assignee_is_me = filter.assignee_is_me, - "[composio:linear] fetch_tasks" - ); - - let mut args = json!({ - "first": max.min(100) as u64, - "orderBy": "updatedAt", - }); - if filter.assignee_is_me { - let resp = ctx - .execute(ACTION_LIST_USERS, Some(json!({ "isMe": true }))) - .await - .map_err(|e| format!("[composio:linear] {ACTION_LIST_USERS}: {e:#}"))?; - // Fail closed: a failed viewer lookup must not silently widen - // the query beyond "assigned to me". - if !resp.successful { - return Err(format!( - "[composio:linear] {ACTION_LIST_USERS}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - let viewer_id = normalization::extract_viewer_id(&resp.data).ok_or_else(|| { - "[composio:linear] LINEAR_LIST_LINEAR_USERS returned no viewer id".to_string() - })?; - args["assigneeId"] = json!(viewer_id); - } - if let Some(team) = filter - .team_id - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()) - { - args["teamId"] = json!(team); - } - merge_extra(&mut args, &filter.extra); - - let resp = ctx - .execute(ACTION_LIST_ISSUES, Some(args)) - .await - .map_err(|e| format!("[composio:linear] {ACTION_LIST_ISSUES}: {e:#}"))?; - if !resp.successful { - return Err(format!( - "[composio:linear] {ACTION_LIST_ISSUES}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - - let want_state = filter - .state - .as_deref() - .map(|s| s.trim().to_ascii_lowercase()) - .filter(|s| !s.is_empty()); - - let mut out: Vec = Vec::new(); - for issue in normalization::extract_issues(&resp.data) { - if out.len() >= max { - break; - } - let Some(nt) = normalize_linear_issue(&issue) else { - continue; - }; - if let Some(ref want) = want_state { - let matches = nt - .status - .as_deref() - .map(|s| s.to_ascii_lowercase() == *want) - .unwrap_or(false); - if !matches { - continue; - } - } - out.push(nt); - } - tracing::debug!(count = out.len(), "[composio:linear] fetch_tasks complete"); - Ok(out) - } -} - -/// Map a raw Linear issue payload into a [`NormalizedTask`]. -fn normalize_linear_issue(issue: &serde_json::Value) -> Option { - let external_id = pick_str(issue, ISSUE_ID_PATHS)?; - let title = normalization::extract_issue_title(issue) - .unwrap_or_else(|| format!("Linear issue {external_id}")); - Some(NormalizedTask { - external_id, - source_id: String::new(), - provider: "linear".to_string(), - kind: TaskKind::Generic, - title, - body: pick_str(issue, &["description", "data.description"]), - url: pick_str(issue, &["url", "data.url"]), - status: pick_str(issue, &["state.name", "data.state.name", "state.type"]), - assignee: pick_str(issue, &["assignee.name", "data.assignee.name"]), - due: pick_str(issue, &["dueDate", "data.dueDate"]), - labels: extract_linear_labels(issue), - priority: pick_str(issue, &["priorityLabel", "data.priorityLabel"]), - updated_at: normalization::extract_issue_updated(issue), - raw: issue.clone(), - }) -} - -/// Extract label names from a Linear issue (`labels.nodes[].name`). -fn extract_linear_labels(issue: &serde_json::Value) -> Vec { - let arr = issue - .get("labels") - .or_else(|| issue.get("data").and_then(|d| d.get("labels"))) - .and_then(|l| l.get("nodes")) - .and_then(|v| v.as_array()); - match arr { - Some(items) => items - .iter() - .filter_map(|l| l.get("name").and_then(|n| n.as_str())) - .map(|s| s.to_string()) - .collect(), - None => Vec::new(), - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/linear/tests.rs b/crates/tinymemory-core/src/sync/composio/providers/linear/tests.rs deleted file mode 100644 index c84ace80..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/linear/tests.rs +++ /dev/null @@ -1,172 +0,0 @@ -//! Unit tests for the Linear provider. - -use super::normalization::{ - extract_issue_title, extract_issue_updated, extract_issues, extract_pagination_cursor, - extract_viewer, extract_viewer_id, -}; -use super::LinearProvider; -use crate::sync::composio::providers::ComposioProvider; -use serde_json::json; - -// ── extract_issues ─────────────────────────────────────────────────── - -#[test] -fn extract_issues_walks_common_shapes() { - let v1 = json!({ "data": { "nodes": [{"id": "i1"}] } }); - let v2 = json!({ "nodes": [{"id": "i2"}, {"id": "i3"}] }); - let v3 = json!({ "data": { "issues": { "nodes": [{"id": "i4"}] } } }); - let v4 = json!({ "foo": "bar" }); - assert_eq!(extract_issues(&v1).len(), 1); - assert_eq!(extract_issues(&v2).len(), 2); - assert_eq!(extract_issues(&v3).len(), 1); - assert_eq!(extract_issues(&v4).len(), 0); -} - -// ── extract_issue_title ────────────────────────────────────────────── - -#[test] -fn extract_issue_title_finds_title_field() { - let issue = json!({ "id": "i1", "title": "Fix the login bug" }); - assert_eq!( - extract_issue_title(&issue), - Some("Fix the login bug".into()) - ); -} - -#[test] -fn extract_issue_title_falls_back_to_wrapped_data() { - let issue = json!({ "data": { "title": "Wrapped issue" } }); - assert_eq!(extract_issue_title(&issue), Some("Wrapped issue".into())); -} - -#[test] -fn extract_issue_title_falls_back_to_identifier() { - let issue = json!({ "identifier": "ENG-99" }); - assert_eq!(extract_issue_title(&issue), Some("ENG-99".into())); -} - -// ── extract_issue_updated ──────────────────────────────────────────── - -#[test] -fn extract_issue_updated_handles_camel_case() { - let issue = json!({ "updatedAt": "2026-03-01T12:00:00.000Z" }); - assert_eq!( - extract_issue_updated(&issue), - Some("2026-03-01T12:00:00.000Z".to_string()) - ); -} - -#[test] -fn extract_issue_updated_handles_wrapped_data() { - let issue = json!({ "data": { "updatedAt": "2026-01-15T08:30:00.000Z" } }); - assert_eq!( - extract_issue_updated(&issue), - Some("2026-01-15T08:30:00.000Z".to_string()) - ); -} - -// ── extract_viewer ─────────────────────────────────────────────────── - -#[test] -fn extract_viewer_finds_first_node() { - let data = json!({ "data": { "nodes": [{ "id": "usr_1", "email": "a@b.com" }] } }); - let v = extract_viewer(&data).expect("viewer found"); - assert_eq!(v["id"], "usr_1"); -} - -#[test] -fn extract_viewer_from_top_level_nodes() { - let data = json!({ "nodes": [{ "id": "usr_2" }] }); - let v = extract_viewer(&data).expect("viewer found"); - assert_eq!(v["id"], "usr_2"); -} - -#[test] -fn extract_viewer_fallback_direct_object() { - let data = json!({ "id": "usr_direct", "name": "Alice" }); - let v = extract_viewer(&data).expect("viewer found"); - assert_eq!(v["id"], "usr_direct"); -} - -#[test] -fn extract_viewer_returns_none_when_absent() { - let data = json!({ "foo": "bar" }); - assert!(extract_viewer(&data).is_none()); -} - -// ── extract_pagination_cursor ──────────────────────────────────────── - -#[test] -fn extract_pagination_cursor_returns_cursor_on_has_next_page() { - let data = json!({ - "data": { - "pageInfo": { "hasNextPage": true, "endCursor": "abc123" } - } - }); - assert_eq!(extract_pagination_cursor(&data), Some("abc123".to_string())); -} - -#[test] -fn extract_pagination_cursor_returns_none_on_last_page() { - let data = json!({ - "pageInfo": { "hasNextPage": false, "endCursor": "xyz" } - }); - assert!(extract_pagination_cursor(&data).is_none()); -} - -#[test] -fn extract_pagination_cursor_returns_none_when_absent() { - let data = json!({ "nodes": [{"id": "i1"}] }); - assert!(extract_pagination_cursor(&data).is_none()); -} - -// ── extract_viewer_id ──────────────────────────────────────────────── - -#[test] -fn extract_viewer_id_from_data_nodes() { - let data = json!({ "data": { "nodes": [{ "id": "usr_abc" }] } }); - assert_eq!(extract_viewer_id(&data), Some("usr_abc".to_string())); -} - -#[test] -fn extract_viewer_id_returns_none_when_absent() { - let data = json!({ "foo": "bar" }); - assert!(extract_viewer_id(&data).is_none()); -} - -// ── provider metadata ──────────────────────────────────────────────── - -#[test] -fn provider_metadata_is_stable() { - let p = LinearProvider::new(); - assert_eq!(p.toolkit_slug(), "linear"); - assert_eq!(p.sync_interval_secs(), Some(30 * 60)); - assert!(p.curated_tools().is_some()); -} - -#[test] -fn curated_tools_contains_core_sync_surface() { - let p = LinearProvider::new(); - let curated = p.curated_tools().expect("LINEAR_CURATED is registered"); - let slugs: Vec<&str> = curated.iter().map(|t| t.slug).collect(); - assert!( - slugs.contains(&"LINEAR_LIST_LINEAR_USERS"), - "LINEAR_LIST_LINEAR_USERS must be in curated catalog" - ); - assert!( - slugs.contains(&"LINEAR_LIST_LINEAR_ISSUES"), - "LINEAR_LIST_LINEAR_ISSUES must be in curated catalog" - ); -} - -#[test] -fn default_impl_matches_new() { - let a = LinearProvider::new(); - let b = ::default(); - assert_eq!(a.toolkit_slug(), b.toolkit_slug()); - assert_eq!(a.sync_interval_secs(), b.sync_interval_secs()); - assert_eq!( - a.curated_tools().map(<[_]>::len), - b.curated_tools().map(<[_]>::len), - ); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/linear/tools.rs b/crates/tinymemory-core/src/sync/composio/providers/linear/tools.rs deleted file mode 100644 index 9eb61b5f..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/linear/tools.rs +++ /dev/null @@ -1,90 +0,0 @@ -//! Curated catalog of Linear Composio actions. - -use crate::sync::composio::providers::tool_scope::{CuratedTool, ToolScope}; - -pub const LINEAR_CURATED: &[CuratedTool] = &[ - CuratedTool { - slug: "LINEAR_LIST_LINEAR_USERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "LINEAR_LIST_LINEAR_ISSUES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "LINEAR_GET_LINEAR_ISSUE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "LINEAR_SEARCH_ISSUES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "LINEAR_LIST_LINEAR_TEAMS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "LINEAR_LIST_LINEAR_PROJECTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "LINEAR_GET_LINEAR_PROJECT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "LINEAR_LIST_LINEAR_STATES", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "LINEAR_GET_CYCLES_BY_TEAM_ID", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "LINEAR_LIST_LINEAR_LABELS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "LINEAR_CREATE_LINEAR_ISSUE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "LINEAR_UPDATE_ISSUE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "LINEAR_CREATE_LINEAR_COMMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "LINEAR_UPDATE_LINEAR_COMMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "LINEAR_CREATE_ATTACHMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "LINEAR_CREATE_ISSUE_RELATION", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "LINEAR_CREATE_LINEAR_PROJECT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "LINEAR_UPDATE_LINEAR_PROJECT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "LINEAR_CREATE_LINEAR_LABEL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "LINEAR_DELETE_LINEAR_ISSUE", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "LINEAR_REMOVE_ISSUE_LABEL", - scope: ToolScope::Admin, - }, -]; diff --git a/crates/tinymemory-core/src/sync/composio/providers/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/mod.rs deleted file mode 100644 index 9637d302..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/mod.rs +++ /dev/null @@ -1,615 +0,0 @@ -//! Provider-specific code for Composio toolkits. -//! -//! Each Composio toolkit (gmail, notion, slack, …) can register a -//! [`ComposioProvider`] implementation that knows how to: -//! -//! * Fetch a normalized **user profile** for a connected account. -//! * Run an **initial / periodic sync** that pulls fresh data from the -//! upstream service via the backend-proxied -//! `ComposioClient`. -//! * React to **trigger webhooks** that arrive over the -//! `composio:trigger` Socket.IO bridge. -//! * React to **OAuth handoff completion** so the very first sync can -//! run as soon as a user connects an account. -//! -//! Providers are pure Rust — there is no JS sandbox involved. They are -//! the native counterpart to the QuickJS skill bundles in -//! `tinyhumansai/openhuman-skills`, but specialized for Composio's API -//! surface and run inside the core process directly. -//! -//! ## Registry & dispatch -//! -//! The [`registry`] module owns a process-global `HashMap>`. The composio event bus subscriber -//! (`super::bus::ComposioTriggerSubscriber`) and the periodic sync -//! task both look up providers by toolkit slug and call into them. -//! -//! ## Why a trait, not a giant `match` -//! -//! Each provider has provider-specific shapes (gmail returns -//! emailAddress + messagesTotal, notion returns workspaces + pages, …) -//! and a different idea of what "sync" means. A trait keeps each -//! provider's implementation isolated, individually testable, and -//! easy to add without touching the dispatch layer. - -mod descriptions; -pub(crate) mod helpers; -mod scope_lookup; -pub mod tool_scope; -mod traits; -mod types; -pub mod user_scopes; - -pub mod catalogs; -pub mod catalogs_business; -pub mod catalogs_google; -pub mod catalogs_messaging; -pub mod catalogs_microsoft; -pub mod catalogs_productivity; -pub mod catalogs_social_media; -pub mod clickup; -pub mod github; -pub mod gmail; -pub mod linear; -pub mod notion; -pub mod profile; -pub mod profile_md; -pub mod registry; -pub mod slack; -pub mod sync_state; - -use crate::composio_host::ComposioCapability; - -const CAPABILITY_TOOLKITS: &[&str] = &[ - "gmail", - "notion", - "slack", - "clickup", - "github", - "discord", - "googlecalendar", - "googledrive", - "googledocs", - "googlesheets", - "outlook", - "microsoft_teams", - "linear", - "jira", - "trello", - "asana", - "dropbox", - "twitter", - "spotify", - "telegram", - "whatsapp", - "shopify", - "stripe", - "hubspot", - "salesforce", - "airtable", - "figma", - "youtube", - "one_drive", - "excel", - "todoist", -]; - -fn native_provider_sync_interval(toolkit: &str) -> Option { - match toolkit { - "gmail" => Some(gmail::GmailProvider::new().sync_interval_secs()), - "notion" => Some(notion::NotionProvider::new().sync_interval_secs()), - "slack" => Some(slack::SlackProvider::new().sync_interval_secs()), - "clickup" => Some(clickup::ClickUpProvider::new().sync_interval_secs()), - "github" => Some(github::GitHubProvider::new().sync_interval_secs()), - "linear" => Some(linear::LinearProvider::new().sync_interval_secs()), - _ => None, - } - .flatten() -} - -fn has_native_provider(toolkit: &str) -> bool { - matches!( - toolkit, - "gmail" | "notion" | "slack" | "clickup" | "github" | "linear" - ) -} - -/// Static overview of the Composio integrations supported by this core build. -/// -/// This deliberately does not consult the live Composio backend/direct tenant: -/// it is an observability surface for OpenHuman's own capability tiers. Use -/// `composio_list_toolkits` / `composio_list_connections` when callers need -/// the currently signed-in user's allowlist or OAuth state. -pub fn capability_matrix() -> Vec { - CAPABILITY_TOOLKITS - .iter() - .map(|toolkit| { - let native_provider = has_native_provider(toolkit); - let catalog = catalog_for_toolkit(toolkit); - let sync_interval_secs = native_provider_sync_interval(toolkit); - ComposioCapability { - toolkit: (*toolkit).to_string(), - description: toolkit_description(toolkit).to_string(), - native_provider, - curated_tools: catalog.is_some(), - curated_tool_count: catalog.map_or(0, <[CuratedTool]>::len), - tool_execution: catalog.is_some(), - user_profile: native_provider, - initial_sync: native_provider, - periodic_sync: sync_interval_secs.is_some(), - sync_interval_secs, - trigger_webhooks: native_provider, - memory_ingest: native_provider, - } - }) - .collect() -} - -/// Static toolkit → curated catalog map. -/// -/// This is consulted by the meta-tool layer alongside any registered -/// provider's [`ComposioProvider::curated_tools`]. It lets toolkits -/// without a full native provider still benefit from curated -/// whitelisting. -/// -/// Lookup key is the lowercased prefix returned by -/// [`toolkit_from_slug`] applied to the action slug — e.g. -/// `GOOGLECALENDAR_CREATE_EVENT` → `"googlecalendar"`. Multi-segment -/// prefixes like `MICROSOFT_TEAMS_*` return their known toolkit slug. -/// Synchronous visibility check for a Composio action slug given a -/// pre-loaded user scope preference. -/// -/// Returns `true` if the action should appear in the agent's tool -/// surface — i.e. it's in the toolkit's curated whitelist (or the -/// toolkit has no curation) **and** the user's scope pref allows its -/// classification. Falls back to [`classify_unknown`] for un-curated -/// toolkits. -/// -/// Use this when the user pref has already been loaded for the -/// toolkit (typical inside a `for slug in toolkits {...}` loop where -/// awaiting once per toolkit is cheaper than once per action). -pub fn is_action_visible_with_pref(slug: &str, pref: &UserScopePref) -> bool { - let Some(toolkit) = toolkit_from_slug(slug) else { - return true; - }; - let catalog = get_provider(&toolkit) - .and_then(|p| p.curated_tools()) - .or_else(|| catalog_for_toolkit(&toolkit)); - match catalog { - Some(catalog) => match find_curated(catalog, slug) { - Some(curated) => pref.allows(curated.scope), - None => false, - }, - None => pref.allows(classify_unknown(slug)), - } -} - -pub fn catalog_for_toolkit(toolkit: &str) -> Option<&'static [CuratedTool]> { - match toolkit.trim().to_ascii_lowercase().as_str() { - // Native providers - "gmail" => Some(gmail::GMAIL_CURATED), - "notion" => Some(notion::NOTION_CURATED), - "github" => Some(github::GITHUB_CURATED), - "linear" => Some(linear::LINEAR_CURATED), - // Catalog-only toolkits - "slack" => Some(catalogs::SLACK_CURATED), - "discord" => Some(catalogs::DISCORD_CURATED), - "googlecalendar" | "google_calendar" => Some(catalogs::GOOGLECALENDAR_CURATED), - "googledrive" | "google_drive" => Some(catalogs::GOOGLEDRIVE_CURATED), - "googledocs" | "google_docs" => Some(catalogs::GOOGLEDOCS_CURATED), - "googlesheets" | "google_sheets" => Some(catalogs::GOOGLESHEETS_CURATED), - "outlook" => Some(catalogs::OUTLOOK_CURATED), - // Keep the legacy "microsoft" alias while toolkit_from_slug now - // returns the precise "microsoft_teams" slug for Teams actions. - "microsoft" | "microsoft_teams" => Some(catalogs::MICROSOFT_TEAMS_CURATED), - "jira" => Some(catalogs::JIRA_CURATED), - "trello" => Some(catalogs::TRELLO_CURATED), - "asana" => Some(catalogs::ASANA_CURATED), - "clickup" => Some(clickup::CLICKUP_CURATED), - "dropbox" => Some(catalogs::DROPBOX_CURATED), - "twitter" => Some(catalogs::TWITTER_CURATED), - "spotify" => Some(catalogs::SPOTIFY_CURATED), - "telegram" => Some(catalogs::TELEGRAM_CURATED), - "whatsapp" => Some(catalogs::WHATSAPP_CURATED), - "shopify" => Some(catalogs::SHOPIFY_CURATED), - "stripe" => Some(catalogs::STRIPE_CURATED), - "hubspot" => Some(catalogs::HUBSPOT_CURATED), - "salesforce" => Some(catalogs::SALESFORCE_CURATED), - "airtable" => Some(catalogs::AIRTABLE_CURATED), - "figma" => Some(catalogs::FIGMA_CURATED), - "youtube" => Some(catalogs::YOUTUBE_CURATED), - // ONE_DRIVE_* slugs extract to "one" via toolkit_from_slug; - // alias both the prefix and the canonical UI/backend slugs. - "one" | "one_drive" | "onedrive" => Some(catalogs::ONE_DRIVE_CURATED), - "excel" => Some(catalogs::EXCEL_CURATED), - "todoist" => Some(catalogs::TODOIST_CURATED), - _ => None, - } -} - -/// All toolkit slugs that have a curated agent-ready catalog. -/// -/// Source of truth for the UI "preview / agent integration coming -/// soon" badge: any connected toolkit whose slug is NOT in this list -/// can be authorized but lacks a curated tool surface, so the agent -/// can't use it productively. -/// -/// Returned in sorted order to keep the RPC response stable across -/// builds. -pub fn agent_ready_toolkits() -> Vec<&'static str> { - let mut slugs: Vec<&'static str> = vec![ - // Native providers - "gmail", - "notion", - "github", - // Catalog-only toolkits - "slack", - "discord", - "googlecalendar", - "googledrive", - "googledocs", - "googlesheets", - "outlook", - "microsoft_teams", - "linear", - "jira", - "trello", - "asana", - "dropbox", - "twitter", - "spotify", - "telegram", - "whatsapp", - "shopify", - "stripe", - "hubspot", - "salesforce", - "airtable", - "figma", - "youtube", - "one_drive", - "excel", - "todoist", - ]; - slugs.sort_unstable(); - slugs -} - -pub use descriptions::toolkit_description; -pub(crate) use helpers::{first_array_str, merge_extra}; -// `pick_str` is a provider payload normaliser and lives in tinycortex; it is -// re-exported here so the ~40 in-tree call sites keep resolving unchanged. -// Note this is deliberately NOT `providers::common::pick_str`, which coerces -// numbers to strings — see the doc comments on both definitions. -pub use registry::{ - all_providers, get_provider, init_default_providers, register_provider, ProviderArc, -}; -pub use scope_lookup::{curated_scope_for, toolkit_has_scope}; -pub(crate) use tinymemory_sync::helpers::pick_str; -pub use tool_scope::{classify_unknown, find_curated, toolkit_from_slug, CuratedTool, ToolScope}; -pub use traits::{resolve_sync_interval_secs, sync_interval_env_var, ComposioProvider}; -pub use types::{ - ComposioUsage, ComposioUsageHandle, GithubFetchMode, NormalizedTask, ProviderContext, - ProviderUserProfile, SyncOutcome, SyncReason, TaskContainer, TaskFetchFilter, TaskKind, -}; -pub use user_scopes::{load_or_default as load_user_scope_or_default, UserScopePref}; - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - - #[test] - fn pick_str_finds_first_non_empty_match() { - let v = json!({ - "data": { "user": { "email": " user@example.com ", "name": "" } }, - "fallback": "fallback@example.com" - }); - // first path empty -> falls through - assert_eq!( - pick_str(&v, &["data.user.name", "data.user.email"]), - Some("user@example.com".to_string()) - ); - // missing path -> falls through to fallback - assert_eq!( - pick_str(&v, &["data.missing", "fallback"]), - Some("fallback@example.com".to_string()) - ); - // nothing matches - assert_eq!(pick_str(&v, &["nope.nope"]), None); - } - - #[test] - fn sync_outcome_elapsed_ms_is_safe_when_finish_lt_start() { - let mut o = SyncOutcome { - started_at_ms: 100, - finished_at_ms: 50, - ..Default::default() - }; - assert_eq!(o.elapsed_ms(), 0); - o.finished_at_ms = 250; - assert_eq!(o.elapsed_ms(), 150); - } - - #[test] - fn pick_str_returns_none_for_non_string_values() { - let v = json!({ "count": 42, "flag": true, "empty": "", "whitespace": " " }); - assert_eq!(pick_str(&v, &["count"]), None); - assert_eq!(pick_str(&v, &["flag"]), None); - assert_eq!(pick_str(&v, &["empty"]), None); - assert_eq!(pick_str(&v, &["whitespace"]), None); - } - - #[test] - fn pick_str_respects_path_order() { - let v = json!({ "a": "first", "b": "second" }); - assert_eq!(pick_str(&v, &["a", "b"]), Some("first".into())); - assert_eq!(pick_str(&v, &["b", "a"]), Some("second".into())); - } - - #[test] - fn sync_reason_as_str_matches_enum_variant() { - assert_eq!(SyncReason::ConnectionCreated.as_str(), "connection_created"); - assert_eq!(SyncReason::Periodic.as_str(), "periodic"); - assert_eq!(SyncReason::Manual.as_str(), "manual"); - } - - #[test] - fn sync_reason_serde_is_snake_case() { - let s = serde_json::to_string(&SyncReason::ConnectionCreated).unwrap(); - assert_eq!(s, "\"connection_created\""); - let back: SyncReason = serde_json::from_str(&s).unwrap(); - assert_eq!(back, SyncReason::ConnectionCreated); - } - - // Note: `toolkit_has_scope` tests now live in `scope_lookup.rs` - // alongside the implementation. - - #[test] - fn catalog_for_toolkit_resolves_new_microsoft_and_todoist_slugs() { - // Newly added catalogs (#2283): OneDrive, Excel, Todoist must be - // discoverable both by their canonical UI slug AND by the - // prefix that `toolkit_from_slug` extracts from action slugs. - assert!(catalog_for_toolkit("one_drive").is_some()); - assert!(catalog_for_toolkit("onedrive").is_some()); - // ONE_DRIVE_GET_FILE → toolkit_from_slug() → "one" - assert!(catalog_for_toolkit("one").is_some()); - assert!(catalog_for_toolkit("excel").is_some()); - assert!(catalog_for_toolkit("todoist").is_some()); - } - - #[test] - fn agent_ready_toolkits_includes_new_catalogs_and_is_sorted() { - let slugs = agent_ready_toolkits(); - assert!(slugs.contains(&"one_drive")); - assert!(slugs.contains(&"excel")); - assert!(slugs.contains(&"todoist")); - // Spot-check legacy entries still present. - assert!(slugs.contains(&"gmail")); - assert!(slugs.contains(&"slack")); - // Uncurated toolkit must NOT appear — guarantees the UI badge - // logic can rely on this set to flag "preview" toolkits. - assert!(!slugs.contains(&"sharepoint")); - assert!(!slugs.contains(&"clickup")); - // Stable order across builds — the RPC consumer caches it. - let mut expected = slugs.clone(); - expected.sort_unstable(); - assert_eq!(slugs, expected); - } - - #[test] - fn capability_matrix_includes_new_catalog_only_toolkits() { - let matrix = capability_matrix(); - for slug in ["one_drive", "excel", "todoist"] { - let row = matrix - .iter() - .find(|entry| entry.toolkit == slug) - .unwrap_or_else(|| panic!("{slug} capability row missing")); - assert!(!row.native_provider, "{slug} should not be native"); - assert!(row.curated_tools, "{slug} should be catalogued"); - assert!( - row.curated_tool_count > 0, - "{slug} catalog should be non-empty" - ); - assert!( - row.tool_execution, - "{slug} tool execution should be enabled" - ); - // No profile/sync/memory ingest — catalog-only. - assert!(!row.user_profile); - assert!(!row.initial_sync); - assert!(!row.periodic_sync); - assert!(!row.memory_ingest); - } - } - - #[test] - fn capability_matrix_distinguishes_native_from_catalog_only_toolkits() { - let matrix = capability_matrix(); - - let gmail = matrix - .iter() - .find(|entry| entry.toolkit == "gmail") - .expect("gmail capability row"); - assert!(gmail.native_provider); - assert!(gmail.curated_tools); - assert!(gmail.curated_tool_count > 0); - assert!(gmail.user_profile); - assert!(gmail.initial_sync); - assert!(gmail.periodic_sync); - assert_eq!(gmail.sync_interval_secs, Some(15 * 60)); - assert!(gmail.trigger_webhooks); - assert!(gmail.memory_ingest); - - let google_calendar = matrix - .iter() - .find(|entry| entry.toolkit == "googlecalendar") - .expect("googlecalendar capability row"); - assert!(!google_calendar.native_provider); - assert!(google_calendar.curated_tools); - assert!(google_calendar.curated_tool_count > 0); - assert!(google_calendar.tool_execution); - assert!(!google_calendar.user_profile); - assert!(!google_calendar.initial_sync); - assert!(!google_calendar.periodic_sync); - assert_eq!(google_calendar.sync_interval_secs, None); - assert!(!google_calendar.memory_ingest); - } - - #[test] - fn capability_matrix_includes_clickup_as_native_memory_provider() { - // Locks in the per-issue #2288 registration: a ClickUp row must - // appear in the capability matrix with the same native-provider - // flags Gmail/Notion/Slack already carry (`memory_ingest`, - // `periodic_sync`, non-zero `sync_interval_secs`). If a future - // change drops one of the four registration touchpoints - // (CAPABILITY_TOOLKITS, has_native_provider, - // native_provider_sync_interval, catalog_for_toolkit) this test - // fails loud rather than silently degrading the provider to - // catalog-only status. - let matrix = capability_matrix(); - let clickup = matrix - .iter() - .find(|entry| entry.toolkit == "clickup") - .expect("clickup capability row"); - assert!(clickup.native_provider, "clickup must be native"); - assert!(clickup.curated_tools, "clickup must have a curated catalog"); - assert!( - clickup.curated_tool_count > 0, - "clickup catalog must be non-empty" - ); - assert!(clickup.user_profile); - assert!(clickup.initial_sync); - assert!(clickup.periodic_sync); - assert_eq!(clickup.sync_interval_secs, Some(30 * 60)); - assert!(clickup.memory_ingest); - } - - #[test] - fn capability_matrix_includes_linear_as_native_memory_provider() { - // Per-issue #2400 registration: a Linear row must appear in - // the capability matrix as a native memory-ingest provider, - // matching gmail / notion / slack / clickup. If a future - // change drops one of the five registration touchpoints - // (CAPABILITY_TOOLKITS, has_native_provider, - // native_provider_sync_interval, catalog_for_toolkit, - // toolkit_description) this test fails loud rather than - // silently degrading the provider to catalog-only status. - let matrix = capability_matrix(); - let linear = matrix - .iter() - .find(|entry| entry.toolkit == "linear") - .expect("linear capability row"); - assert!(linear.native_provider, "linear must be native"); - assert!(linear.curated_tools, "linear must have a curated catalog"); - assert!( - linear.curated_tool_count > 0, - "linear catalog must be non-empty" - ); - assert!(linear.user_profile); - assert!(linear.initial_sync); - assert!(linear.periodic_sync); - assert_eq!(linear.sync_interval_secs, Some(30 * 60)); - assert!(linear.memory_ingest); - } - - #[test] - fn capability_matrix_includes_github_as_native_memory_provider() { - let matrix = capability_matrix(); - let github = matrix - .iter() - .find(|entry| entry.toolkit == "github") - .expect("github capability row"); - assert!(github.native_provider, "github must be native"); - assert!(github.curated_tools, "github must have a curated catalog"); - assert!( - github.curated_tool_count > 0, - "github catalog must be non-empty" - ); - assert!(github.user_profile); - assert!(github.initial_sync); - assert!(github.periodic_sync); - assert_eq!(github.sync_interval_secs, Some(30 * 60)); - assert!(github.memory_ingest); - } - - #[test] - fn toolkit_description_known_slugs_are_distinct_and_non_empty() { - let known = [ - "gmail", - "notion", - "github", - "slack", - "discord", - "google_calendar", - "google_drive", - "google_docs", - "google_sheets", - "outlook", - "microsoft_teams", - "linear", - "jira", - "trello", - "asana", - "dropbox", - "twitter", - "spotify", - "telegram", - "whatsapp", - "twilio", - "shopify", - "stripe", - "hubspot", - "salesforce", - "airtable", - "figma", - "youtube", - "calendar", - ]; - let fallback = toolkit_description("__definitely_unknown_slug__"); - for slug in known { - let desc = toolkit_description(slug); - assert!(!desc.is_empty(), "{slug} description must not be empty"); - assert_ne!( - desc, fallback, - "known slug `{slug}` must not map to the generic fallback" - ); - } - } - - #[test] - fn toolkit_description_unknown_slug_uses_generic_fallback() { - assert_eq!( - toolkit_description("not_a_real_toolkit_123"), - "Interact with this connected service via its available actions" - ); - assert_eq!( - toolkit_description(""), - "Interact with this connected service via its available actions" - ); - } - - #[test] - fn toolkit_description_is_case_sensitive() { - // The match is lowercase-only by convention; an uppercase slug - // should fall through to the generic description. Explicitly - // documenting this guards against accidental case-insensitive - // matching sneaking in later. - let fallback = toolkit_description("__fallback__"); - assert_eq!(toolkit_description("GMAIL"), fallback); - assert_eq!(toolkit_description("Notion"), fallback); - } - - #[test] - fn provider_user_profile_default_is_empty() { - let p = ProviderUserProfile::default(); - assert!(p.toolkit.is_empty()); - assert!(p.connection_id.is_none()); - assert!(p.display_name.is_none()); - assert!(p.email.is_none()); - assert!(p.username.is_none()); - assert!(p.avatar_url.is_none()); - assert!(p.profile_url.is_none()); - assert!(p.extras.is_null()); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/notion/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/notion/mod.rs deleted file mode 100644 index 138fe603..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/notion/mod.rs +++ /dev/null @@ -1,11 +0,0 @@ -// The payload normalisers moved to tinycortex (they are pure Value -// transforms, i.e. driver-side). Aliased under the old module name so -// every `normalization::extract_*` call site below stays unchanged. -use tinymemory_sync::notion as normalization; -mod provider; -#[cfg(test)] -mod tests; -pub mod tools; - -pub use provider::NotionProvider; -pub use tools::NOTION_CURATED; diff --git a/crates/tinymemory-core/src/sync/composio/providers/notion/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/notion/provider.rs deleted file mode 100644 index af56b6df..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/notion/provider.rs +++ /dev/null @@ -1,411 +0,0 @@ -//! Notion provider — incremental sync with per-item persistence. -//! -//! On each sync pass: -//! -//! 1. Load persistent [`SyncState`] from the KV store. -//! 2. Check the daily request budget — bail early if exhausted. -//! 3. Fetch a page of recently edited pages via `NOTION_FETCH_DATA`, -//! sorted by `last_edited_time` descending. When a cursor exists -//! we can stop as soon as we see pages older than the cursor. -//! 4. Deduplicate against `synced_ids` in the state. Pages that have -//! been *edited* since their last sync are re-persisted (the cursor -//! is based on `last_edited_time`, so an edited page appears again). -//! 5. Persist each **new or updated** page as its own memory document. -//! 6. Paginate (up to budget) until no more results or all items in the -//! page are older than the cursor. -//! 7. Advance the cursor and save state. - -use async_trait::async_trait; -use serde_json::{json, Value}; - -use super::normalization; -use crate::sync::composio::providers::{ - first_array_str, merge_extra, pick_str, resolve_sync_interval_secs, ComposioProvider, - CuratedTool, NormalizedTask, ProviderContext, ProviderUserProfile, TaskContainer, - TaskFetchFilter, TaskKind, -}; - -pub(crate) const ACTION_GET_ABOUT_ME: &str = "NOTION_GET_ABOUT_ME"; -pub(crate) const ACTION_FETCH_DATA: &str = "NOTION_FETCH_DATA"; -pub(crate) const ACTION_QUERY_DATABASE: &str = "NOTION_QUERY_DATABASE"; -pub(crate) const ACTION_SEARCH_NOTION_PAGE: &str = "NOTION_SEARCH_NOTION_PAGE"; - -/// Paths for extracting a page's unique ID. -pub(crate) const PAGE_ID_PATHS: &[&str] = &["id", "data.id", "pageId", "data.pageId"]; - -/// Paths for extracting the `last_edited_time` used as sync cursor. -pub(crate) const PAGE_EDITED_PATHS: &[&str] = &[ - "last_edited_time", - "data.last_edited_time", - "lastEditedTime", - "data.lastEditedTime", -]; - -pub struct NotionProvider; - -impl NotionProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for NotionProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for NotionProvider { - fn toolkit_slug(&self) -> &'static str { - "notion" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(super::tools::NOTION_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - Some(resolve_sync_interval_secs("notion", 30 * 60)) - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:notion] fetch_user_profile via {ACTION_GET_ABOUT_ME}" - ); - - let resp = ctx - .execute(ACTION_GET_ABOUT_ME, Some(json!({}))) - .await - .map_err(|e| format!("[composio:notion] {ACTION_GET_ABOUT_ME} failed: {e:#}"))?; - - if !resp.successful { - let err = resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!("[composio:notion] {ACTION_GET_ABOUT_ME}: {err}")); - } - - // `data` is already the inner Composio response payload — paths - // here are relative to it. For bot-token connections the - // top-level `name` is the *integration's* name (e.g. "Composio"), - // and the actual owning user lives at `bot.owner.user.*`. Probe - // the bot-owner paths first so identity reflects the user (#1365). - let data = &resp.data; - let display_name = pick_str(data, &["bot.owner.user.name", "user.name", "name"]); - let email = pick_str( - data, - &[ - "bot.owner.user.person.email", - "user.person.email", - "person.email", - "email", - ], - ); - let username = pick_str(data, &["bot.owner.user.id", "user.id", "id"]); - let avatar_url = pick_str( - data, - &["bot.owner.user.avatar_url", "user.avatar_url", "avatar_url"], - ); - let profile_url = pick_str(data, &["url", "profile_url", "profile.url"]); - - Ok(ProviderUserProfile { - toolkit: "notion".to_string(), - connection_id: ctx.connection_id.clone(), - display_name, - email, - username, - avatar_url, - profile_url, - extras: data.clone(), - }) - } - - /// Incremental sync. Notion was the first provider migrated to the generic - /// `orchestrator`: - /// the per-item loop, dedup, `max_items` cap, `sync_depth_days` window, and - /// cursor handling all live in `run_sync`; the Notion-specific primitives - /// (page fetch, dedup key, body fetch, ingest) live in `super::source`. - async fn fetch_tasks( - &self, - ctx: &ProviderContext, - filter: &TaskFetchFilter, - ) -> Result, String> { - let max = filter.effective_max(); - let database_id = filter - .database_id - .as_deref() - .map(str::trim) - .filter(|s| !s.is_empty()); - - tracing::debug!( - connection_id = ?ctx.connection_id, - max, - has_database = database_id.is_some(), - "[composio:notion] fetch_tasks" - ); - - // A configured board (database) uses NOTION_QUERY_DATABASE; - // otherwise fall back to NOTION_FETCH_DATA (recent pages), the - // same action the periodic sync uses. - let (action, mut args) = match database_id { - Some(db) => ( - ACTION_QUERY_DATABASE, - json!({ - "database_id": db, - "page_size": max.min(100) as u32, - "sorts": [ { "timestamp": "last_edited_time", "direction": "descending" } ], - }), - ), - None => ( - ACTION_FETCH_DATA, - json!({ - "page_size": max.min(100) as u32, - "filter": { "value": "page", "property": "object" }, - "sort": { "direction": "descending", "timestamp": "last_edited_time" }, - }), - ), - }; - merge_extra(&mut args, &filter.extra); - - let resp = ctx - .execute(action, Some(args)) - .await - .map_err(|e| format!("[composio:notion] {action}: {e:#}"))?; - if !resp.successful { - return Err(format!( - "[composio:notion] {action}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - - // Optional client-side status filter — Notion status properties - // are user-defined, so we match on the normalized status rather - // than building a server-side property filter. - let want_status = filter - .status - .as_deref() - .map(|s| s.trim().to_ascii_lowercase()) - .filter(|s| !s.is_empty()); - - let mut out: Vec = Vec::new(); - for page in normalization::extract_results(&resp.data) { - if out.len() >= max { - break; - } - let Some(nt) = normalize_notion_page(&page) else { - continue; - }; - if let Some(ref want) = want_status { - let matches = nt - .status - .as_deref() - .map(|s| s.to_ascii_lowercase() == *want) - .unwrap_or(false); - if !matches { - continue; - } - } - out.push(nt); - } - tracing::debug!(count = out.len(), "[composio:notion] fetch_tasks complete"); - Ok(out) - } - - /// List the Notion databases (tables) the connected integration can see, - /// via `NOTION_SEARCH_NOTION_PAGE` filtered to database objects, so the - /// task-source UI can offer a picker for `database_id`. Only databases the - /// integration has been *shared with* in Notion are returned. - async fn list_databases(&self, ctx: &ProviderContext) -> Result, String> { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:notion] list_databases via {ACTION_SEARCH_NOTION_PAGE}" - ); - // Composio's NOTION_SEARCH_NOTION_PAGE *flattens* Notion's native - // `filter: { value, property }` into top-level `filter_value` / - // `filter_property` params and silently drops the nested form (which - // returned only pages). We send the flat params here; the nested - // `filter` is kept too as a belt-and-braces hint for any variant that - // honours it, and the parser still drops any stray `page` items. - let args = json!({ - "query": "", - "filter_value": "database", - "filter_property": "object", - "filter": { "value": "database", "property": "object" }, - "page_size": 100, - }); - let resp = ctx - .execute(ACTION_SEARCH_NOTION_PAGE, Some(args)) - .await - .map_err(|e| format!("[composio:notion] {ACTION_SEARCH_NOTION_PAGE}: {e:#}"))?; - if !resp.successful { - return Err(format!( - "[composio:notion] {ACTION_SEARCH_NOTION_PAGE}: {}", - resp.error.unwrap_or_else(|| "provider failure".into()) - )); - } - - tracing::info!( - successful = resp.successful, - data_is_array = resp.data.is_array(), - data_keys = ?resp.data.as_object().map(|o| o.keys().cloned().collect::>()), - "[composio:notion] list_databases raw response shape" - ); - let out = parse_database_results(&resp.data); - tracing::info!( - count = out.len(), - "[composio:notion] list_databases complete" - ); - Ok(out) - } - - async fn on_trigger( - &self, - ctx: &ProviderContext, - trigger: &str, - _payload: &Value, - ) -> Result<(), String> { - tracing::info!( - connection_id = ?ctx.connection_id, - trigger = %trigger, - "[composio:notion] on_trigger" - ); - let Some(connection_id) = ctx.connection_id.as_deref() else { - return Err("[composio:notion] trigger missing connection_id".to_string()); - }; - if let Err(e) = crate::sync::pipelines::host::run_composio_connection( - "notion", - connection_id, - ctx.config.as_ref(), - None, - None, - ) - .await - { - tracing::warn!( - error = %e, - "[composio:notion] trigger-driven sync failed (non-fatal)" - ); - } - Ok(()) - } -} - -/// Map a raw Notion page payload into a [`NormalizedTask`]. -/// -/// Notion databases are user-defined, so property extraction is -/// best-effort against common property names (`Status`, `Assignee`, -/// `Due`). Anything unmatched is simply left `None` — the raw payload is -/// preserved for enrichment. -fn normalize_notion_page(page: &serde_json::Value) -> Option { - let external_id = pick_str(page, PAGE_ID_PATHS)?; - let title = normalization::extract_page_title(page) - .unwrap_or_else(|| format!("Notion page {external_id}")); - Some(NormalizedTask { - external_id, - source_id: String::new(), - provider: "notion".to_string(), - kind: TaskKind::Generic, - title, - body: None, - url: pick_str(page, &["url", "data.url"]), - status: pick_str( - page, - &[ - "properties.Status.status.name", - "properties.Status.select.name", - "data.properties.Status.status.name", - ], - ), - assignee: first_array_str( - page, - &[ - "properties.Assignee.people", - "data.properties.Assignee.people", - ], - &["name"], - ), - due: pick_str( - page, - &[ - "properties.Due.date.start", - "data.properties.Due.date.start", - ], - ), - labels: Vec::new(), - priority: pick_str( - page, - &[ - "properties.Priority.select.name", - "data.properties.Priority.select.name", - ], - ), - updated_at: pick_str(page, PAGE_EDITED_PATHS), - raw: page.clone(), - }) -} - -/// Map a `NOTION_SEARCH_NOTION_PAGE` response into the database containers -/// the UI picker needs. -/// -/// We send a server-side `object: database` filter, so the response is -/// already scoped — we therefore *trust* it and only drop items explicitly -/// typed as `page`. This is intentional: Composio's response items don't -/// always carry a top-level `object` field, and an over-strict -/// "keep only object==database" check silently dropped every database. -/// Pure (no I/O) so it is unit-testable. -pub(super) fn parse_database_results(data: &serde_json::Value) -> Vec { - let results = normalization::extract_results(data); - let mut kinds: std::collections::BTreeMap = std::collections::BTreeMap::new(); - let mut out: Vec = Vec::new(); - for item in &results { - let object = pick_str(item, &["object", "data.object"]); - *kinds - .entry(object.clone().unwrap_or_else(|| "".to_string())) - .or_default() += 1; - // Trust the server-side database filter: keep databases / data_sources - // *and* objectless items; only drop items explicitly typed as pages. - if object.as_deref() == Some("page") { - continue; - } - let Some(id) = pick_str(item, PAGE_ID_PATHS) else { - continue; - }; - let title = extract_database_title(item).unwrap_or_else(|| format!("Notion database {id}")); - out.push(TaskContainer { id, title }); - } - tracing::info!( - raw = results.len(), - kept = out.len(), - object_kinds = ?kinds, - "[composio:notion] parse_database_results" - ); - out -} - -/// Extract a Notion database's display title from its top-level `title` -/// rich-text array (`title[].plain_text`), tolerant of the Composio `data` -/// wrapper. Returns `None` for an untitled / shapeless database. -fn extract_database_title(db: &serde_json::Value) -> Option { - let arr = db - .get("title") - .or_else(|| db.get("data").and_then(|d| d.get("title"))) - .and_then(|v| v.as_array())?; - let text: String = arr - .iter() - .filter_map(|t| { - t.get("plain_text").and_then(|p| p.as_str()).or_else(|| { - t.get("text") - .and_then(|x| x.get("content")) - .and_then(|c| c.as_str()) - }) - }) - .collect(); - let text = text.trim(); - (!text.is_empty()).then(|| text.to_string()) -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/notion/tests.rs b/crates/tinymemory-core/src/sync/composio/providers/notion/tests.rs deleted file mode 100644 index 0ddf92c0..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/notion/tests.rs +++ /dev/null @@ -1,119 +0,0 @@ -//! Unit tests for the Notion provider. - -use super::normalization::{extract_notion_cursor, extract_page_title, extract_results}; -use super::NotionProvider; -use crate::sync::composio::providers::ComposioProvider; -use serde_json::json; - -#[test] -fn extract_results_walks_common_shapes() { - let v1 = json!({ "data": { "results": [{"id": "p1"}] } }); - let v2 = json!({ "results": [{"id": "p2"}, {"id": "p3"}] }); - let v3 = json!({ "data": {} }); - assert_eq!(extract_results(&v1).len(), 1); - assert_eq!(extract_results(&v2).len(), 2); - assert_eq!(extract_results(&v3).len(), 0); -} - -#[test] -fn extract_notion_cursor_finds_nested() { - let v = json!({ "data": { "next_cursor": "abc123" } }); - assert_eq!(extract_notion_cursor(&v), Some("abc123".to_string())); -} - -#[test] -fn extract_notion_cursor_none_when_missing() { - let v = json!({ "data": { "has_more": false } }); - assert_eq!(extract_notion_cursor(&v), None); -} - -#[test] -fn extract_page_title_from_properties() { - let page = json!({ - "id": "page-1", - "properties": { - "Name": { - "type": "title", - "title": [ - { "plain_text": "My " }, - { "plain_text": "Page Title" } - ] - } - } - }); - assert_eq!(extract_page_title(&page), Some("My Page Title".to_string())); -} - -#[test] -fn extract_page_title_fallback_to_top_level() { - let page = json!({ "title": "Fallback Title" }); - assert_eq!( - extract_page_title(&page), - Some("Fallback Title".to_string()) - ); -} - -#[test] -fn extract_page_title_returns_none_when_missing() { - let page = json!({ "id": "p1" }); - assert_eq!(extract_page_title(&page), None); -} - -#[test] -fn provider_metadata_is_stable() { - let p = NotionProvider::new(); - assert_eq!(p.toolkit_slug(), "notion"); - assert_eq!(p.sync_interval_secs(), Some(30 * 60)); -} - -#[test] -fn default_impl_matches_new() { - let _a = NotionProvider::new(); - let _b = ::default(); -} - -// ── parse_database_results (list_databases parser) ─────────────────────────── - -#[test] -fn parse_database_results_keeps_databases_and_extracts_title() { - use super::provider::parse_database_results; - let data = json!({ - "results": [ - { - "object": "database", - "id": "db-1", - "title": [{ "plain_text": "Engineering " }, { "plain_text": "Tasks" }] - }, - // A page hit must be filtered out — list_databases is databases only. - { "object": "page", "id": "pg-9", "title": [{ "plain_text": "Some page" }] }, - // Newest API exposes databases as `data_source`. - { "object": "data_source", "id": "db-2", "title": [{ "plain_text": "Roadmap" }] }, - // Untitled database falls back to a synthesized label. - { "object": "database", "id": "db-3", "title": [] } - ] - }); - let dbs = parse_database_results(&data); - assert_eq!( - dbs.len(), - 3, - "two named databases + one data_source, page dropped" - ); - assert_eq!(dbs[0].id, "db-1"); - assert_eq!(dbs[0].title, "Engineering Tasks"); - assert_eq!(dbs[1].id, "db-2"); - assert_eq!(dbs[1].title, "Roadmap"); - assert_eq!(dbs[2].title, "Notion database db-3"); -} - -#[test] -fn parse_database_results_handles_data_wrapper_and_empty() { - use super::provider::parse_database_results; - let wrapped = json!({ "data": { "results": [ - { "object": "database", "id": "x", "title": [{ "plain_text": "Wrapped" }] } - ] } }); - let dbs = parse_database_results(&wrapped); - assert_eq!(dbs.len(), 1); - assert_eq!(dbs[0].title, "Wrapped"); - - assert!(parse_database_results(&json!({ "results": [] })).is_empty()); -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/notion/tools.rs b/crates/tinymemory-core/src/sync/composio/providers/notion/tools.rs deleted file mode 100644 index 89f4efc5..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/notion/tools.rs +++ /dev/null @@ -1,196 +0,0 @@ -//! Curated catalog of Notion Composio actions exposed to the agent. - -use crate::sync::composio::providers::tool_scope::{CuratedTool, ToolScope}; - -pub const NOTION_CURATED: &[CuratedTool] = &[ - // ── Read: search & fetch ──────────────────────────────────────── - CuratedTool { - slug: "NOTION_SEARCH_NOTION_PAGE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_FETCH_DATA", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_FETCH_DATABASE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_FETCH_ROW", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_FETCH_BLOCK_METADATA", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_FETCH_BLOCK_CONTENTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_FETCH_ALL_BLOCK_CONTENTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_FETCH_COMMENTS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_GET_PAGE_MARKDOWN", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_GET_PAGE_PROPERTY_ACTION", - scope: ToolScope::Read, - }, - // ── Read: query & retrieve ────────────────────────────────────── - CuratedTool { - slug: "NOTION_QUERY_DATABASE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_QUERY_DATABASE_WITH_FILTER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_QUERY_DATA_SOURCE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_RETRIEVE_PAGE", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_RETRIEVE_COMMENT", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_RETRIEVE_DATABASE_PROPERTY", - scope: ToolScope::Read, - }, - // ── Read: profile / users / files ─────────────────────────────── - CuratedTool { - slug: "NOTION_LIST_USERS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_GET_ABOUT_USER", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_GET_ABOUT_ME", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_LIST_FILE_UPLOADS", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_RETRIEVE_FILE_UPLOAD", - scope: ToolScope::Read, - }, - CuratedTool { - slug: "NOTION_LIST_DATA_SOURCE_TEMPLATES", - scope: ToolScope::Read, - }, - // ── Write: create ─────────────────────────────────────────────── - CuratedTool { - slug: "NOTION_CREATE_NOTION_PAGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_CREATE_DATABASE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_CREATE_COMMENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_CREATE_FILE_UPLOAD", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_SEND_FILE_UPLOAD", - scope: ToolScope::Write, - }, - // ── Write: update / append ────────────────────────────────────── - CuratedTool { - slug: "NOTION_UPDATE_PAGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_UPDATE_BLOCK", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_UPDATE_ROW_DATABASE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_INSERT_ROW_DATABASE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_INSERT_ROW_FROM_NL", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_REPLACE_PAGE_CONTENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_ADD_PAGE_CONTENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_ADD_MULTIPLE_PAGE_CONTENT", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_APPEND_BLOCK_CHILDREN", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_APPEND_TEXT_BLOCKS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_APPEND_TASK_BLOCKS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_APPEND_CODE_BLOCKS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_APPEND_MEDIA_BLOCKS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_APPEND_LAYOUT_BLOCKS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_APPEND_TABLE_BLOCKS", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_DUPLICATE_PAGE", - scope: ToolScope::Write, - }, - CuratedTool { - slug: "NOTION_MOVE_PAGE", - scope: ToolScope::Write, - }, - // ── Admin: destructive ────────────────────────────────────────── - CuratedTool { - slug: "NOTION_DELETE_BLOCK", - scope: ToolScope::Admin, - }, - CuratedTool { - slug: "NOTION_ARCHIVE_NOTION_PAGE", - scope: ToolScope::Admin, - }, -]; diff --git a/crates/tinymemory-core/src/sync/composio/providers/profile.rs b/crates/tinymemory-core/src/sync/composio/providers/profile.rs deleted file mode 100644 index baf9a22b..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/profile.rs +++ /dev/null @@ -1,821 +0,0 @@ -//! Profile persistence — maps [`ProviderUserProfile`] (and provider-specific -//! `extras`) into [`IdentityKind`]-tagged facet rows so the self-identity -//! matcher can join directly against the memory tree's `EntityKind` and the -//! structural sender field on chunks. -//! -//! Schema: `user_profile.facet_type='skill'`, -//! `key = "skill:{toolkit}:{conn_id}:{identity_kind}"`, `value` = -//! canonicalized identifier. Confidence is set per-kind so the matcher can -//! refuse to auto-promote weak signals (display_name) to `is_self`. -//! -//! One [`ProviderUserProfile`] expands to multiple rows — including -//! identifiers carried in `extras` that the previous fixed-fields shape -//! dropped on the floor (e.g. Slack screen-name handle). -//! -//! Callers invoke [`persist_provider_profile`] after every successful -//! `fetch_user_profile` call — from `on_connection_created`, periodic syncs, -//! and the `composio_get_user_profile` / `composio_refresh_all_identities` -//! RPC ops. - -use super::ProviderUserProfile; -use crate::learning_candidate::{ - self as learning_candidate, CueFamily, EvidenceRef, FacetClass, LearningCandidate, -}; -use crate::store::profile::FacetType; -use serde_json::Value; -use std::collections::BTreeMap; - -// ──────────────────────────────────────────────────────────────────────── -// IdentityKind — the matching axis -// ──────────────────────────────────────────────────────────────────────── - -/// Shape of an identifier persisted against a connection. Mirrors the -/// matching dimensions of the memory tree's -/// `crate::tree::score::extract::EntityKind` so the -/// self-check is a direct `(toolkit, kind, value)` lookup. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum IdentityKind { - /// Platform-canonical immutable id — Slack `U123ABC`, Notion UUID. - UserId, - Email, - /// `@`-style screen name, canonicalised without the leading `@`. - Handle, - /// E.164 phone number. - Phone, - /// Human display label. Weak signal — never auto-promotes to is_self. - DisplayName, - /// Not for matching; kept for UI / prompt rendering. - AvatarUrl, - /// Not for matching; kept for UI / prompt rendering. - ProfileUrl, -} - -impl IdentityKind { - pub fn as_str(self) -> &'static str { - match self { - Self::UserId => "user_id", - Self::Email => "email", - Self::Handle => "handle", - Self::Phone => "phone", - Self::DisplayName => "display_name", - Self::AvatarUrl => "avatar_url", - Self::ProfileUrl => "profile_url", - } - } - - pub fn parse(s: &str) -> Option { - Some(match s { - "user_id" => Self::UserId, - "email" => Self::Email, - "handle" => Self::Handle, - "phone" => Self::Phone, - "display_name" => Self::DisplayName, - "avatar_url" => Self::AvatarUrl, - "profile_url" => Self::ProfileUrl, - _ => return None, - }) - } - - /// Confidence the matcher records on the row. Hard kinds auto-promote - /// a chunk to `is_self`; weak kinds require corroboration. - pub fn confidence(self) -> f64 { - match self { - Self::UserId | Self::Phone => 1.00, - Self::Email => 0.95, - Self::Handle => 0.70, - Self::DisplayName => 0.40, - Self::AvatarUrl | Self::ProfileUrl => 0.50, - } - } - - /// True if this kind is a real identity signal worth running through - /// the matcher (vs. UI-only fields). - pub fn is_matchable(self) -> bool { - matches!( - self, - Self::UserId | Self::Email | Self::Handle | Self::Phone | Self::DisplayName - ) - } -} - -/// Canonicalize a raw value for storage and lookup. The same routine runs -/// on the entity side at match time, so equality of canonical forms is the -/// matcher's only test — no `COLLATE NOCASE`, no per-call lowercasing. -pub fn canonicalize(kind: IdentityKind, raw: &str) -> Option { - let trimmed = raw.trim(); - if trimmed.is_empty() { - return None; - } - Some(match kind { - IdentityKind::Email => trimmed.to_lowercase(), - IdentityKind::Handle => trimmed.trim_start_matches('@').to_lowercase(), - IdentityKind::Phone => trimmed - .chars() - .filter(|c| c.is_ascii_digit() || *c == '+') - .collect(), - IdentityKind::DisplayName => trimmed.split_whitespace().collect::>().join(" "), - IdentityKind::UserId | IdentityKind::AvatarUrl | IdentityKind::ProfileUrl => { - trimmed.to_string() - } - }) -} - -// ──────────────────────────────────────────────────────────────────────── -// Persist -// ──────────────────────────────────────────────────────────────────────── - -/// Persist a provider profile as one facet row per (kind, value). Returns -/// the number of rows written. Silently no-ops if the memory client isn't -/// ready (startup race / unauthenticated CLI). -pub fn persist_provider_profile(profile: &ProviderUserProfile) -> usize { - let Some(client) = crate::global::client_if_ready() else { - tracing::debug!( - toolkit = %profile.toolkit, - "[composio:profile] memory client not ready, skipping persist" - ); - return 0; - }; - let store = client.profile_store(); - - let now = now_secs(); - let toolkit = normalize_token(&profile.toolkit); - let identifier = profile - .connection_id - .as_deref() - .map(normalize_token) - .filter(|v| !v.is_empty()) - .unwrap_or_else(|| "default".to_string()); - - let rows = expand_identity_rows(&toolkit, profile); - - let mut written = 0usize; - for (kind, value) in rows { - let key = format!("skill:{toolkit}:{identifier}:{}", kind.as_str()); - let facet_id = format!("skill-{toolkit}-{identifier}-{}", kind.as_str()); - - if let Err(e) = store.upsert_provider_facet( - &facet_id, - &FacetType::Workflow, - &key, - &value, - kind.confidence(), - None, - now, - ) { - tracing::warn!( - toolkit = %toolkit, - identifier = %identifier, - kind = kind.as_str(), - error = %e, - "[composio:profile] profile_upsert failed (non-fatal)" - ); - continue; - } - - // Phase 3 (#566): also emit a LearningCandidate so the stability detector - // can score provider data alongside other evidence on the next rebuild. - // We use the `identity/` key prefix for provider identity fields. - if kind.is_matchable() { - let identity_key = format!("{}:{}", normalize_token(&toolkit), kind.as_str()); - let candidate = LearningCandidate { - class: FacetClass::Identity, - key: identity_key, - value: value.clone(), - cue_family: CueFamily::Structural, - evidence: EvidenceRef::Provider { - toolkit: toolkit.clone(), - connection_id: identifier.clone(), - field: kind.as_str().to_string(), - }, - initial_confidence: kind.confidence(), - observed_at: now, - }; - learning_candidate::global().push(candidate); - } - - written += 1; - } - - if written > 0 { - tracing::debug!( - toolkit = %toolkit, - identifier = %identifier, - rows_written = written, - "[composio:profile] persisted identity rows (+ emitted Identity candidates)" - ); - } - written -} - -/// Expand a [`ProviderUserProfile`] (and provider-specific `extras`) into -/// the canonical (kind, value) rows. **All per-toolkit quirks live here**; -/// the matcher only sees normalized tuples. -fn expand_identity_rows( - toolkit: &str, - profile: &ProviderUserProfile, -) -> Vec<(IdentityKind, String)> { - let mut rows: Vec<(IdentityKind, String)> = Vec::new(); - let mut push = |kind: IdentityKind, raw: Option<&str>| { - if let Some(v) = raw.and_then(|s| canonicalize(kind, s)) { - rows.push((kind, v)); - } - }; - - push(IdentityKind::DisplayName, profile.display_name.as_deref()); - push(IdentityKind::Email, profile.email.as_deref()); - push(IdentityKind::AvatarUrl, profile.avatar_url.as_deref()); - push(IdentityKind::ProfileUrl, profile.profile_url.as_deref()); - - match toolkit { - "slack" => { - // After the auth.test + users.info fix in slack/provider.rs: - // profile.username == Slack user_id (e.g. U123ABC) - // extras.handle == Slack screen_name (e.g. "cyrus") - // extras.team_* → workspace context, not identity - push(IdentityKind::UserId, profile.username.as_deref()); - push(IdentityKind::Handle, json_str(&profile.extras, "handle")); - } - "notion" => { - // Notion's `username` is the user UUID - // (`data.bot.owner.user.id` per notion/provider.rs). - push(IdentityKind::UserId, profile.username.as_deref()); - } - "gmail" => { - // Email + display_name only — no platform user_id worth matching. - } - _ => { - // Unknown toolkit: best-effort. If `username` is set treat it - // as a handle so weak-match logic (medium confidence) applies. - push(IdentityKind::Handle, profile.username.as_deref()); - } - } - - rows -} - -fn json_str<'a>(v: &'a Value, key: &str) -> Option<&'a str> { - v.get(key).and_then(|x| x.as_str()) -} - -// ──────────────────────────────────────────────────────────────────────── -// Read paths -// ──────────────────────────────────────────────────────────────────────── - -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub struct ConnectedIdentity { - pub source: String, - pub identifier: String, - pub display_name: Option, - pub email: Option, - pub handle: Option, - pub phone: Option, - pub user_id: Option, - pub avatar_url: Option, - pub profile_url: Option, -} - -/// Load all provider-sourced identities, grouped by `(source, conn_id)`. -/// Rows whose last segment is not a known [`IdentityKind`] are silently -/// skipped — that includes legacy `username` rows from before the rewrite. -pub fn load_connected_identities() -> Vec { - let Some(client) = crate::global::client_if_ready() else { - tracing::debug!("[composio:profile] load_connected_identities: memory client not ready"); - return Vec::new(); - }; - let facets = match client.profile_store().facets_by_type(&FacetType::Workflow) { - Ok(f) => f, - Err(error) => { - tracing::warn!( - error = %error, - "[composio:profile] load_connected_identities: profile_facets_by_type failed" - ); - return Vec::new(); - } - }; - - let mut grouped: BTreeMap<(String, String), ConnectedIdentity> = BTreeMap::new(); - for facet in facets { - let Some((source, identifier, kind_str)) = parse_skill_identity_key(&facet.key) else { - continue; - }; - let Some(kind) = IdentityKind::parse(&kind_str) else { - continue; - }; - let entry = grouped - .entry((source.clone(), identifier.clone())) - .or_insert_with(|| ConnectedIdentity { - source, - identifier, - ..Default::default() - }); - match kind { - IdentityKind::DisplayName => entry.display_name = Some(facet.value), - IdentityKind::Email => entry.email = Some(facet.value), - IdentityKind::Handle => entry.handle = Some(facet.value), - IdentityKind::Phone => entry.phone = Some(facet.value), - IdentityKind::UserId => entry.user_id = Some(facet.value), - IdentityKind::AvatarUrl => entry.avatar_url = Some(facet.value), - IdentityKind::ProfileUrl => entry.profile_url = Some(facet.value), - } - } - grouped.into_values().collect() -} - -/// Direct self-check for the entity matcher and the chunk-build hook. -/// Returns true if any connection of `toolkit` has a row with this -/// `(kind, value)` after canonicalization. Non-matchable kinds -/// (avatar_url, profile_url) always return false. -pub fn is_self_identity(toolkit: &str, kind: IdentityKind, raw_value: &str) -> bool { - if !kind.is_matchable() { - return false; - } - let Some(canonical) = canonicalize(kind, raw_value) else { - return false; - }; - let Some(client) = crate::global::client_if_ready() else { - return false; - }; - let key_pattern = format!("skill:{}:%:{}", normalize_token(toolkit), kind.as_str()); - client - .profile_store() - .skill_identity_matches(&key_pattern, &canonical) -} - -/// Cross-toolkit variant — matches against every connected provider's -/// rows of this kind. Used for marking memory-tree entity rows: an email -/// in a Slack message that matches the user's Gmail address is still -/// "me," regardless of which source produced the chunk. -pub fn is_self_identity_any_toolkit(kind: IdentityKind, raw_value: &str) -> bool { - if !kind.is_matchable() { - return false; - } - let Some(canonical) = canonicalize(kind, raw_value) else { - return false; - }; - let Some(client) = crate::global::client_if_ready() else { - return false; - }; - let key_pattern = format!("skill:%:%:{}", kind.as_str()); - client - .profile_store() - .skill_identity_matches(&key_pattern, &canonical) -} - -/// Render a compact section for prompt injection. Skips `user_id` (not -/// human-readable), prefixes `handle` with `@`. -pub fn render_connected_identities_section(identities: &[ConnectedIdentity]) -> String { - if identities.is_empty() { - return String::new(); - } - let mut out = String::from("## Connected Identities\n\n"); - for id in identities { - let mut fields = Vec::::new(); - if let Some(v) = id.display_name.as_deref() { - let v = sanitize_prompt_value(v); - if !v.is_empty() { - fields.push(v); - } - } - if let Some(v) = id.email.as_deref() { - let v = sanitize_prompt_value(v); - if !v.is_empty() { - fields.push(v); - } - } - if let Some(v) = id.handle.as_deref() { - let v = sanitize_prompt_value(v); - if !v.is_empty() { - fields.push(format!("@{v}")); - } - } - if let Some(v) = id.profile_url.as_deref() { - let v = sanitize_prompt_value(v); - if !v.is_empty() { - fields.push(v); - } - } - if fields.is_empty() { - continue; - } - let identifier = sanitize_prompt_value(&id.identifier); - out.push_str(&format!( - "- {} ({}): {}\n", - title_case(&id.source), - identifier, - fields.join(" | ") - )); - } - if out.trim() == "## Connected Identities" { - return String::new(); - } - out -} - -/// Delete every row for a `(source, conn_id)` pair — used on disconnect. -pub fn delete_connected_identity_facets(source: &str, identifier: &str) -> usize { - // `persist_provider_profile` writes keys with `normalize_token`-applied - // segments; compare against the same normalized form here so a caller - // passing the raw toolkit/connection_id still matches stored rows - // (otherwise rows would survive disconnect and the user-tagger would - // keep treating the removed account as the user — #1381 review). - let source = normalize_token(source); - let identifier = normalize_token(identifier); - let Some(client) = crate::global::client_if_ready() else { - tracing::debug!( - source = %source, - identifier = %identifier, - "[composio:profile] delete_connected_identity_facets: memory client not ready" - ); - return 0; - }; - let store = client.profile_store(); - let Ok(facets) = store.facets_by_type(&FacetType::Workflow) else { - return 0; - }; - let mut deleted = 0usize; - for facet in facets { - let Some((s, i, _kind)) = parse_skill_identity_key(&facet.key) else { - continue; - }; - if s == source && i == identifier { - // Same swallow as before: a disconnect must not fail because one - // row was already gone. - if store.delete_by_facet_id(&facet.facet_id).unwrap_or(false) { - deleted += 1; - } - } - } - deleted -} - -// ──────────────────────────────────────────────────────────────────────── -// Helpers -// ──────────────────────────────────────────────────────────────────────── - -fn parse_skill_identity_key(key: &str) -> Option<(String, String, String)> { - let mut parts = key.split(':'); - let prefix = parts.next()?; - let source = parts.next()?; - let identifier = parts.next()?; - let kind = parts.next()?; - if prefix != "skill" || parts.next().is_some() { - return None; - } - Some((source.to_string(), identifier.to_string(), kind.to_string())) -} - -fn normalize_token(raw: &str) -> String { - let mut out = String::with_capacity(raw.len()); - for ch in raw.chars() { - let lower = ch.to_ascii_lowercase(); - if lower.is_ascii_alphanumeric() || lower == '-' || lower == '_' { - out.push(lower); - } else { - out.push('_'); - } - } - out.trim_matches('_').to_string() -} - -pub fn normalize_connection_identifier(raw: &str) -> String { - normalize_token(raw) -} - -fn title_case(raw: &str) -> String { - let mut chars = raw.chars(); - match chars.next() { - Some(first) => first.to_ascii_uppercase().to_string() + chars.as_str(), - None => String::new(), - } -} - -fn sanitize_prompt_value(raw: &str) -> String { - let replaced = raw.replace(['\n', '\r', '\t'], " ").replace('|', "/"); - replaced.split_whitespace().collect::>().join(" ") -} - -fn now_secs() -> f64 { - use std::time::{SystemTime, UNIX_EPOCH}; - SystemTime::now() - .duration_since(UNIX_EPOCH) - .map(|d| d.as_secs_f64()) - .unwrap_or(0.0) -} - -// ──────────────────────────────────────────────────────────────────────── -// Tests -// ──────────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use crate::store::profile::{self, profile_load_all, PROFILE_INIT_SQL}; - use parking_lot::Mutex; - use rusqlite::Connection; - use serde_json::json; - use std::sync::Arc; - - fn setup_db() -> Arc> { - let conn = Connection::open_in_memory().unwrap(); - conn.execute_batch(PROFILE_INIT_SQL).unwrap(); - Arc::new(Mutex::new(conn)) - } - - // ── IdentityKind ─────────────────────────────────────────────── - - #[test] - fn identity_kind_round_trips_through_str() { - for kind in [ - IdentityKind::UserId, - IdentityKind::Email, - IdentityKind::Handle, - IdentityKind::Phone, - IdentityKind::DisplayName, - IdentityKind::AvatarUrl, - IdentityKind::ProfileUrl, - ] { - assert_eq!(IdentityKind::parse(kind.as_str()), Some(kind)); - } - } - - #[test] - fn identity_kind_parse_rejects_unknown() { - assert_eq!(IdentityKind::parse("username"), None); - assert_eq!(IdentityKind::parse(""), None); - assert_eq!(IdentityKind::parse("UserId"), None); - } - - #[test] - fn matchable_kinds_exclude_url_fields() { - assert!(IdentityKind::UserId.is_matchable()); - assert!(IdentityKind::Email.is_matchable()); - assert!(IdentityKind::Handle.is_matchable()); - assert!(IdentityKind::Phone.is_matchable()); - assert!(IdentityKind::DisplayName.is_matchable()); - assert!(!IdentityKind::AvatarUrl.is_matchable()); - assert!(!IdentityKind::ProfileUrl.is_matchable()); - } - - #[test] - fn confidence_orders_hard_above_weak() { - assert!(IdentityKind::UserId.confidence() > IdentityKind::Email.confidence()); - assert!(IdentityKind::Email.confidence() > IdentityKind::Handle.confidence()); - assert!(IdentityKind::Handle.confidence() > IdentityKind::DisplayName.confidence()); - } - - // ── canonicalize ────────────────────────────────────────────── - - #[test] - fn canonicalize_email_lowercases_and_trims() { - assert_eq!( - canonicalize(IdentityKind::Email, " Cyrus@Example.COM "), - Some("cyrus@example.com".to_string()) - ); - } - - #[test] - fn canonicalize_handle_strips_at_and_lowercases() { - assert_eq!( - canonicalize(IdentityKind::Handle, "@Cyrus"), - Some("cyrus".to_string()) - ); - assert_eq!( - canonicalize(IdentityKind::Handle, "cyrus"), - Some("cyrus".to_string()) - ); - } - - #[test] - fn canonicalize_phone_keeps_only_digits_and_plus() { - assert_eq!( - canonicalize(IdentityKind::Phone, "+1 (555) 123-4567"), - Some("+15551234567".to_string()) - ); - } - - #[test] - fn canonicalize_display_name_collapses_whitespace() { - assert_eq!( - canonicalize(IdentityKind::DisplayName, " Cyrus Smith "), - Some("Cyrus Smith".to_string()) - ); - } - - #[test] - fn canonicalize_user_id_preserved_as_is() { - // Slack user_ids are case-sensitive; do not lowercase. - assert_eq!( - canonicalize(IdentityKind::UserId, "U123ABC"), - Some("U123ABC".to_string()) - ); - } - - #[test] - fn canonicalize_empty_returns_none() { - assert_eq!(canonicalize(IdentityKind::Email, ""), None); - assert_eq!(canonicalize(IdentityKind::Email, " "), None); - } - - // ── expand_identity_rows ────────────────────────────────────── - - fn fixture_profile( - toolkit: &str, - username: Option<&str>, - extras: Value, - ) -> ProviderUserProfile { - ProviderUserProfile { - toolkit: toolkit.into(), - connection_id: Some("conn-1".into()), - display_name: Some("Cyrus Smith".into()), - email: Some("cyrus@example.com".into()), - username: username.map(str::to_string), - avatar_url: None, - profile_url: Some("https://example.com/cyrus".into()), - extras, - } - } - - #[test] - fn expand_slack_promotes_username_to_user_id_and_extras_handle() { - let p = fixture_profile("slack", Some("U123ABC"), json!({ "handle": "cyrus" })); - let rows = expand_identity_rows("slack", &p); - - assert!(rows.contains(&(IdentityKind::UserId, "U123ABC".to_string()))); - assert!(rows.contains(&(IdentityKind::Handle, "cyrus".to_string()))); - assert!(rows.contains(&(IdentityKind::Email, "cyrus@example.com".to_string()))); - assert!(rows.contains(&(IdentityKind::DisplayName, "Cyrus Smith".to_string()))); - assert!(rows.contains(&( - IdentityKind::ProfileUrl, - "https://example.com/cyrus".to_string() - ))); - } - - #[test] - fn expand_gmail_skips_username_with_no_user_id_concept() { - let p = fixture_profile("gmail", None, Value::Null); - let rows = expand_identity_rows("gmail", &p); - - assert!(rows - .iter() - .all(|(k, _)| !matches!(k, IdentityKind::UserId | IdentityKind::Handle))); - assert!(rows.contains(&(IdentityKind::Email, "cyrus@example.com".to_string()))); - } - - #[test] - fn expand_notion_treats_username_as_user_id() { - let p = fixture_profile( - "notion", - Some("f3c1a8e2-b9b7-4a8d-9d5b-31a2e9f44e2f"), - Value::Null, - ); - let rows = expand_identity_rows("notion", &p); - - assert!(rows.contains(&( - IdentityKind::UserId, - "f3c1a8e2-b9b7-4a8d-9d5b-31a2e9f44e2f".to_string() - ))); - } - - #[test] - fn expand_unknown_toolkit_falls_back_to_handle() { - let p = fixture_profile("hypothetical", Some("alice"), Value::Null); - let rows = expand_identity_rows("hypothetical", &p); - - assert!(rows.contains(&(IdentityKind::Handle, "alice".to_string()))); - } - - #[test] - fn expand_empty_profile_emits_nothing_matchable() { - let p = ProviderUserProfile { - toolkit: "gmail".into(), - connection_id: Some("c-1".into()), - display_name: None, - email: None, - username: None, - avatar_url: None, - profile_url: None, - extras: Value::Null, - }; - let rows = expand_identity_rows("gmail", &p); - assert!(rows.is_empty()); - } - - // ── upsert wiring (uses the underlying profile_upsert directly) ─ - - #[test] - fn upsert_writes_kind_tagged_key() { - let conn = setup_db(); - - profile::profile_upsert( - &conn, - "skill-slack-conn-1-user_id", - &FacetType::Workflow, - "skill:slack:conn-1:user_id", - "U123ABC", - IdentityKind::UserId.confidence(), - None, - 1000.0, - ) - .unwrap(); - - let facets = profile_load_all(&conn).unwrap(); - let row = facets - .iter() - .find(|f| f.key == "skill:slack:conn-1:user_id") - .expect("row exists"); - assert_eq!(row.value, "U123ABC"); - assert!((row.confidence - 1.00).abs() < f64::EPSILON); - } - - #[test] - fn upsert_repeated_increments_evidence() { - let conn = setup_db(); - - for now in [1000.0, 2000.0] { - profile::profile_upsert( - &conn, - "skill-notion-default-email", - &FacetType::Workflow, - "skill:notion:default:email", - "user@workspace.com", - IdentityKind::Email.confidence(), - None, - now, - ) - .unwrap(); - } - - let facets = profile_load_all(&conn).unwrap(); - assert_eq!(facets.len(), 1); - assert_eq!(facets[0].evidence_count, 2); - } - - // ── parse_skill_identity_key ────────────────────────────────── - - #[test] - fn parse_key_round_trip() { - let parsed = parse_skill_identity_key("skill:slack:conn_1:user_id"); - assert_eq!( - parsed, - Some(( - "slack".to_string(), - "conn_1".to_string(), - "user_id".to_string() - )) - ); - } - - #[test] - fn parse_key_rejects_wrong_prefix() { - assert!(parse_skill_identity_key("preference:slack:c:email").is_none()); - } - - #[test] - fn parse_key_rejects_extra_segments() { - assert!(parse_skill_identity_key("skill:slack:c:email:extra").is_none()); - } - - // ── render ──────────────────────────────────────────────────── - - #[test] - fn render_includes_handle_with_at_and_omits_user_id() { - let rendered = render_connected_identities_section(&[ConnectedIdentity { - source: "slack".into(), - identifier: "T01ABC".into(), - display_name: Some("Cyrus Smith".into()), - email: Some("cyrus@example.com".into()), - handle: Some("cyrus".into()), - phone: None, - user_id: Some("U123ABC".into()), - avatar_url: None, - profile_url: None, - }]); - assert!(rendered.contains("## Connected Identities")); - assert!(rendered.contains("- Slack (T01ABC): Cyrus Smith | cyrus@example.com | @cyrus")); - assert!( - !rendered.contains("U123ABC"), - "user_id should not appear in prompt" - ); - } - - #[test] - fn render_empty_list_returns_empty_string() { - assert_eq!(render_connected_identities_section(&[]), ""); - } - - // ── now_secs sanity ─────────────────────────────────────────── - - #[test] - fn now_secs_returns_recent_unix_seconds() { - let t = now_secs(); - assert!(t > 1_000_000_000.0); - } - - #[test] - fn persist_returns_zero_when_memory_client_not_ready() { - // Exercise the early-return branch. Global client may or may - // not be initialised in the test binary depending on ordering. - let p = fixture_profile("gmail", None, Value::Null); - let _ = persist_provider_profile(&p); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/profile_md.rs b/crates/tinymemory-core/src/sync/composio/providers/profile_md.rs deleted file mode 100644 index f2c471d8..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/profile_md.rs +++ /dev/null @@ -1,718 +0,0 @@ -//! `PROFILE.md` markdown bridge — mirrors managed facet blocks into -//! `{workspace_dir}/PROFILE.md` so the agent prompt loader -//! (`agent/prompts/mod.rs::UserFilesSection`) picks them up on the next -//! turn. -//! -//! ## Block convention -//! -//! Each managed section lives between a pair of HTML comment markers: -//! -//! ```md -//! -//! ##
-//! -//! -//! -//! -//! ``` -//! -//! Anything outside the markers is left untouched, so user-authored prose -//! or hand-edited bullets are preserved across provider reconnects or -//! cache rebuilds. -//! -//! All operations are best-effort — errors are logged rather than -//! propagated, matching the PII-discipline pattern used in -//! `on_connection_created`. - -use super::ProviderUserProfile; -use std::fs; -use std::io; -use std::path::Path; - -// ── Legacy connected-accounts constants (kept for internal helpers) ─────────── - -const CA_BLOCK: &str = "connected-accounts"; -const CA_HEADING: &str = "## Connected Accounts"; -const FILE_HEADER: &str = "# User Profile\n"; - -/// All managed block names, in the order they are appended when a new -/// `PROFILE.md` is created. -pub const BLOCKS: &[&str] = &[ - "connected-accounts", // written by provider path (merge_provider_into_profile_md) - "style", - "identity", - "tooling", - "vetoes", - "goals", -]; - -// ── Public API ──────────────────────────────────────────────────────────────── - -/// Upsert the per-toolkit bullet for `profile` inside the managed -/// `connected-accounts` block of `{workspace_dir}/PROFILE.md`. -/// -/// Creates the file with a `# User Profile` header if it does not exist. -/// Idempotent — re-connecting the same toolkit replaces the existing -/// bullet rather than duplicating it. -pub fn merge_provider_into_profile_md( - workspace_dir: &Path, - profile: &ProviderUserProfile, -) -> io::Result<()> { - let toolkit = normalize_token(&profile.toolkit); - if toolkit.is_empty() { - return Ok(()); - } - // Require a real connection_id so the bullet keys match what the - // disconnect path (`composio_delete_connection`) will look up. - let identifier = profile - .connection_id - .as_deref() - .map(normalize_token) - .filter(|v| !v.is_empty()); - let identifier = match identifier { - Some(id) => id, - None => { - tracing::debug!( - toolkit = %toolkit, - "[composio:profile_md] skipping merge — connection_id missing or empty" - ); - return Ok(()); - } - }; - - let bullet = match render_provider_bullet(&toolkit, &identifier, profile) { - Some(b) => b, - None => return Ok(()), - }; - - let path = workspace_dir.join("PROFILE.md"); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - let existing = match fs::read_to_string(&path) { - Ok(s) => s, - Err(e) if e.kind() == io::ErrorKind::NotFound => String::new(), - Err(e) => return Err(e), - }; - - let updated = upsert_provider_bullet(&existing, &toolkit, &identifier, &bullet); - fs::write(&path, updated)?; - tracing::debug!( - target_file = "PROFILE.md", - toolkit = %toolkit, - identifier = %identifier, - "[composio:profile_md] merged provider profile into PROFILE.md" - ); - Ok(()) -} - -/// Remove the per-toolkit bullet for `(source, identifier)` from the -/// managed Connected Accounts block. If the block becomes empty the whole -/// block is dropped. Missing file or missing block are no-ops. -pub fn remove_provider_from_profile_md( - workspace_dir: &Path, - source: &str, - identifier: &str, -) -> io::Result<()> { - let path = workspace_dir.join("PROFILE.md"); - let existing = match fs::read_to_string(&path) { - Ok(s) => s, - Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), - Err(e) => return Err(e), - }; - let toolkit = normalize_token(source); - let identifier = normalize_token(identifier); - if toolkit.is_empty() || identifier.is_empty() { - return Ok(()); - } - let updated = remove_provider_bullet(&existing, &toolkit, &identifier); - if updated != existing { - fs::write(&path, updated)?; - tracing::debug!( - target_file = "PROFILE.md", - toolkit = %toolkit, - identifier = %identifier, - "[composio:profile_md] removed provider bullet from PROFILE.md" - ); - } - Ok(()) -} - -/// Upsert a generic managed block. -/// -/// * `block_name` — one of [`BLOCKS`] (e.g. `"style"`, `"identity"`). -/// * `section_heading` — heading rendered inside the block (e.g. `"## Style"`). -/// * `body_markdown` — pre-rendered content (bullets, prose). Must not -/// contain the block markers themselves. -/// -/// Creates `PROFILE.md` if it does not exist. If the block is absent it is -/// appended at the end of the file. If the block exists its body is replaced -/// in-place — content outside the markers is left byte-for-byte untouched. -/// -/// An empty `body_markdown` renders a `*(no entries yet)*` placeholder -/// instead of deleting the block; this preserves the block's position for the -/// next write. -/// -/// Idempotent: calling with the same inputs twice produces the same file. -pub fn replace_managed_block( - workspace_dir: &Path, - block_name: &str, - section_heading: &str, - body_markdown: String, -) -> io::Result<()> { - let path = workspace_dir.join("PROFILE.md"); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - - let existing = match fs::read_to_string(&path) { - Ok(s) => s, - Err(e) if e.kind() == io::ErrorKind::NotFound => String::new(), - Err(e) => return Err(e), - }; - - let updated = upsert_block(&existing, block_name, section_heading, &body_markdown); - fs::write(&path, updated)?; - tracing::debug!( - block_name = %block_name, - "[composio:profile_md] replaced managed block '{}' in PROFILE.md", - block_name - ); - Ok(()) -} - -// ── Connected-accounts internals ────────────────────────────────────────────── - -fn render_provider_bullet( - toolkit: &str, - identifier: &str, - profile: &ProviderUserProfile, -) -> Option { - let mut fields: Vec = Vec::new(); - if let Some(v) = profile.display_name.as_deref().map(sanitize) { - if !v.is_empty() { - fields.push(v); - } - } - if let Some(v) = profile.email.as_deref().map(sanitize) { - if !v.is_empty() { - fields.push(v); - } - } - if let Some(v) = profile.username.as_deref().map(sanitize) { - if !v.is_empty() { - fields.push(format!("@{v}")); - } - } - if let Some(v) = profile.profile_url.as_deref().map(sanitize) { - if !v.is_empty() { - fields.push(v); - } - } - if fields.is_empty() { - return None; - } - let marker = bullet_marker(toolkit, identifier); - Some(format!( - "- {marker} **{title}** ({identifier}): {fields}", - title = title_case(toolkit), - identifier = identifier, - fields = fields.join(" | ") - )) -} - -fn bullet_marker(toolkit: &str, identifier: &str) -> String { - format!("") -} - -/// Insert or replace `bullet` inside the connected-accounts managed block. -fn upsert_provider_bullet(existing: &str, toolkit: &str, identifier: &str, bullet: &str) -> String { - let marker = bullet_marker(toolkit, identifier); - let start_tag = block_start(CA_BLOCK); - let end_tag = block_end(CA_BLOCK); - let (prefix, block_body, suffix) = split_any_block(existing, &start_tag, &end_tag); - - let mut lines: Vec = block_body - .lines() - .filter(|l| !l.contains(&marker)) - .map(|l| l.to_string()) - .collect(); - lines.push(bullet.to_string()); - - let mut bullets = lines - .into_iter() - .filter(|l| l.trim_start().starts_with("- ") -} - -/// Build the end marker for `block_name`. -pub fn block_end(block_name: &str) -> String { - format!("") -} - -/// Insert or replace a generic managed block in `existing`. -/// -/// If the block is absent it is appended. If it exists its body (between the -/// markers) is replaced. Content outside the markers is returned unchanged. -fn upsert_block( - existing: &str, - block_name: &str, - section_heading: &str, - body_markdown: &str, -) -> String { - let start_tag = block_start(block_name); - let end_tag = block_end(block_name); - - let body = if body_markdown.trim().is_empty() { - "*(no entries yet)*".to_string() - } else { - body_markdown.to_string() - }; - - let block = format!("{start_tag}\n{section_heading}\n\n{body}\n\n{end_tag}"); - - let (prefix, _old_body, suffix) = split_any_block(existing, &start_tag, &end_tag); - - if prefix == existing { - // Block was absent — append. - assemble(existing, &block, "") - } else { - assemble(&prefix, &block, &suffix) - } -} - -/// Split `existing` around the markers `[start_tag, end_tag]`. -/// -/// Returns `(prefix, block_body, suffix)`. If no block is present, -/// `prefix` is the full string and `block_body` / `suffix` are empty. -/// `block_body` is the content *between* the markers (excluding the -/// markers themselves). -fn split_any_block(existing: &str, start_tag: &str, end_tag: &str) -> (String, String, String) { - if let (Some(start), Some(end)) = (existing.find(start_tag), existing.find(end_tag)) { - if end > start { - let prefix = existing[..start].to_string(); - let body = existing[start + start_tag.len()..end].to_string(); - let suffix_start = end + end_tag.len(); - let suffix = existing[suffix_start..].to_string(); - return (prefix, body, suffix); - } - } - (existing.to_string(), String::new(), String::new()) -} - -/// Assemble `prefix + block + suffix`, normalising the newlines immediately -/// adjacent to the managed block while leaving the user's bytes elsewhere -/// untouched. -fn assemble(prefix: &str, block: &str, suffix: &str) -> String { - if block.is_empty() { - // Removing the block entirely. - let p = prefix.trim_end_matches('\n'); - let s = suffix.trim_start_matches('\n'); - let mut out = String::with_capacity(p.len() + s.len() + 2); - out.push_str(p); - if !p.is_empty() { - out.push('\n'); - if !s.is_empty() { - out.push('\n'); - } - } - out.push_str(s); - if !out.is_empty() && !out.ends_with('\n') { - out.push('\n'); - } - return out; - } - - let mut out = String::new(); - if prefix.trim().is_empty() { - // Seed with a header on first creation. - out.push_str(FILE_HEADER); - out.push('\n'); - } else { - let p = prefix.trim_end_matches('\n'); - out.push_str(p); - out.push_str("\n\n"); - } - out.push_str(block); - if suffix.is_empty() { - out.push('\n'); - } else { - let s = suffix.trim_start_matches('\n'); - if s.is_empty() { - // Suffix was only whitespace — end with single newline, no blank line. - out.push('\n'); - } else { - out.push_str("\n\n"); - out.push_str(s); - if !out.ends_with('\n') { - out.push('\n'); - } - } - } - out -} - -// ── Token / string helpers ──────────────────────────────────────────────────── - -fn normalize_token(raw: &str) -> String { - let mut out = String::with_capacity(raw.len()); - for ch in raw.chars() { - let lower = ch.to_ascii_lowercase(); - if lower.is_ascii_alphanumeric() || lower == '-' || lower == '_' { - out.push(lower); - } else { - out.push('_'); - } - } - out.trim_matches('_').to_string() -} - -fn title_case(raw: &str) -> String { - let mut chars = raw.chars(); - match chars.next() { - Some(first) => first.to_ascii_uppercase().to_string() + chars.as_str(), - None => String::new(), - } -} - -fn sanitize(raw: &str) -> String { - let replaced = raw.replace(['\n', '\r', '\t'], " ").replace('|', "/"); - replaced.split_whitespace().collect::>().join(" ") -} - -// ── Tests ───────────────────────────────────────────────────────────────────── - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - // ── merge_provider_into_profile_md (legacy API, unchanged) ─────────────── - - fn sample(toolkit: &str, conn: &str) -> ProviderUserProfile { - ProviderUserProfile { - toolkit: toolkit.into(), - connection_id: Some(conn.into()), - display_name: Some("Jane Doe".into()), - email: Some("jane@example.com".into()), - username: Some("janedoe".into()), - avatar_url: None, - profile_url: Some("https://example.com/jane".into()), - extras: serde_json::Value::Null, - } - } - - #[test] - fn creates_file_when_missing() { - let tmp = TempDir::new().unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(body.starts_with("# User Profile"), "body was:\n{body}"); - let start = block_start(CA_BLOCK); - let end = block_end(CA_BLOCK); - assert!(body.contains(&start)); - assert!(body.contains(CA_HEADING)); - assert!(body.contains("**Gmail** (c-1):")); - assert!(body.contains("jane@example.com")); - assert!(body.contains("@janedoe")); - assert!(body.contains(&end)); - } - - #[test] - fn upsert_is_idempotent_for_same_toolkit_connection() { - let tmp = TempDir::new().unwrap(); - let mut p = sample("gmail", "c-1"); - merge_provider_into_profile_md(tmp.path(), &p).unwrap(); - p.display_name = Some("Jane D.".into()); - merge_provider_into_profile_md(tmp.path(), &p).unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - let occurrences = body.matches("acct:gmail:c-1").count(); - assert_eq!(occurrences, 1, "duplicate bullet:\n{body}"); - assert!(body.contains("Jane D.")); - assert!(!body.contains("Jane Doe")); - } - - #[test] - fn multiple_toolkits_render_separate_bullets() { - let tmp = TempDir::new().unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("twitter", "c-2")).unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(body.contains("acct:gmail:c-1")); - assert!(body.contains("acct:twitter:c-2")); - let start = block_start(CA_BLOCK); - let end = block_end(CA_BLOCK); - assert_eq!(body.matches(&start).count(), 1); - assert_eq!(body.matches(&end).count(), 1); - } - - #[test] - fn preserves_user_authored_content_outside_block() { - let tmp = TempDir::new().unwrap(); - let path = tmp.path().join("PROFILE.md"); - fs::write( - &path, - "# User Profile\n\nSome bio paragraph from LinkedIn.\n\n## Key facts\n- a\n- b\n", - ) - .unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - let body = fs::read_to_string(&path).unwrap(); - assert!(body.contains("Some bio paragraph from LinkedIn.")); - assert!(body.contains("## Key facts")); - assert!(body.contains("- a")); - assert!(body.contains("acct:gmail:c-1")); - } - - #[test] - fn skips_when_no_useful_fields() { - let tmp = TempDir::new().unwrap(); - let p = ProviderUserProfile { - toolkit: "gmail".into(), - connection_id: Some("c-1".into()), - display_name: Some(" ".into()), - email: None, - username: Some("".into()), - avatar_url: None, - profile_url: None, - extras: serde_json::Value::Null, - }; - merge_provider_into_profile_md(tmp.path(), &p).unwrap(); - assert!(!tmp.path().join("PROFILE.md").exists()); - } - - #[test] - fn remove_drops_specific_bullet() { - let tmp = TempDir::new().unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("twitter", "c-2")).unwrap(); - remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(!body.contains("acct:gmail:c-1")); - assert!(body.contains("acct:twitter:c-2")); - } - - #[test] - fn remove_drops_block_when_empty() { - let tmp = TempDir::new().unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - let start = block_start(CA_BLOCK); - let end = block_end(CA_BLOCK); - assert!(!body.contains(&start), "block remained:\n{body}"); - assert!(!body.contains(&end)); - assert!(body.starts_with("# User Profile")); - } - - #[test] - fn remove_is_noop_when_file_missing() { - let tmp = TempDir::new().unwrap(); - remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); - assert!(!tmp.path().join("PROFILE.md").exists()); - } - - #[test] - fn skips_when_connection_id_missing() { - let tmp = TempDir::new().unwrap(); - let p = ProviderUserProfile { - toolkit: "gmail".into(), - connection_id: None, - display_name: Some("Jane".into()), - email: Some("jane@example.com".into()), - username: None, - avatar_url: None, - profile_url: None, - extras: serde_json::Value::Null, - }; - merge_provider_into_profile_md(tmp.path(), &p).unwrap(); - assert!(!tmp.path().join("PROFILE.md").exists()); - } - - #[test] - fn preserves_indentation_and_blank_lines_around_block() { - let tmp = TempDir::new().unwrap(); - let path = tmp.path().join("PROFILE.md"); - let original = "# User Profile\n\n indented bio line\n\n## Notes\n- alpha\n- beta\n\n"; - fs::write(&path, original).unwrap(); - merge_provider_into_profile_md(tmp.path(), &sample("gmail", "c-1")).unwrap(); - let body = fs::read_to_string(&path).unwrap(); - assert!(body.contains(" indented bio line")); - assert!(body.contains("## Notes\n- alpha\n- beta")); - let start = block_start(CA_BLOCK); - let end = block_end(CA_BLOCK); - assert!(body.contains(&start) && body.contains(&end)); - remove_provider_from_profile_md(tmp.path(), "gmail", "c-1").unwrap(); - let after = fs::read_to_string(&path).unwrap(); - assert!(after.contains(" indented bio line")); - assert!(after.contains("## Notes\n- alpha\n- beta")); - assert!(!after.contains(&start)); - } - - #[test] - fn sanitize_strips_pipes_and_newlines() { - assert_eq!(sanitize("foo\nbar"), "foo bar"); - assert_eq!(sanitize("a | b"), "a / b"); - assert_eq!(sanitize(" multi space "), "multi space"); - } - - // ── replace_managed_block ───────────────────────────────────────────────── - - #[test] - fn replace_managed_block_creates_file_if_missing() { - let tmp = TempDir::new().unwrap(); - replace_managed_block( - tmp.path(), - "style", - "## Style", - "- **verbosity**: terse".into(), - ) - .unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(body.contains("# User Profile"), "missing header:\n{body}"); - assert!(body.contains(&block_start("style"))); - assert!(body.contains("## Style")); - assert!(body.contains("- **verbosity**: terse")); - assert!(body.contains(&block_end("style"))); - } - - #[test] - fn replace_managed_block_appends_block_when_absent() { - let tmp = TempDir::new().unwrap(); - let path = tmp.path().join("PROFILE.md"); - fs::write(&path, "# User Profile\n\nSome existing text.\n").unwrap(); - replace_managed_block( - tmp.path(), - "identity", - "## Identity", - "- **name**: Alice".into(), - ) - .unwrap(); - let body = fs::read_to_string(&path).unwrap(); - // Existing content preserved. - assert!(body.contains("Some existing text.")); - // New block appended. - assert!(body.contains(&block_start("identity"))); - assert!(body.contains("## Identity")); - assert!(body.contains("- **name**: Alice")); - } - - #[test] - fn replace_managed_block_replaces_body_in_place() { - let tmp = TempDir::new().unwrap(); - replace_managed_block( - tmp.path(), - "style", - "## Style", - "- **verbosity**: verbose".into(), - ) - .unwrap(); - replace_managed_block( - tmp.path(), - "style", - "## Style", - "- **verbosity**: terse".into(), - ) - .unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(body.contains("terse")); - assert!(!body.contains("verbose")); - // Only one start marker. - assert_eq!(body.matches(&block_start("style")).count(), 1); - } - - #[test] - fn replace_managed_block_preserves_other_blocks_and_user_text() { - let tmp = TempDir::new().unwrap(); - // Write two blocks. - replace_managed_block( - tmp.path(), - "style", - "## Style", - "- **verbosity**: terse".into(), - ) - .unwrap(); - replace_managed_block( - tmp.path(), - "identity", - "## Identity", - "- **name**: Bob".into(), - ) - .unwrap(); - // Update only style. - replace_managed_block( - tmp.path(), - "style", - "## Style", - "- **verbosity**: verbose".into(), - ) - .unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - // Identity block untouched. - assert!(body.contains("- **name**: Bob")); - // Style updated. - assert!(body.contains("verbose")); - assert!(!body.contains("terse")); - } - - #[test] - fn replace_managed_block_empty_body_renders_placeholder() { - let tmp = TempDir::new().unwrap(); - replace_managed_block(tmp.path(), "goals", "## Goals", String::new()).unwrap(); - let body = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert!(body.contains("*(no entries yet)*")); - // Block markers still present. - assert!(body.contains(&block_start("goals"))); - assert!(body.contains(&block_end("goals"))); - } - - #[test] - fn replace_managed_block_idempotent_on_repeat_invocation() { - let tmp = TempDir::new().unwrap(); - let content = "- **verbosity**: terse".to_string(); - replace_managed_block(tmp.path(), "style", "## Style", content.clone()).unwrap(); - let body1 = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - replace_managed_block(tmp.path(), "style", "## Style", content).unwrap(); - let body2 = fs::read_to_string(tmp.path().join("PROFILE.md")).unwrap(); - assert_eq!(body1, body2, "second write should be idempotent"); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/registry.rs b/crates/tinymemory-core/src/sync/composio/providers/registry.rs deleted file mode 100644 index c08e79b9..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/registry.rs +++ /dev/null @@ -1,151 +0,0 @@ -//! Process-global registry of [`ComposioProvider`] implementations. -//! -//! There is exactly one provider per toolkit slug — the trait is not -//! a fan-out fan-in dispatch, it is a 1:1 mapping. This keeps trigger -//! routing simple (`HashMap::get(toolkit)` → call) and avoids the -//! "which subscriber wins" ambiguity that would come with multiple -//! providers per toolkit. -//! -//! The registry is initialised once at startup via -//! [`init_default_providers`] and is intentionally write-rare: tests -//! can register additional providers ad-hoc, but the production path -//! only writes during the startup hook. - -use std::collections::HashMap; -use std::sync::{Arc, OnceLock, RwLock}; - -use super::ComposioProvider; - -/// Reference-counted handle to a registered provider. -pub type ProviderArc = Arc; - -/// Backing storage for the global registry. -/// -/// `RwLock>` is fine here — registration happens at -/// startup and lookups are very fast (no contention in steady state). -type Registry = RwLock>; - -static REGISTRY: OnceLock = OnceLock::new(); - -fn registry() -> &'static Registry { - REGISTRY.get_or_init(|| RwLock::new(HashMap::new())) -} - -/// Register or replace a provider for its toolkit slug. -/// -/// Idempotent — re-registering the same toolkit overwrites the -/// previous entry, which is what tests rely on for setup/teardown. -pub fn register_provider(provider: ProviderArc) { - let slug = provider.toolkit_slug().to_string(); - if slug.is_empty() { - tracing::warn!("[composio:registry] refusing to register provider with empty slug"); - return; - } - let mut guard = registry() - .write() - .expect("composio provider registry poisoned"); - let was_present = guard.insert(slug.clone(), provider).is_some(); - if was_present { - tracing::debug!(toolkit = %slug, "[composio:registry] replaced existing provider"); - } else { - tracing::info!(toolkit = %slug, "[composio:registry] provider registered"); - } -} - -/// Look up the provider for a toolkit slug, if one is registered. -pub fn get_provider(toolkit: &str) -> Option { - let key = toolkit.trim(); - if key.is_empty() { - return None; - } - let guard = registry() - .read() - .expect("composio provider registry poisoned"); - guard.get(key).cloned() -} - -/// Snapshot of every registered provider, in unspecified order. Used -/// by the periodic sync scheduler to walk every toolkit. -pub fn all_providers() -> Vec { - let guard = registry() - .read() - .expect("composio provider registry poisoned"); - guard.values().cloned().collect() -} - -/// Register the built-in providers shipped with the core. Called once -/// from `start_channels` / `bootstrap_core_runtime` startup paths. -/// -/// Idempotent: re-running just re-registers (no-op in practice). -pub fn init_default_providers() { - register_provider(Arc::new(super::clickup::ClickUpProvider::new())); - register_provider(Arc::new(super::github::GitHubProvider::new())); - register_provider(Arc::new(super::gmail::GmailProvider::new())); - register_provider(Arc::new(super::linear::LinearProvider::new())); - register_provider(Arc::new(super::notion::NotionProvider::new())); - register_provider(Arc::new(super::slack::SlackProvider::new())); - tracing::info!( - count = all_providers().len(), - "[composio:registry] default providers initialised" - ); -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::sync::composio::providers::{ProviderContext, ProviderUserProfile}; - use async_trait::async_trait; - - struct DummyProvider { - slug: &'static str, - } - - #[async_trait] - impl ComposioProvider for DummyProvider { - fn toolkit_slug(&self) -> &'static str { - self.slug - } - async fn fetch_user_profile( - &self, - _ctx: &ProviderContext, - ) -> Result { - Ok(ProviderUserProfile::default()) - } - } - - #[test] - fn register_and_lookup_roundtrip() { - register_provider(Arc::new(DummyProvider { - slug: "test_dummy_a", - })); - let p = get_provider("test_dummy_a").expect("provider should be registered"); - assert_eq!(p.toolkit_slug(), "test_dummy_a"); - } - - #[test] - fn lookup_unknown_returns_none() { - assert!(get_provider("__definitely_not_a_real_toolkit__").is_none()); - } - - #[test] - fn register_replaces_existing() { - register_provider(Arc::new(DummyProvider { - slug: "test_dummy_b", - })); - register_provider(Arc::new(DummyProvider { - slug: "test_dummy_b", - })); - // Still exactly one entry under that slug. - let count_with_b = all_providers() - .iter() - .filter(|p| p.toolkit_slug() == "test_dummy_b") - .count(); - assert_eq!(count_with_b, 1); - } - - #[test] - fn empty_slug_is_rejected() { - register_provider(Arc::new(DummyProvider { slug: "" })); - assert!(get_provider("").is_none()); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/scope_lookup.rs b/crates/tinymemory-core/src/sync/composio/providers/scope_lookup.rs deleted file mode 100644 index e3bae028..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/scope_lookup.rs +++ /dev/null @@ -1,78 +0,0 @@ -//! Scope-lookup operational helpers for the curated tool catalogs. -//! -//! Lives in a sibling module (extracted from the formerly-thick -//! `providers/mod.rs`) to keep the module entrypoint export-focused — -//! matches the project rule "keep mod.rs light; operational logic in -//! ops.rs / store.rs / types.rs" from CLAUDE.md. -//! -//! - [`curated_scope_for`] answers "what scope does this action slug -//! require?" — used by `composio::ops` to render `gated_tools` -//! unlock hints. -//! - [`toolkit_has_scope`] answers "does this toolkit have any -//! actions at the given scope?" — currently used by tests; intended -//! for future UI hints (grey-out a toggle that unlocks nothing). -//! -//! Both walk the native provider catalog first, then fall back to the -//! static `catalog_for_toolkit` map — so the answers match what -//! [`super::is_action_visible_with_pref`] would gate against. - -use super::tool_scope::{find_curated, toolkit_from_slug, ToolScope}; -use super::{catalog_for_toolkit, get_provider}; - -/// Look up the curated scope for `slug` if it appears in any registered -/// catalog (native provider's `curated_tools()` first, then the fallback -/// catalog from [`super::catalog_for_toolkit`]). Returns `None` for -/// genuinely uncurated slugs — callers that want a defensible heuristic -/// for those should fall back to [`super::classify_unknown`] explicitly. -/// -/// Sibling of [`super::is_action_visible_with_pref`]: that one decides -/// "visible?", this one returns "what scope is required?" so callers -/// (e.g. the `gated_tools` partition in -/// `composio::ops::fetch_connected_integrations`) can render a useful -/// unlock hint to the agent without re-doing the catalog walk. -pub fn curated_scope_for(slug: &str) -> Option { - let toolkit = toolkit_from_slug(slug)?; - let catalog = get_provider(&toolkit) - .and_then(|p| p.curated_tools()) - .or_else(|| catalog_for_toolkit(&toolkit))?; - find_curated(catalog, slug).map(|c| c.scope) -} - -/// Does any curated action for `toolkit` require `scope`? -/// -/// Currently used by this module's tests only (added when the -/// now-removed `composio_enable_scope` meta-tool needed a no-op -/// short-circuit). Kept because the same probe is useful any time we -/// ask "would flipping the {scope} bit unlock anything in this -/// toolkit?" — e.g. a UI hint that greys out a toggle with no effect. -/// -/// Walks both the native provider catalog and the fallback -/// [`super::catalog_for_toolkit`] so the answer matches what -/// [`super::is_action_visible_with_pref`] would gate against. -pub fn toolkit_has_scope(toolkit: &str, scope: ToolScope) -> bool { - let catalog = get_provider(toolkit) - .and_then(|p| p.curated_tools()) - .or_else(|| catalog_for_toolkit(toolkit)); - match catalog { - Some(cat) => cat.iter().any(|t| t.scope == scope), - None => false, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn toolkit_has_scope_distinguishes_gated_from_ungated_scopes() { - // gmail catalog includes destructive verbs (delete / trash / - // batch_delete), so admin-gating actually unlocks something. - assert!(toolkit_has_scope("gmail", ToolScope::Admin)); - assert!(toolkit_has_scope("gmail", ToolScope::Read)); - assert!(toolkit_has_scope("gmail", ToolScope::Write)); - // Case-insensitive toolkit slug → still routes to the catalog. - assert!(toolkit_has_scope("GMAIL", ToolScope::Admin)); - // Unknown toolkit → no catalog → no scope is "gating" anything. - assert!(!toolkit_has_scope("nonexistent-toolkit", ToolScope::Admin)); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/slack/mod.rs b/crates/tinymemory-core/src/sync/composio/providers/slack/mod.rs deleted file mode 100644 index 39b745a5..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/slack/mod.rs +++ /dev/null @@ -1,19 +0,0 @@ -//! Composio-backed Slack provider. -//! -//! The provider is wired into the periodic-sync scheduler (see -//! [`super::registry::init_default_providers`]) and fires -//! `SLACK_LIST_CONVERSATIONS` + `SLACK_FETCH_CONVERSATION_HISTORY` -//! against the user's Composio-authorized Slack connection. The reusable -//! synchronization and ingestion engine is owned by tinycortex. - -// The Slack post-processor moved to tinycortex (a pure Value transform, i.e. -// driver-side). Re-exported under the old module name — `pub`, not a plain -// `use`, because `tests/raw_coverage/memory_threads_raw_coverage_e2e.rs` -// imports this path directly. -pub use tinymemory_sync::slack_post_process as post_process; -pub mod types; - -mod provider; - -pub use provider::{run_backfill_via_search, SlackProvider, BACKFILL_DAYS}; -pub use types::{SlackChannel, SlackMessage}; diff --git a/crates/tinymemory-core/src/sync/composio/providers/slack/provider.rs b/crates/tinymemory-core/src/sync/composio/providers/slack/provider.rs deleted file mode 100644 index 3256d95c..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/slack/provider.rs +++ /dev/null @@ -1,337 +0,0 @@ -//! Composio-backed Slack provider. -//! -//! Drives Slack history ingestion **without** a user-managed bot token -//! — authorization lives in the user's Composio Slack connection, and -//! the actual API calls fan out through [`ComposioClient::execute_tool`] -//! against Composio's action catalog (`SLACK_LIST_CONVERSATIONS`, -//! `SLACK_FETCH_CONVERSATION_HISTORY`, `SLACK_FETCH_TEAM_INFO`, …). -//! -//! The product provider retains profile lookup, trigger handling, and response -//! normalization. Channel enumeration, cursors, history paging, and memory -//! ingestion execute in tinycortex's Slack sync pipeline. -//! -//! ## Idempotency -//! -//! Source id is `slack:{connection_id}` — stable per workspace. Chunk -//! IDs are stable, so repeated synchronization updates the same documents. - -use crate::sync::composio::providers::{ - pick_str, resolve_sync_interval_secs, ComposioProvider, CuratedTool, ProviderContext, - ProviderUserProfile, SyncOutcome, -}; -use async_trait::async_trait; -use serde_json::{json, Value}; - -/// Composio action slug for team/workspace profile fetch. -const ACTION_FETCH_TEAM_INFO: &str = "SLACK_FETCH_TEAM_INFO"; -/// Composio action slug for Slack `auth.test` — returns the authed -/// user's id, handle, and team. Required for self-identity capture. -const ACTION_AUTH_TEST: &str = "SLACK_TEST_AUTH"; -/// Composio action slug for Slack `users.info` — returns the user's -/// profile (email, real_name, avatar). Optional; needs `users:read.email` -/// scope for the email field. -const ACTION_USERS_INFO: &str = "SLACK_RETRIEVE_DETAILED_USER_INFORMATION"; - -/// Default backfill window (days) applied when a channel has no -/// cursor yet. -pub const BACKFILL_DAYS: i64 = 6; - -/// Sync cadence for provider catalog scheduling. -const SYNC_INTERVAL_SECS: u64 = 15 * 60; - -pub struct SlackProvider; - -impl SlackProvider { - pub fn new() -> Self { - Self - } -} - -impl Default for SlackProvider { - fn default() -> Self { - Self::new() - } -} - -#[async_trait] -impl ComposioProvider for SlackProvider { - fn toolkit_slug(&self) -> &'static str { - "slack" - } - - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - Some(crate::sync::composio::providers::catalogs::SLACK_CURATED) - } - - fn sync_interval_secs(&self) -> Option { - Some(resolve_sync_interval_secs("slack", SYNC_INTERVAL_SECS)) - } - - fn post_process_action_result( - &self, - slug: &str, - arguments: Option<&serde_json::Value>, - data: &mut serde_json::Value, - ) { - super::post_process::post_process(slug, arguments, data); - } - - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result { - tracing::debug!( - connection_id = ?ctx.connection_id, - "[composio:slack] fetch_user_profile via {ACTION_AUTH_TEST}" - ); - - // Step 1 — auth.test: required. Returns user_id (canonical sender - // id on Slack messages), the user's handle, and the team. - let auth_resp = ctx - .execute(ACTION_AUTH_TEST, Some(json!({}))) - .await - .map_err(|e| format!("[composio:slack] {ACTION_AUTH_TEST} failed: {e:#}"))?; - - if !auth_resp.successful { - let err = auth_resp - .error - .clone() - .unwrap_or_else(|| "provider reported failure".to_string()); - return Err(format!("[composio:slack] {ACTION_AUTH_TEST}: {err}")); - } - - // `auth_data` is the inner Composio payload — paths are relative - // to it. Slack's auth.test returns user_id/user/team/team_id at - // the top of `data`. - let auth_data = &auth_resp.data; - let user_id = pick_str(auth_data, &["user_id"]); - let handle = pick_str(auth_data, &["user"]); - let team_id = pick_str(auth_data, &["team_id"]); - let team_name = pick_str(auth_data, &["team"]); - - // Step 2 — users.info: optional. Needs `users:read.email` scope - // for `email`; falls back to `auth.test` data on missing-scope or - // any other failure so the profile still carries user_id+handle. - let mut display_name: Option = None; - let mut email: Option = None; - let mut avatar_url: Option = None; - - if let Some(uid) = user_id.as_deref() { - match ctx - .execute(ACTION_USERS_INFO, Some(json!({ "user": uid }))) - .await - { - Ok(info) if info.successful => { - let d = &info.data; - email = pick_str(d, &["user.profile.email", "profile.email"]); - display_name = pick_str( - d, - &[ - "user.profile.real_name", - "user.real_name", - "user.profile.display_name", - ], - ); - avatar_url = pick_str(d, &["user.profile.image_192", "user.profile.image_72"]); - } - Ok(info) => { - tracing::info!( - connection_id = ?ctx.connection_id, - error = ?info.error, - "[composio:slack] {ACTION_USERS_INFO} returned non-success — \ - falling back to auth.test data only (likely missing users:read scope)" - ); - } - Err(e) => { - tracing::info!( - connection_id = ?ctx.connection_id, - error = %e, - "[composio:slack] {ACTION_USERS_INFO} call failed — \ - falling back to auth.test data only" - ); - } - } - } - - // Step 3 — team_info: optional. Adds workspace context to `extras` - // (email_domain, icon) so the prompt section / UI can show it. - let (team_email_domain, team_icon) = - match ctx.execute(ACTION_FETCH_TEAM_INFO, Some(json!({}))).await { - Ok(resp) if resp.successful => { - let d = &resp.data; - let domain = pick_str(d, &["team.email_domain", "email_domain"]); - let icon = pick_str(d, &["team.icon.image_132", "team.icon.image_68"]); - (domain, icon) - } - _ => (None, None), - }; - - // Display name preference: users.info real_name > auth.test handle - // > team_name (last-resort so the prompt isn't empty). - let final_display_name = display_name - .clone() - .or_else(|| handle.clone()) - .or_else(|| team_name.clone()); - - // Profile URL: users.info doesn't return one for the user - // directly; the workspace URL is acceptable as a navigational - // fallback. (Slack user profile pages are workspace-scoped and - // not stably linkable from auth.test alone.) - let profile_url = pick_str(auth_data, &["url"]); - - let avatar_url = avatar_url.or(team_icon); - - let profile = ProviderUserProfile { - toolkit: "slack".to_string(), - connection_id: ctx.connection_id.clone(), - display_name: final_display_name, - email, - // username carries the platform-canonical sender id so the - // self-identity matcher can compare against Slack message - // sender_user_id directly. Handle moves into `extras` — - // `expand_identity_rows` lifts it back out as IdentityKind::Handle. - username: user_id, - avatar_url, - profile_url, - extras: json!({ - "handle": handle, - "team_id": team_id, - "team_name": team_name, - "team_email_domain": team_email_domain, - }), - }; - - let has_email = profile.email.is_some(); - let email_domain = profile - .email - .as_deref() - .and_then(|e| e.split('@').nth(1)) - .map(|d| d.to_string()); - tracing::info!( - connection_id = ?profile.connection_id, - has_email, - email_domain = ?email_domain, - has_user_id = profile.username.is_some(), - "[composio:slack] fetched user profile" - ); - Ok(profile) - } - - /// Slack rides the generic orchestrator. Channel enumeration + the user - /// directory backfill happen in `super::source::SlackSource::preamble`; - /// per-channel `conversations.history` pagination, the per-channel `oldest` - /// watermark, dedup, the `max_items` cap, and per-channel error tolerance - /// all live in `run_sync`. The Slack-specific primitives live in - /// `super::source`. - async fn on_trigger( - &self, - ctx: &ProviderContext, - trigger: &str, - _payload: &Value, - ) -> Result<(), String> { - if trigger.to_ascii_uppercase().contains("MESSAGE") { - let Some(connection_id) = ctx.connection_id.as_deref() else { - return Err("[composio:slack] trigger missing connection_id".to_string()); - }; - if let Err(e) = crate::sync::pipelines::host::run_composio_connection( - "slack", - connection_id, - ctx.config.as_ref(), - None, - None, - ) - .await - { - tracing::warn!( - error = %e, - "[composio:slack] trigger-driven sync failed (non-fatal)" - ); - } - } - Ok(()) - } -} - -// ── Search-based backfill (one-shot) ──────────────────────────────── - -/// Compatibility wrapper for the tinycortex workspace-wide search pipeline. -pub async fn run_backfill_via_search( - ctx: &ProviderContext, - backfill_days: i64, -) -> Result { - let connection_id = ctx - .connection_id - .as_deref() - .ok_or_else(|| "[composio:slack] search backfill missing connection_id".to_string())?; - let started_at_ms = now_ms(); - let outcome = crate::sync::pipelines::host::run_slack_search_backfill( - connection_id, - backfill_days, - ctx.config.as_ref(), - ) - .await - .map_err(|error| error.to_string())?; - Ok(SyncOutcome { - toolkit: "slack".into(), - connection_id: Some(connection_id.into()), - reason: "manual".into(), - items_ingested: outcome.records_ingested as usize, - started_at_ms, - finished_at_ms: now_ms(), - summary: outcome - .note - .unwrap_or_else(|| "slack search-backfill complete".into()), - details: serde_json::json!({ - "more_pending": outcome.more_pending, - "actions_called": outcome.actions_called, - "provider_cost_usd": outcome.provider_cost_usd, - }), - }) -} - -fn now_ms() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64 -} -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn toolkit_slug_is_stable() { - assert_eq!(SlackProvider::new().toolkit_slug(), "slack"); - } - - #[test] - fn sync_interval_matches_constant() { - assert_eq!( - SlackProvider::new().sync_interval_secs(), - Some(SYNC_INTERVAL_SECS) - ); - } - - #[test] - fn curated_tools_returns_slack_catalog() { - let tools = SlackProvider::new().curated_tools().unwrap(); - assert!(tools - .iter() - .any(|t| t.slug == "SLACK_FETCH_CONVERSATION_HISTORY")); - assert!(tools.iter().any(|t| t.slug == "SLACK_LIST_CONVERSATIONS")); - } - - #[test] - fn post_process_action_result_delegates_to_post_process_module() { - let provider = SlackProvider::new(); - let mut data = serde_json::json!({ - "channels": [{"id": "C1", "name": "eng", "is_private": false}] - }); - // Calling with an unknown slug should be a no-op. - provider.post_process_action_result("SLACK_UNKNOWN_ACTION", None, &mut data); - assert!( - data.get("channels").is_some(), - "no-op slug must not mutate data" - ); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/slack/types.rs b/crates/tinymemory-core/src/sync/composio/providers/slack/types.rs deleted file mode 100644 index 42714427..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/slack/types.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! Canonical types for the Composio-backed Slack provider. -//! -//! These types are independent of the Composio/Slack API payload shape. -//! They remain as compatibility wire types for product RPC responses and -//! backfill reporting; tinycortex owns runtime parsing and ingestion. -//! -//! The old `Bucket` struct (6-hour UTC window) has been removed — the -//! memory tree's L0 seal cascade handles batching after PR #1348, so -//! tinycortex owns batching and incremental persistence. - -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; - -/// A single message fetched from Slack's `conversations.history` or -/// `search.messages`. -/// -/// The Slack API represents `ts` as a decimal string like -/// `"1714003200.123456"` where the integer part is Unix seconds and the -/// fractional part is a per-workspace message sequence. We retain the -/// original string in `ts_raw` so it can round-trip back to the API -/// (e.g. as the `oldest` cursor on the next poll, and as the permalink -/// suffix for provenance). -/// -/// `channel_name`, `is_private`, `author_id`, and `permalink` are added -/// vs the old `memory::slack_ingestion::types::SlackMessage` because we no -/// longer carry a separate `SlackChannel` through the ingest path — -/// per-message context is self-contained. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct SlackMessage { - /// Channel ID this message belongs to (e.g. `"C0123456"`). - pub channel_id: String, - /// Human-readable channel name (e.g. `"eng"`). Injected by the enricher - /// from the channel directory; may be empty for search results whose - /// channel was not listed. - pub channel_name: String, - /// `true` if this is a private channel the bot has been invited to. - pub is_private: bool, - /// Resolved display name of the author. Falls back to the raw user id - /// when the user directory doesn't have an entry for this id. - pub author: String, - /// Raw Slack user id (e.g. `"U01234"`). Retained alongside the resolved - /// `author` so downstream code can still look up or log the stable id. - pub author_id: String, - /// Message body (plain text; may contain Slack-flavoured markdown). - pub text: String, - /// Canonical timestamp derived from `ts_raw`. - pub timestamp: DateTime, - /// Raw Slack `ts` string (used for API cursors + archive URLs). - pub ts_raw: String, - /// Root thread `ts` if this message is a reply; `None` for top-level - /// messages. Retained for future thread-aware ingestion. - pub thread_ts: Option, - /// Resolved HTTPS permalink, if Composio includes it in the response. - /// Falls back to the `slack://archives/…` scheme in ingest. - pub permalink: Option, -} - -/// A Slack channel visible to the bot, as returned by `conversations.list`. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct SlackChannel { - /// Channel ID (stable across renames). - pub id: String, - /// Human-readable name (e.g. `"eng"` → rendered as `"#eng"` in headers). - /// May change if admins rename the channel. - pub name: String, - /// `true` if this is a private channel the bot has been invited to. - pub is_private: bool, -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/sync_state.rs b/crates/tinymemory-core/src/sync/composio/providers/sync_state.rs deleted file mode 100644 index 22751850..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/sync_state.rs +++ /dev/null @@ -1,326 +0,0 @@ -//! Cursor, dedup and daily-budget state for Composio sync (#18 §B2). -//! -//! Owned here, engine-neutral, persisted through the [`SyncStateStore`] KV -//! seam — any provider whose KV family can get/set a JSON value can carry -//! sync state. This was a re-export of the engine's copy; §B2 asks for the -//! state to be engine-neutral, and the type is nothing but serde shapes over -//! std/chrono, so owning it costs one copy. -//! -//! The engine keeps its own copy for its internal pipelines until §B1's -//! orchestrator move retires them. The two persist under the same KV -//! namespace with the same serde shape; `the_state_namespace_is_pinned` and -//! `state_line_format_is_pinned` below hold this copy to that contract. - -use std::collections::{HashMap, HashSet}; - -use async_trait::async_trait; -use chrono::Utc; -use serde::{Deserialize, Serialize}; - -/// The KV namespace every persisted sync cursor lives under. -/// -/// Durable: changing it strands every cursor. See the pin test. -pub const KV_NAMESPACE: &str = STATE_NAMESPACE; - -pub const DEFAULT_DAILY_REQUEST_LIMIT: u32 = 500; -pub const STATE_NAMESPACE: &str = "composio-sync-state"; - -#[async_trait] -pub trait SyncStateStore: Send + Sync { - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result>; - async fn set( - &self, - namespace: &str, - key: &str, - value: &serde_json::Value, - ) -> anyhow::Result<()>; -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct DailyBudget { - pub date: String, - pub requests_used: u32, - pub limit: u32, -} - -impl Default for DailyBudget { - fn default() -> Self { - Self { - date: today(), - requests_used: 0, - limit: DEFAULT_DAILY_REQUEST_LIMIT, - } - } -} - -impl DailyBudget { - pub fn remaining(&self) -> u32 { - if self.date != today() { - self.limit - } else { - self.limit.saturating_sub(self.requests_used) - } - } - - pub fn is_exhausted(&self) -> bool { - self.remaining() == 0 - } - - pub fn record_requests(&mut self, count: u32) { - let today = today(); - if self.date != today { - self.date = today; - self.requests_used = 0; - } - self.requests_used = self.requests_used.saturating_add(count); - } - - pub fn record_request(&mut self) { - self.record_requests(1); - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SyncState { - pub toolkit: String, - pub connection_id: String, - #[serde(default)] - pub cursor: Option, - #[serde(default)] - pub synced_ids: HashSet, - #[serde(default)] - pub item_versions: HashMap, - #[serde(default)] - pub daily_budget: DailyBudget, - #[serde(default)] - pub last_seen_id: Option, - #[serde(default)] - pub last_sync_at_ms: Option, - #[serde(skip)] - pub run_requests: u32, - #[serde(skip)] - pub run_provider_cost_usd: f64, -} - -impl SyncState { - pub fn new(toolkit: impl Into, connection_id: impl Into) -> Self { - Self { - toolkit: toolkit.into(), - connection_id: connection_id.into(), - cursor: None, - synced_ids: HashSet::new(), - item_versions: HashMap::new(), - daily_budget: DailyBudget::default(), - last_seen_id: None, - last_sync_at_ms: None, - run_requests: 0, - run_provider_cost_usd: 0.0, - } - } - - pub fn key(toolkit: &str, connection_id: &str) -> String { - format!("{toolkit}:{connection_id}") - } - - pub fn is_synced(&self, id: &str) -> bool { - self.synced_ids.contains(id) - } - - pub fn mark_synced(&mut self, id: impl Into) { - self.synced_ids.insert(id.into()); - } - - pub fn advance_cursor(&mut self, cursor: impl Into) { - self.cursor = Some(cursor.into()); - } - - pub fn set_last_seen_id(&mut self, id: impl Into) { - self.last_seen_id = Some(id.into()); - } - - pub fn set_last_sync_at_ms(&mut self, timestamp_ms: u64) { - self.last_sync_at_ms = Some(timestamp_ms); - } - - pub fn budget_exhausted(&self) -> bool { - self.daily_budget.is_exhausted() - } - - pub fn budget_remaining(&self) -> u32 { - self.daily_budget.remaining() - } - - pub fn record_requests(&mut self, count: u32) { - self.daily_budget.record_requests(count); - self.run_requests = self.run_requests.saturating_add(count); - } - - pub fn record_action(&mut self, attempts: u32, cost_usd: f64) { - self.record_requests(attempts.max(1)); - if cost_usd.is_finite() && cost_usd > 0.0 { - self.run_provider_cost_usd += cost_usd; - } - } - - pub async fn load( - store: &dyn SyncStateStore, - toolkit: &str, - connection_id: &str, - ) -> anyhow::Result { - let key = Self::key(toolkit, connection_id); - match store.get(STATE_NAMESPACE, &key).await? { - Some(value) => { - let mut state: Self = serde_json::from_value(value)?; - if state.daily_budget.date != today() { - state.daily_budget.date = today(); - state.daily_budget.requests_used = 0; - } - Ok(state) - } - None => Ok(Self::new(toolkit, connection_id)), - } - } - - pub async fn save(&self, store: &dyn SyncStateStore) -> anyhow::Result<()> { - let value = serde_json::to_value(self)?; - store - .set( - STATE_NAMESPACE, - &Self::key(&self.toolkit, &self.connection_id), - &value, - ) - .await - } -} - -/// First non-empty string at any of `paths` (dot-separated) in `item`. -/// -/// Removed in the §B1a move as dead within this workspace; restored because -/// OpenHuman's raw-coverage integration tests import and exercise it through -/// the pin — "dead here" was measured with too small a grep. -pub fn extract_item_id(item: &serde_json::Value, paths: &[&str]) -> Option { - paths.iter().find_map(|path| { - let value = path - .split('.') - .try_fold(item, |current, segment| current.get(segment))?; - value - .as_str() - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) - }) -} - -fn today() -> String { - Utc::now().format("%Y-%m-%d").to_string() -} - -#[cfg(test)] -mod tests { - use std::collections::HashMap; - use std::sync::Mutex; - - use super::*; - - #[derive(Default)] - struct MemoryStateStore(Mutex>); - - #[async_trait] - impl SyncStateStore for MemoryStateStore { - async fn get( - &self, - namespace: &str, - key: &str, - ) -> anyhow::Result> { - Ok(self - .0 - .lock() - .unwrap() - .get(&format!("{namespace}:{key}")) - .cloned()) - } - - async fn set( - &self, - namespace: &str, - key: &str, - value: &serde_json::Value, - ) -> anyhow::Result<()> { - self.0 - .lock() - .unwrap() - .insert(format!("{namespace}:{key}"), value.clone()); - Ok(()) - } - } - - #[tokio::test] - async fn state_round_trips_cursor_dedup_and_budget() { - let store = MemoryStateStore::default(); - let mut state = SyncState::new("gmail", "conn-1"); - state.advance_cursor("cursor-2"); - state.mark_synced("message-1"); - state.record_requests(3); - state.save(&store).await.unwrap(); - - let loaded = SyncState::load(&store, "gmail", "conn-1").await.unwrap(); - assert_eq!(loaded.cursor.as_deref(), Some("cursor-2")); - assert!(loaded.is_synced("message-1")); - assert_eq!(loaded.daily_budget.requests_used, 3); - } - - /// The namespace is durable: every persisted Composio sync cursor lives - /// under this string, so a change strands all of them. The engine's copy - /// must agree; failing here means a coordinated migration, never a local - /// edit. - #[test] - fn the_state_namespace_is_pinned() { - assert_eq!( - KV_NAMESPACE, "composio-sync-state", - "the Composio sync-state KV namespace changed; every persisted \ - cursor is stored under the old value and needs migrating" - ); - assert_eq!(STATE_NAMESPACE, KV_NAMESPACE); - } - - /// The engine persists the same state with its own copy of this type. - /// Pins the serialised shape so the copies cannot drift silently. - #[test] - fn state_line_format_is_pinned() { - let mut state = SyncState::new("gmail", "conn-1"); - state.daily_budget.date = "2026-01-02".into(); - state.daily_budget.requests_used = 3; - state.advance_cursor("c2"); - state.mark_synced("m1"); - state.item_versions.insert("m1".into(), "v1".into()); - state.set_last_seen_id("m1"); - state.set_last_sync_at_ms(1_000); - let value = serde_json::to_value(&state).unwrap(); - assert_eq!( - value, - serde_json::json!({ - "toolkit": "gmail", - "connection_id": "conn-1", - "cursor": "c2", - "synced_ids": ["m1"], - "item_versions": {"m1": "v1"}, - "daily_budget": {"date": "2026-01-02", "requests_used": 3, "limit": 500}, - "last_seen_id": "m1", - "last_sync_at_ms": 1000 - }) - ); - } - - #[test] - fn stale_budget_reports_full_and_resets_on_record() { - let mut budget = DailyBudget { - date: "2000-01-01".into(), - requests_used: 499, - limit: 500, - }; - assert_eq!(budget.remaining(), 500); - budget.record_requests(1); - assert_eq!(budget.requests_used, 1); - assert_eq!(budget.remaining(), 499); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/tool_scope.rs b/crates/tinymemory-core/src/sync/composio/providers/tool_scope.rs deleted file mode 100644 index 9923e370..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/tool_scope.rs +++ /dev/null @@ -1,200 +0,0 @@ -//! Per-action scope classification (read / write / admin) plus the -//! [`CuratedTool`] catalog type that providers use to whitelist the -//! actions they want surfaced to the agent. -//! -//! Composio publishes 60+ actions per toolkit; most are noise for the -//! agent's planning loop. Each provider exports a hand-curated -//! [`CuratedTool`] slice via [`super::ComposioProvider::curated_tools`] -//! that pares the surface down to a useful subset and tags every action -//! with a [`ToolScope`] so per-user scope preferences can gate execution. - -use serde::{Deserialize, Serialize}; - -/// Classification of how invasive an action is. -/// -/// Used both to filter the agent's visible tool list and to enforce -/// per-user scope preferences at execution time. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum ToolScope { - /// Pure reads — `GET` / `FETCH` / `LIST` / `SEARCH` / `GET_PROFILE`. - Read, - /// Side-effectful actions that create or mutate user data — - /// `SEND` / `CREATE` / `UPDATE` / `REPLY` / `APPEND`. - Write, - /// Destructive or permission-changing actions — `DELETE` / `TRASH` / - /// `REMOVE` / `MODIFY_LABELS` / `SHARE`. - Admin, -} - -impl ToolScope { - pub fn as_str(self) -> &'static str { - match self { - ToolScope::Read => "read", - ToolScope::Write => "write", - ToolScope::Admin => "admin", - } - } -} - -/// One curated entry in a provider's tool catalog. -/// -/// `slug` is the Composio action slug as returned by `composio_list_tools` -/// (e.g. `"GMAIL_SEND_EMAIL"`). `scope` controls whether the action is -/// gated by the user's read / write / admin preference. -#[derive(Debug, Clone, Copy)] -pub struct CuratedTool { - pub slug: &'static str, - pub scope: ToolScope, -} - -/// Heuristic fallback when we need to gate a tool that isn't in any -/// provider's curated list. Prefer the curated classification when -/// available; only call this when [`super::ComposioProvider::curated_tools`] -/// returned `None` or didn't include the slug. -pub fn classify_unknown(slug: &str) -> ToolScope { - let upper = slug.to_ascii_uppercase(); - // Admin verbs are checked first so e.g. `MODIFY_LABELS` doesn't slip - // into the Write bucket on the `UPDATE`-substring rule. - const ADMIN: &[&str] = &[ - "DELETE", - "TRASH", - "REMOVE", - "MODIFY_LABELS", - "SHARE", - "REVOKE", - "DESTROY", - ]; - const WRITE: &[&str] = &[ - "SEND", "CREATE", "UPDATE", "REPLY", "APPEND", "INSERT", "ADD", "POST", "PATCH", "WRITE", - "DRAFT", - ]; - if ADMIN.iter().any(|kw| upper.contains(kw)) { - return ToolScope::Admin; - } - if WRITE.iter().any(|kw| upper.contains(kw)) { - return ToolScope::Write; - } - ToolScope::Read -} - -/// Look up a slug inside a curated catalog. -pub fn find_curated<'a>(catalog: &'a [CuratedTool], slug: &str) -> Option<&'a CuratedTool> { - catalog.iter().find(|t| t.slug.eq_ignore_ascii_case(slug)) -} - -/// Extract the toolkit slug from a Composio action slug. -/// -/// Most Composio action slugs follow `__…` -/// (e.g. `GMAIL_SEND_EMAIL` → `gmail`). A few toolkit identifiers contain -/// underscores themselves; those need known-prefix handling so connected -/// toolkit checks do not drop actions such as `ZOHO_MAIL_*`. -pub fn toolkit_from_slug(slug: &str) -> Option { - let trimmed = slug.trim(); - if trimmed.is_empty() { - return None; - } - const MULTI_SEGMENT_TOOLKIT_PREFIXES: &[(&str, &str)] = &[ - ("MICROSOFT_TEAMS_", "microsoft_teams"), - ("ONE_DRIVE_", "one_drive"), - ("ZOHO_MAIL_", "zoho_mail"), - ]; - let upper = trimmed.to_ascii_uppercase(); - for (prefix, toolkit) in MULTI_SEGMENT_TOOLKIT_PREFIXES { - if upper.starts_with(prefix) { - return Some((*toolkit).to_string()); - } - } - let prefix = trimmed.split('_').next()?; - if prefix.is_empty() { - None - } else { - Some(prefix.to_ascii_lowercase()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn classify_unknown_picks_admin_for_destructive_verbs() { - assert_eq!(classify_unknown("GMAIL_DELETE_EMAIL"), ToolScope::Admin); - assert_eq!(classify_unknown("GMAIL_TRASH_EMAIL"), ToolScope::Admin); - assert_eq!(classify_unknown("GMAIL_MODIFY_LABELS"), ToolScope::Admin); - } - - #[test] - fn classify_unknown_picks_write_for_mutating_verbs() { - assert_eq!(classify_unknown("GMAIL_SEND_EMAIL"), ToolScope::Write); - assert_eq!(classify_unknown("NOTION_CREATE_PAGE"), ToolScope::Write); - assert_eq!(classify_unknown("NOTION_UPDATE_PAGE"), ToolScope::Write); - } - - #[test] - fn classify_unknown_defaults_to_read() { - assert_eq!(classify_unknown("GMAIL_FETCH_EMAILS"), ToolScope::Read); - assert_eq!(classify_unknown("NOTION_SEARCH"), ToolScope::Read); - assert_eq!(classify_unknown("GMAIL_GET_PROFILE"), ToolScope::Read); - } - - #[test] - fn classify_unknown_admin_takes_precedence_over_write() { - // MODIFY_LABELS contains no write verb but DELETE_DRAFT does — make - // sure the admin check wins. - assert_eq!(classify_unknown("GMAIL_DELETE_DRAFT"), ToolScope::Admin); - } - - #[test] - fn toolkit_from_slug_extracts_lowercase_prefix() { - assert_eq!( - toolkit_from_slug("GMAIL_SEND_EMAIL"), - Some("gmail".to_string()) - ); - assert_eq!( - toolkit_from_slug("NOTION_FETCH_DATA"), - Some("notion".to_string()) - ); - assert_eq!(toolkit_from_slug(""), None); - assert_eq!( - toolkit_from_slug("noUnderscore"), - Some("nounderscore".into()) - ); - } - - #[test] - fn toolkit_from_slug_handles_known_multi_segment_toolkits() { - assert_eq!( - toolkit_from_slug("ZOHO_MAIL_SEND_EMAIL"), - Some("zoho_mail".to_string()) - ); - assert_eq!( - toolkit_from_slug("ONE_DRIVE_GET_FILE"), - Some("one_drive".to_string()) - ); - assert_eq!( - toolkit_from_slug("MICROSOFT_TEAMS_SEND_MESSAGE"), - Some("microsoft_teams".to_string()) - ); - } - - #[test] - fn find_curated_is_case_insensitive() { - let catalog = &[CuratedTool { - slug: "GMAIL_SEND_EMAIL", - scope: ToolScope::Write, - }]; - assert!(find_curated(catalog, "gmail_send_email").is_some()); - assert!(find_curated(catalog, "GMAIL_SEND_EMAIL").is_some()); - assert!(find_curated(catalog, "GMAIL_DELETE_EMAIL").is_none()); - } - - #[test] - fn tool_scope_serializes_lowercase() { - assert_eq!(serde_json::to_string(&ToolScope::Read).unwrap(), "\"read\""); - assert_eq!( - serde_json::to_string(&ToolScope::Admin).unwrap(), - "\"admin\"" - ); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/traits.rs b/crates/tinymemory-core/src/sync/composio/providers/traits.rs deleted file mode 100644 index 6f8b8543..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/traits.rs +++ /dev/null @@ -1,405 +0,0 @@ -//! The core provider trait for Composio toolkit implementations. - -use async_trait::async_trait; - -use super::tool_scope::CuratedTool; -use super::types::{ - NormalizedTask, ProviderContext, ProviderUserProfile, SyncOutcome, SyncReason, TaskContainer, - TaskFetchFilter, -}; - -/// Native provider implementation for a specific Composio toolkit. -/// -/// All methods are async and return `Result<_, String>` so the bus -/// subscriber + RPC layer can forward errors as user-visible strings -/// without `anyhow` round-tripping. -#[async_trait] -pub trait ComposioProvider: Send + Sync { - /// Toolkit slug (e.g. `"gmail"`). Must match the slug Composio / - /// the backend allowlist uses — the registry keys on this. - fn toolkit_slug(&self) -> &'static str; - - /// Suggested periodic sync interval in seconds. Return `None` to - /// opt out of the periodic scheduler entirely (e.g. for write-only - /// providers like Slack send-message). - fn sync_interval_secs(&self) -> Option { - Some(15 * 60) - } - - /// Curated whitelist of Composio actions this provider considers - /// useful for the agent, classified by [`super::tool_scope::ToolScope`]. - /// - /// When `Some(&[...])`, the meta-tool layer hides every action not - /// in this list from `composio_list_tools` and rejects execution of - /// any slug not in this list (or whose scope is disabled in the - /// user's pref). - /// - /// Default: `None` — toolkits without a curated catalog (e.g. - /// integrations not yet hand-tuned) pass through all actions and - /// rely on the [`super::tool_scope::classify_unknown`] heuristic for - /// scope gating. - fn curated_tools(&self) -> Option<&'static [CuratedTool]> { - None - } - - /// Fetch a normalized user profile for the current connection in - /// `ctx`. Most providers implement this by calling a provider - /// "get profile / about me" action via `super::super::ops::composio_execute`. - async fn fetch_user_profile( - &self, - ctx: &ProviderContext, - ) -> Result; - - /// Compatibility entry point backed exclusively by tinycortex. - async fn sync(&self, ctx: &ProviderContext, reason: SyncReason) -> Result { - let connection_id = ctx.connection_id.as_deref().ok_or_else(|| { - format!( - "[composio:{}] sync missing connection_id", - self.toolkit_slug() - ) - })?; - let started_at_ms = now_ms(); - let outcome = crate::sync::pipelines::host::run_composio_connection( - self.toolkit_slug(), - connection_id, - ctx.config.as_ref(), - ctx.max_items, - ctx.sync_depth_days, - ) - .await - .map_err(|error| error.to_string())?; - Ok(SyncOutcome { - toolkit: self.toolkit_slug().into(), - connection_id: Some(connection_id.into()), - reason: reason.as_str().into(), - items_ingested: outcome.records_ingested as usize, - started_at_ms, - finished_at_ms: now_ms(), - summary: outcome.note.unwrap_or_else(|| "sync completed".into()), - details: serde_json::json!({ - "more_pending": outcome.more_pending, - "actions_called": outcome.actions_called, - "provider_cost_usd": outcome.provider_cost_usd, - }), - }) - } - - /// Fetch a filtered set of work items as structured - /// [`NormalizedTask`]s — the read path that powers the - /// `task_sources` domain. - /// - /// Unlike [`Self::sync`], this does **not** persist anything into - /// the memory store; it *returns* normalized tasks so the caller can - /// enrich them and route them onto the agent's todo board. `filter` - /// is provider-agnostic — implementations read only the fields that - /// apply to their toolkit and translate them into their own action - /// slug + arguments, then map the upstream payload back into - /// `NormalizedTask`. Implementations must honour - /// [`TaskFetchFilter::effective_max`] as an upper bound on the - /// number of tasks returned. - /// - /// Default impl: `Err` — providers without a task surface (e.g. - /// gmail, slack) opt out, exactly as - /// [`Self::sync_interval_secs`] returning `None` opts out of the - /// periodic scheduler. - async fn fetch_tasks( - &self, - ctx: &ProviderContext, - filter: &TaskFetchFilter, - ) -> Result, String> { - let _ = (ctx, filter); - Err(format!( - "[composio:{}] provider has no task-fetch surface", - self.toolkit_slug() - )) - } - - /// List the selectable containers the connected account exposes — - /// today Notion databases — so the task-source UI can offer a picker - /// instead of a raw-id text field. - /// - /// Default impl: `Err` — providers without a container surface opt out, - /// mirroring [`Self::fetch_tasks`]. - async fn list_databases(&self, ctx: &ProviderContext) -> Result, String> { - let _ = ctx; - Err(format!( - "[composio:{}] provider has no database/container surface", - self.toolkit_slug() - )) - } - - /// Standardized identity callback for provider implementations. - /// - /// Providers can override this to customize how identity fragments - /// are persisted. Default behavior stores a normalized identity - /// fragment in profile facets via `skill:{source}:{identifier}:{field}` - /// keys and returns the number of facets written. - fn identity_set(&self, profile: &ProviderUserProfile) -> usize { - super::profile::persist_provider_profile(profile) - } - - /// Hook fired when an OAuth handoff completes - /// (the host's `DomainEvent::ComposioConnectionCreated`). - /// - /// Default impl: fetch and persist the user profile. Initial memory - /// ingestion is dispatched separately through tinycortex by the bus. - /// Providers can override to add provider-specific bootstrapping - /// (e.g. registering Composio triggers, seeding labels, …). - async fn on_connection_created(&self, ctx: &ProviderContext) -> Result<(), String> { - let toolkit = self.toolkit_slug(); - tracing::info!( - toolkit = %toolkit, - connection_id = ?ctx.connection_id, - "[composio:provider] on_connection_created: fetching user profile" - ); - match self.fetch_user_profile(ctx).await { - Ok(profile) => { - // PII discipline: do not log raw display_name or email. - // We log only presence indicators and the email domain - // (non-PII) so the trace is debuggable without leaking - // the user's identity. Provider-specific impls follow - // the same convention. - let has_display_name = profile.display_name.is_some(); - let has_email = profile.email.is_some(); - let email_domain = profile - .email - .as_deref() - .and_then(|e| e.split('@').nth(1)) - .map(|d| d.to_string()); - tracing::info!( - toolkit = %toolkit, - has_display_name, - has_email, - email_domain = ?email_domain, - "[composio:provider] user profile fetched" - ); - - // Persist profile fields into the local user_profile - // facet table so display_name / email / avatar are - // available to the agent context and UI without a - // round-trip to the upstream provider. - let facets = self.identity_set(&profile); - tracing::debug!( - toolkit = %toolkit, - facets_written = facets, - "[composio:provider] identity_set persisted profile facets" - ); - - // Mirror the same identity fragment into PROFILE.md so - // it lands in the agent's prompt context on the next - // turn (the facets table feeds queries; PROFILE.md - // feeds the system prompt). - if let Err(e) = super::profile_md::merge_provider_into_profile_md( - ctx.config.workspace_dir(), - &profile, - ) { - tracing::warn!( - toolkit = %toolkit, - error = %e, - "[composio:provider] PROFILE.md merge failed (non-fatal)" - ); - } - } - Err(e) => { - tracing::warn!( - toolkit = %toolkit, - error = %e, - "[composio:provider] user profile fetch failed (continuing to sync)" - ); - } - } - Ok(()) - } - - /// Hook fired immediately after a Composio action executed against - /// this toolkit returns a **successful** response. The provider may - /// mutate `data` in place to reshape the upstream payload before it - /// is handed back to the agent / RPC caller (e.g. convert Gmail's - /// HTML message bodies to markdown to save context tokens). - /// - /// `slug` is the full action slug (e.g. `"GMAIL_FETCH_EMAILS"`) so - /// providers can dispatch per action. `arguments` is the caller's - /// original argument object — providers can read opt-out flags from - /// it (e.g. `raw_html: true` to preserve raw HTML). - /// - /// Errors from upstream are not routed here; only `successful` - /// responses. Default impl is a no-op so providers that have nothing - /// to rewrite don't need to override. - fn post_process_action_result( - &self, - slug: &str, - arguments: Option<&serde_json::Value>, - data: &mut serde_json::Value, - ) { - let _ = (slug, arguments, data); - } - - /// Hook fired when a Composio trigger webhook arrives for this - /// toolkit. `payload` is the raw provider payload as forwarded by - /// the backend. Implementations should be defensive — payload - /// shapes vary across triggers. - /// - /// Default impl: log and no-op. Most providers will want to - /// override this to react to specific triggers. - async fn on_trigger( - &self, - ctx: &ProviderContext, - trigger: &str, - payload: &serde_json::Value, - ) -> Result<(), String> { - tracing::debug!( - toolkit = %self.toolkit_slug(), - trigger = %trigger, - connection_id = ?ctx.connection_id, - payload_bytes = payload.to_string().len(), - "[composio:provider] on_trigger (default no-op)" - ); - Ok(()) - } -} - -fn now_ms() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64 -} - -/// Build the env var name read by [`resolve_sync_interval_secs`] for a -/// given toolkit slug. Exposed so tests (and `.env.example`) can stay in -/// lockstep with the runtime lookup without re-implementing the casing. -pub fn sync_interval_env_var(toolkit: &str) -> String { - format!( - "OPENHUMAN_COMPOSIO_{}_SYNC_INTERVAL_SECS", - toolkit.to_ascii_uppercase() - ) -} - -/// Resolve the effective periodic sync interval (seconds) for a provider. -/// Reads `OPENHUMAN_COMPOSIO__SYNC_INTERVAL_SECS` if set; -/// otherwise returns `default_secs`. A non-positive or unparseable value -/// is rejected with a `warn` and the default is used — `0` would burn the -/// scheduler in a tight loop, so it is never honoured. -/// -/// Each provider's `sync_interval_secs()` impl calls this with its own -/// compile-time default so operators can independently slow down a -/// chatty toolkit (e.g. Slack) without rebuilding. -pub fn resolve_sync_interval_secs(toolkit: &str, default_secs: u64) -> u64 { - let key = sync_interval_env_var(toolkit); - match std::env::var(&key) { - Ok(s) => match s.trim().parse::() { - Ok(n) if n >= 1 => n, - _ => { - static WARNED: std::sync::Once = std::sync::Once::new(); - WARNED.call_once(|| { - tracing::warn!( - env = %key, - value = %s, - default = default_secs, - "[composio:provider] sync-interval env override not a positive u64; using default" - ); - }); - default_secs - } - }, - Err(_) => default_secs, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn sync_interval_env_var_uppercases_slug() { - assert_eq!( - sync_interval_env_var("slack"), - "OPENHUMAN_COMPOSIO_SLACK_SYNC_INTERVAL_SECS" - ); - assert_eq!( - sync_interval_env_var("GitHub"), - "OPENHUMAN_COMPOSIO_GITHUB_SYNC_INTERVAL_SECS" - ); - } - - /// RAII guard for env var save/restore so the test does not leak - /// state to siblings within the same process. - struct EnvGuard { - key: String, - previous: Option, - } - - impl EnvGuard { - fn set(key: &str, value: &str) -> Self { - let previous = std::env::var(key).ok(); - std::env::set_var(key, value); - Self { - key: key.to_string(), - previous, - } - } - fn unset(key: &str) -> Self { - let previous = std::env::var(key).ok(); - std::env::remove_var(key); - Self { - key: key.to_string(), - previous, - } - } - } - - impl Drop for EnvGuard { - fn drop(&mut self) { - match self.previous.take() { - Some(v) => std::env::set_var(&self.key, v), - None => std::env::remove_var(&self.key), - } - } - } - - // Bundled into a single `#[test]` so cargo's per-test parallelism - // does not race on the shared env var. Each scenario explicitly - // drops its guard before the next so the env is in a known state. - #[test] - fn resolve_sync_interval_honors_per_toolkit_env() { - let _lock = crate::test_env_lock::TEST_ENV_LOCK - .lock() - .unwrap_or_else(std::sync::PoisonError::into_inner); - - let key = sync_interval_env_var("slack"); - let default = 15 * 60; - - // Unset → default. - let _g = EnvGuard::unset(&key); - assert_eq!(resolve_sync_interval_secs("slack", default), default); - drop(_g); - - // Valid override slows the cadence. - let _g = EnvGuard::set(&key, "3600"); - assert_eq!(resolve_sync_interval_secs("slack", default), 3600); - drop(_g); - - // Whitespace tolerated. - let _g = EnvGuard::set(&key, " 1800 "); - assert_eq!(resolve_sync_interval_secs("slack", default), 1800); - drop(_g); - - // Zero rejected (would spin the scheduler). - let _g = EnvGuard::set(&key, "0"); - assert_eq!(resolve_sync_interval_secs("slack", default), default); - drop(_g); - - // Garbage rejected. - let _g = EnvGuard::set(&key, "soon"); - assert_eq!(resolve_sync_interval_secs("slack", default), default); - drop(_g); - - // Per-toolkit scoping: a different toolkit's var does not bleed - // into slack's lookup. - let gmail_key = sync_interval_env_var("gmail"); - let _slack_unset = EnvGuard::unset(&key); - let _gmail_set = EnvGuard::set(&gmail_key, "120"); - assert_eq!(resolve_sync_interval_secs("slack", default), default); - assert_eq!(resolve_sync_interval_secs("gmail", default), 120); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/types.rs b/crates/tinymemory-core/src/sync/composio/providers/types.rs deleted file mode 100644 index 3825c4a5..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/types.rs +++ /dev/null @@ -1,523 +0,0 @@ -//! Shared types for Composio provider implementations. - -use serde::{Deserialize, Serialize}; -use std::sync::{Arc, Mutex}; - -// Test-only: the tests below build a `TestHostConfig` and call -// `MemoryHostConfig` methods on it directly. Production code in this module -// goes through `crate::Config`, so neither name is needed in a non-test build. -#[cfg(test)] -use tinymemory_api::host::{test_support::TestHostConfig, MemoryHostConfig}; - -use crate::composio_host::{self, ComposioExecuteResponse}; -use crate::config_loader as config_rpc; -use crate::Config; - -/// Reason a sync was triggered. Providers can use this to decide -/// whether to do a full backfill or an incremental pull. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SyncReason { - /// First sync immediately after an OAuth handoff completes. - ConnectionCreated, - /// Periodic background sync from the scheduler. - Periodic, - /// Explicit user-driven sync from RPC / UI. - Manual, -} - -impl SyncReason { - pub fn as_str(&self) -> &'static str { - match self { - SyncReason::ConnectionCreated => "connection_created", - SyncReason::Periodic => "periodic", - SyncReason::Manual => "manual", - } - } -} - -/// What kind of work an ingested task implies. GitHub's issues-and-PRs -/// search returns both shapes, and the job differs fundamentally — -/// *resolve* an issue vs *review* a pull request — so providers tag each -/// task and the `task_sources` enrichment phrases the objective / agent -/// prompt accordingly (the triage LLM then knows what to do). Providers -/// that don't distinguish (notion, linear, clickup) leave this `Generic`. -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum TaskKind { - /// No issue/PR distinction — the default for non-code providers. - #[default] - Generic, - /// A tracker issue: the job is to resolve / implement it. - Issue, - /// A pull request: the job is to review it (read the diff, give feedback). - PullRequest, -} - -impl TaskKind { - /// Stable lowercase tag, mirrored into the card's `source_metadata`. - pub fn as_str(&self) -> &'static str { - match self { - TaskKind::Generic => "generic", - TaskKind::Issue => "issue", - TaskKind::PullRequest => "pull_request", - } - } -} - -/// Normalized user profile shape returned by every provider. -/// -/// The shared fields (`display_name`, `email`, `username`, `avatar_url`, -/// `profile_url`) -/// cover what the desktop UI actually needs to render a connected -/// account card. Anything provider-specific (Gmail's `messagesTotal`, -/// Notion's workspace ids, …) goes into [`extras`](Self::extras) so -/// callers don't have to widen the shape every time a new toolkit -/// lands. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ProviderUserProfile { - pub toolkit: String, - pub connection_id: Option, - pub display_name: Option, - pub email: Option, - pub username: Option, - pub avatar_url: Option, - pub profile_url: Option, - /// Provider-specific extras (raw JSON object). - #[serde(default)] - pub extras: serde_json::Value, -} - -/// Result of a provider sync run. Mostly used for logging + UI status. -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct SyncOutcome { - pub toolkit: String, - pub connection_id: Option, - pub reason: String, - pub items_ingested: usize, - pub started_at_ms: u64, - pub finished_at_ms: u64, - pub summary: String, - /// Provider-specific extras (raw JSON object). - #[serde(default)] - pub details: serde_json::Value, -} - -impl SyncOutcome { - pub fn elapsed_ms(&self) -> u64 { - self.finished_at_ms.saturating_sub(self.started_at_ms) - } -} - -/// A provider-agnostic, structured work item produced by -/// [`super::ComposioProvider::fetch_tasks`]. -/// -/// Unlike the `sync()` path — which persists upstream items into the -/// memory store as passive context — `fetch_tasks` *returns* normalized -/// tasks so the `task_sources` domain can enrich them and route them -/// onto the agent's todo board. Every native task provider (github, -/// notion, linear, clickup) maps its upstream payload shape into this -/// common envelope. -/// -/// `source_id` is left empty by providers and stamped by the -/// `task_sources` pipeline with the originating `TaskSource.id` — a -/// provider has no knowledge of which configured source asked for the -/// fetch. -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct NormalizedTask { - /// Upstream provider's stable id for the item (issue/task/page id). - pub external_id: String, - /// The `TaskSource.id` that produced this task. Empty until the - /// pipeline stamps it. - #[serde(default)] - pub source_id: String, - /// Toolkit slug, e.g. `"github"`. - pub provider: String, - /// Whether this task is an issue, a pull request, or undifferentiated. - /// Drives intent-aware objective / prompt phrasing in enrichment. - #[serde(default)] - pub kind: TaskKind, - pub title: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub body: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub url: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub status: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub assignee: Option, - /// Due date as an ISO-8601 string, when the provider exposes one. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub due: Option, - #[serde(default)] - pub labels: Vec, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub priority: Option, - /// Last-updated ISO-8601 timestamp — used for cursor advancement and - /// edit-aware dedup (`{external_id}@{updated_at}`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub updated_at: Option, - /// The raw upstream payload, retained for enrichment / debugging. - #[serde(default)] - pub raw: serde_json::Value, -} - -/// A selectable upstream task container (board / database / list) used to -/// populate a picker so the user chooses from a list instead of pasting a -/// raw id. Today this is a Notion database, later a Linear team or ClickUp -/// list. Surfaced to the task-source UI as `{ id, title }`. -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct TaskContainer { - /// Provider-native id (e.g. a Notion database id) used as the filter id. - pub id: String, - /// Human-readable label for the picker. - pub title: String, -} - -/// Provider-agnostic filter passed into -/// [`super::ComposioProvider::fetch_tasks`]. -/// -/// The `task_sources` domain builds this from a user-configured, -/// per-provider `FilterSpec`. Each provider reads only the fields that -/// apply to it (github reads `repo`/`labels`; notion reads -/// `database_id`; linear/clickup read `team_id`; …) and ignores the -/// rest. `extra` is a free-form escape hatch surfaced in the UI for -/// advanced provider-native query fragments. -/// How the GitHub task-source fetch reaches GitHub. Shipped desktop users -/// connect GitHub via Composio OAuth (no `gh` on PATH, no `GITHUB_TOKEN`), -/// while local dev / self-host setups often have the reverse. `Auto` does the -/// right thing for both; `Composio` / `Local` force a path when the user wants. -#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum GithubFetchMode { - /// Try the connected Composio account first; fall back to local `gh`/REST - /// only when Composio is unavailable. The safe default — no regression for - /// shipped users, still a true fallback for local/dev. - #[default] - Auto, - /// Force the connected Composio account (classic shipped-app behaviour). - Composio, - /// Force local `gh` CLI / REST with a `GH_TOKEN`/`GITHUB_TOKEN` env token. - Local, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct TaskFetchFilter { - /// Scope to items assigned to (or involving) the authenticated user. - #[serde(default)] - pub assignee_is_me: bool, - /// GitHub fetch path selector (Composio vs local `gh`/REST). Default `Auto`. - #[serde(default)] - pub github_fetch_mode: GithubFetchMode, - /// GitHub `owner/name` repository scope. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub repo: Option, - /// GitHub label filter. - #[serde(default)] - pub labels: Vec, - /// Issue/task state filter (e.g. `"open"`, `"todo"`). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub state: Option, - /// Notion database (board) id. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub database_id: Option, - /// Notion status property filter. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub status: Option, - /// Linear / ClickUp team (workspace) id. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub team_id: Option, - /// ClickUp list id. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub list_id: Option, - /// Free-form provider-native filter fragment (advanced). - #[serde(default)] - pub extra: serde_json::Value, - /// Hard cap on how many tasks a single fetch returns. - #[serde(default)] - pub max: u32, -} - -impl TaskFetchFilter { - /// Effective per-fetch item cap, defaulting to a safe bound when the - /// caller leaves `max` unset (0). - pub fn effective_max(&self) -> usize { - if self.max == 0 { - 25 - } else { - self.max as usize - } - } -} - -/// Per-call context handed to provider methods. -/// -/// `connection_id` is `None` when a method runs in a "no specific -/// connection" mode (e.g. an across-the-board periodic sync that -/// already iterated). For per-connection paths it is always populated. -/// -/// **Mode-aware dispatch (#1710)**: pre-fix, `ProviderContext` cached a -/// pre-baked `ComposioClient` built once at construction time. Toggling -/// `composio.mode = "direct"` mid-session left provider syncs still -/// routing through the backend tinyhumans tenant. The current shape -/// keeps an [`Arc`] and resolves the underlying client per call -/// through [`ProviderContext::execute`], mirroring the agent-tool -/// migration in the host's `integrations::composio::tools::ComposioExecuteTool`. -/// Per-sync accumulator for Composio billable-action usage. -/// -/// Lives behind a shared handle on [`ProviderContext`] so the single -/// `execute` chokepoint can tally every action a provider fires during one -/// sync run, regardless of which provider (gmail / slack / github / notion / -/// linear / clickup) or how many pages it paginates. -/// [`crate::sync::composio::run_connection_sync`] returns -/// the final tally alongside the [`SyncOutcome`] for the sync audit log -/// (#3111). -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct ComposioUsage { - /// Count of `execute` calls that returned a response this run. - pub actions_called: u32, - /// Sum of each response's backend-reported `cost_usd`. - pub cost_usd: f64, -} - -/// Shared, interior-mutable handle to a [`ComposioUsage`] tally. Cloning a -/// [`ProviderContext`] shares the same underlying counter, so the count is -/// stable no matter how the context is passed around within a sync. -pub type ComposioUsageHandle = Arc>; - -#[derive(Clone)] -pub struct ProviderContext { - pub config: Arc, - pub toolkit: String, - pub connection_id: Option, - /// Accumulates Composio billable-action usage across this context's - /// lifetime. Defaulted at every construction site; only the sync path - /// (`run_connection_sync`) reads it back. Non-sync callers (agent tools, - /// task-source fetches) leave it at zero — harmless. - pub usage: ComposioUsageHandle, - /// Maximum items to fetch in a single sync pass. - /// - /// Set from the corresponding `MemorySourceEntry.max_items` field at - /// sync-dispatch time. `None` means no cap beyond the provider's own - /// internal upper bounds. - pub max_items: Option, - /// Maximum sync depth window in days. - /// - /// Set from `MemorySourceEntry.sync_depth_days`. When `Some(n)`, the - /// provider only fetches items from the last `n` days. `None` means - /// no additional depth restriction beyond the provider's cursor. - pub sync_depth_days: Option, -} - -impl ProviderContext { - /// Build a context from the current config + a toolkit slug. - /// - /// Returns `None` only when we want to short-circuit early on the - /// "user clearly not signed in" path. In the post-#1710 shape this - /// is determined by attempting a factory resolve via - /// [`composio_host::is_available`] and treating a `false` there as - /// "skip silently" — the same UX as the pre-fix - /// `build_composio_client(...).is_some()` probe, but routed - /// through the mode-aware factory so direct-mode users (no backend - /// session token, BYO key in keychain) aren't falsely treated as - /// signed-out. - pub fn from_config( - config: Arc, - toolkit: impl Into, - connection_id: Option, - ) -> Option { - // Probe the factory: any successful resolve (Backend OR Direct) - // means the user has *some* viable Composio client. Direct-mode - // users typically have no backend session token, which would - // make a `build_composio_client` probe return None and falsely - // skip them. - if composio_host::is_available(&*config) { - Some(Self { - config, - toolkit: toolkit.into(), - connection_id, - usage: ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - }) - } else { - tracing::debug!( - "[composio:provider_context] from_config: no viable Composio client; \ - treating as not-signed-in" - ); - None - } - } - - /// Resolve the underlying composio client via the mode-aware - /// factory and dispatch a single action. This is the canonical - /// way for provider implementations to execute a Composio action - /// — going through here ensures the live `composio.mode` toggle is - /// honoured on every call (#1710). - /// - /// Returns the same [`ComposioExecuteResponse`] shape that - /// `ComposioClient::execute_tool` used to return so existing - /// provider call-sites can swap `ctx.client.execute_tool(...)` for - /// `ctx.execute(...)` with no other changes. - pub async fn execute( - &self, - action: &str, - arguments: Option, - ) -> anyhow::Result { - // [#1710 Wave 4] Reload config fresh per execute so a mid-session - // `composio.mode` toggle takes effect at the very next call. The - // Arc snapshot held by `self` was taken at agent-init time - // and is otherwise stale relative to subsequent set_api_key / - // clear_api_key RPCs. - // - // Use `reload_config_snapshot_with_timeout` (anchored to the snapshot's - // `config_path`) rather than `load_config_with_timeout` (which - // re-resolves `OPENHUMAN_WORKSPACE` from the process env). The config - // path is stable for the lifetime of a `ProviderContext` — it is set - // at context creation from the agent's scoped config — so reading from - // it always reaches the correct user workspace and avoids a data-race - // in tests that share the process env. - let live_config = config_rpc::reload_config_snapshot_with_timeout(&*self.config) - .await - .map_err(|e| { - tracing::warn!( - action = %action, - toolkit = %self.toolkit, - error = %e, - "[composio:provider_context] execute: reload_config failed" - ); - anyhow::anyhow!("composio provider_context: failed to reload live config: {e}") - })?; - // Mode dispatch (backend tenant vs the user's own direct v3 tenant) - // lives in the host's `ComposioHost` impl — this side just asks. - let result = composio_host::execute( - &*live_config, - action, - arguments, - &live_config.composio().entity_id, - self.connection_id.as_deref(), - ) - .await - .map_err(|e| anyhow::anyhow!(e)); - - // Tally billable-action usage at the single chokepoint every provider - // routes through (#3111). We count any *completed* round-trip — even a - // provider-reported failure (`successful == false`) is a billable call - // — and sum the backend-reported `cost_usd`. Transport errors (the - // `Err` arm) never reached Composio, so they don't count. The lock is - // held only for the increment, never across an `.await`. - if let Ok(ref resp) = result { - if let Ok(mut usage) = self.usage.lock() { - usage.actions_called = usage.actions_called.saturating_add(1); - usage.cost_usd += resp.cost_usd; - } - } - result - } - - /// Memory client handle if the global memory singleton is ready. - /// Used by providers that want to persist sync snapshots. - /// - /// Under `cfg(test)` the global singleton is not booted, so build a - /// workspace-scoped client directly instead. - #[cfg(test)] - pub fn memory_client(&self) -> Option { - crate::store::MemoryClient::from_workspace_dir(self.config.workspace_dir().clone()) - .ok() - .map(std::sync::Arc::new) - } - - /// Memory client handle if the global memory singleton is ready. - /// Used by providers that want to persist sync snapshots. - #[cfg(not(test))] - pub fn memory_client(&self) -> Option { - crate::global::client_if_ready() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The whole #3111 tally relies on the `usage` handle being *shared* - /// across `ProviderContext` clones: a provider's `sync` runs against a - /// clone (or the same ctx passed by `&`), accumulates via `execute`, and - /// `run_connection_sync` reads the count back from its own handle. Pin - /// that the `Arc>` is genuinely shared so a clone's increments - /// are visible from the original — if this regressed to a per-clone - /// counter, the audit cost would silently always read zero. - #[test] - fn usage_handle_is_shared_across_context_clones() { - let ctx = ProviderContext { - config: Arc::new(TestHostConfig::default()) as Arc, - toolkit: "gmail".to_string(), - connection_id: None, - usage: ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - }; - let cloned = ctx.clone(); - - // Simulate two `execute` round-trips accumulating on the clone. - { - let mut usage = cloned.usage.lock().expect("lock usage"); - usage.actions_called = usage.actions_called.saturating_add(2); - usage.cost_usd += 0.015; - } - - // The original handle must observe the clone's tally. - let observed = ctx.usage.lock().expect("lock usage"); - assert_eq!(observed.actions_called, 2); - assert!((observed.cost_usd - 0.015).abs() < 1e-9); - } - - /// `ComposioUsage` defaults to a zero tally — the value - /// `run_connection_sync` returns for a sync that fired no Composio - /// actions, and what non-sync `ProviderContext` callers carry. - #[test] - fn composio_usage_defaults_to_zero() { - let usage = ComposioUsage::default(); - assert_eq!(usage.actions_called, 0); - assert_eq!(usage.cost_usd, 0.0); - } - - // `ProviderContext::execute` and `ProviderContext::backend_client` reload - // config from `ctx.config.config_path()` (via `reload_config_snapshot_with_timeout`) - // rather than from the process-global `OPENHUMAN_WORKSPACE`. Tests - // therefore only need to persist the config to `config_path` — no env var - // manipulation required. - - #[tokio::test] - async fn provider_context_execute_backend_branch_without_session_errors_cleanly() { - // Default `Config` (mode = "backend") with no stored session - // token: the factory should return a backend-session error from - // `ctx.execute`. Verifies the backend branch is reachable and - // the error surface is sensible. - let tmp = tempfile::tempdir().expect("tempdir"); - - let mut config = TestHostConfig::default(); - config.config_path = tmp.path().join("config.toml"); - config.workspace_dir = tmp.path().join("workspace"); - config.secrets_encrypt = false; - config.save().await.expect("save fake config to disk"); - - let ctx = ProviderContext { - config: Arc::new(config) as Arc, - toolkit: "gmail".to_string(), - connection_id: None, - usage: ComposioUsageHandle::default(), - max_items: None, - sync_depth_days: None, - }; - let res = ctx.execute("GMAIL_FETCH_EMAILS", None).await; - let err = res.expect_err("no backend session must error"); - let msg = err.to_string(); - assert!( - msg.contains("backend") || msg.contains("session"), - "expected backend-session error, got: {msg}" - ); - } -} diff --git a/crates/tinymemory-core/src/sync/composio/providers/user_scopes.rs b/crates/tinymemory-core/src/sync/composio/providers/user_scopes.rs deleted file mode 100644 index af7beecc..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/user_scopes.rs +++ /dev/null @@ -1,159 +0,0 @@ -//! Per-user, per-toolkit scope preferences. -//! -//! For each Composio toolkit a user has connected (or could connect), -//! we store a [`UserScopePref`] that records whether the agent is -//! allowed to call **read**, **write**, and / or **admin**-classified -//! actions for that toolkit. Defaults are `read=true, write=true, -//! admin=false` — the agent can use the integration productively out of -//! the box, but destructive / permission-changing actions require -//! explicit opt-in. -//! -//! Storage uses the same KV surface as [`super::sync_state`] -//! (`MemoryClient::kv_get` / `kv_set`) under a dedicated namespace so -//! prefs survive process restarts without any extra file management. - -use serde::{Deserialize, Serialize}; - -use crate::store::MemoryClientRef; - -use super::tool_scope::ToolScope; - -/// KV namespace for scope prefs. Separate from `composio-sync-state` so -/// the two never collide. -const KV_NAMESPACE: &str = "composio-user-scopes"; - -/// Per-toolkit scope preference. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub struct UserScopePref { - #[serde(default = "default_true")] - pub read: bool, - #[serde(default = "default_true")] - pub write: bool, - #[serde(default)] - pub admin: bool, -} - -fn default_true() -> bool { - true -} - -impl Default for UserScopePref { - fn default() -> Self { - Self { - read: true, - write: true, - admin: false, - } - } -} - -impl UserScopePref { - /// Returns `true` if the given scope is enabled in this preference. - pub fn allows(&self, scope: ToolScope) -> bool { - match scope { - ToolScope::Read => self.read, - ToolScope::Write => self.write, - ToolScope::Admin => self.admin, - } - } -} - -fn kv_key(toolkit: &str) -> String { - toolkit.trim().to_ascii_lowercase() -} - -/// Load the scope pref for `toolkit`. Returns the default -/// (`read+write`, no `admin`) when nothing is stored or when the KV -/// store can't be reached — the agent should always be able to use -/// connected integrations productively, even if pref storage is -/// temporarily unavailable. -pub async fn load(memory: &MemoryClientRef, toolkit: &str) -> UserScopePref { - let key = kv_key(toolkit); - if key.is_empty() { - return UserScopePref::default(); - } - match memory.kv_get(Some(KV_NAMESPACE), &key).await { - Ok(Some(value)) => match serde_json::from_value::(value) { - Ok(pref) => { - tracing::debug!( - toolkit = %key, - read = pref.read, - write = pref.write, - admin = pref.admin, - "[composio][scopes] pref loaded" - ); - pref - } - Err(e) => { - tracing::warn!( - toolkit = %key, - error = %e, - "[composio][scopes] pref deserialize failed, falling back to default" - ); - UserScopePref::default() - } - }, - Ok(None) => { - tracing::debug!( - toolkit = %key, - "[composio][scopes] no pref stored, using default (read+write)" - ); - UserScopePref::default() - } - Err(e) => { - tracing::warn!( - toolkit = %key, - error = %e, - "[composio][scopes] kv_get failed, falling back to default" - ); - UserScopePref::default() - } - } -} - -/// Persist a scope pref for `toolkit`. -pub async fn save( - memory: &MemoryClientRef, - toolkit: &str, - pref: UserScopePref, -) -> Result<(), String> { - let key = kv_key(toolkit); - if key.is_empty() { - return Err("user_scopes: toolkit must not be empty".to_string()); - } - let value = serde_json::to_value(pref) - .map_err(|e| format!("[composio][scopes] serialize failed: {e}"))?; - memory.kv_set(Some(KV_NAMESPACE), &key, &value).await?; - tracing::info!( - toolkit = %key, - read = pref.read, - write = pref.write, - admin = pref.admin, - "[composio][scopes] pref saved" - ); - Ok(()) -} - -/// Best-effort load that resolves the active memory client itself. Used -/// from the meta-tool layer where we don't have a `MemoryClientRef` in -/// scope. Falls back to the default pref when memory isn't initialised. -pub async fn load_or_default(toolkit: &str) -> UserScopePref { - match crate::global::client_if_ready() { - Some(client) => load(&client, toolkit).await, - None => { - // Match the normalized key form `load()` logs so traces - // grouped by `key` correlate across both code paths. - let key = kv_key(toolkit); - tracing::debug!( - toolkit = %toolkit, - key = %key, - "[composio][scopes] memory not ready, using default pref" - ); - UserScopePref::default() - } - } -} - -#[cfg(test)] -#[path = "user_scopes_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/composio/providers/user_scopes_tests.rs b/crates/tinymemory-core/src/sync/composio/providers/user_scopes_tests.rs deleted file mode 100644 index b6c9e0ee..00000000 --- a/crates/tinymemory-core/src/sync/composio/providers/user_scopes_tests.rs +++ /dev/null @@ -1,103 +0,0 @@ -use super::*; -use crate::store::MemoryClient; -use std::sync::Arc; -use tempfile::TempDir; - -fn make_client() -> (TempDir, Arc) { - crate::test_seams::init(); - let tmp = TempDir::new().unwrap(); - let client = Arc::new( - MemoryClient::from_workspace_dir(tmp.path().join("workspace")) - .expect("memory client should initialize for user-scope tests"), - ); - (tmp, client) -} - -#[test] -fn default_is_read_write_no_admin() { - let p = UserScopePref::default(); - assert!(p.read); - assert!(p.write); - assert!(!p.admin); -} - -#[test] -fn allows_matches_scope() { - let p = UserScopePref { - read: true, - write: false, - admin: false, - }; - assert!(p.allows(ToolScope::Read)); - assert!(!p.allows(ToolScope::Write)); - assert!(!p.allows(ToolScope::Admin)); -} - -#[test] -fn round_trip_serde() { - let p = UserScopePref { - read: true, - write: true, - admin: true, - }; - let v = serde_json::to_value(p).unwrap(); - let back: UserScopePref = serde_json::from_value(v).unwrap(); - assert_eq!(p, back); -} - -#[test] -fn missing_fields_default_to_true_for_read_write() { - // Forward-compat: if we ever drop a field, existing stored - // documents still deserialize sensibly. - let v = serde_json::json!({}); - let p: UserScopePref = serde_json::from_value(v).unwrap(); - assert_eq!(p, UserScopePref::default()); -} - -#[tokio::test] -async fn save_and_load_round_trip_uses_normalized_toolkit_key() { - let (_tmp, client) = make_client(); - let pref = UserScopePref { - read: true, - write: false, - admin: true, - }; - - save(&client, " GMail ", pref).await.unwrap(); - - let loaded = load(&client, "gmail").await; - assert_eq!(loaded, pref); - - let raw = client - .kv_get(Some(KV_NAMESPACE), "gmail") - .await - .unwrap() - .expect("normalized toolkit key should be used"); - assert_eq!(raw.get("write").and_then(|v| v.as_bool()), Some(false)); - assert_eq!(raw.get("admin").and_then(|v| v.as_bool()), Some(true)); -} - -#[tokio::test] -async fn load_falls_back_to_default_when_stored_payload_is_invalid() { - let (_tmp, client) = make_client(); - client - .kv_set( - Some(KV_NAMESPACE), - "gmail", - &serde_json::json!("not-an-object"), - ) - .await - .unwrap(); - - let loaded = load(&client, "gmail").await; - assert_eq!(loaded, UserScopePref::default()); -} - -#[tokio::test] -async fn save_rejects_blank_toolkit() { - let (_tmp, client) = make_client(); - let err = save(&client, " ", UserScopePref::default()) - .await - .unwrap_err(); - assert!(err.contains("toolkit must not be empty")); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/client.rs b/crates/tinymemory-core/src/sync/pipelines/composio/client.rs deleted file mode 100644 index ba1fcff7..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/client.rs +++ /dev/null @@ -1,393 +0,0 @@ -//! Minimal direct/proxied Composio action client. - -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; - -use crate::sync::pipelines::traits::{ComposioMode, ComposioSyncConfig}; - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct ExecuteResponse { - #[serde(default)] - pub data: serde_json::Value, - #[serde(default)] - pub successful: bool, - #[serde(default)] - pub error: Option, - #[serde(rename = "costUsd", default)] - pub cost_usd: f64, - #[serde(rename = "markdownFormatted", default)] - pub markdown_formatted: Option, - #[serde(skip, default = "one_attempt")] - pub attempts: u32, -} - -fn one_attempt() -> u32 { - 1 -} - -#[derive(Debug, thiserror::Error)] -#[error("{message}")] -pub struct ExecuteError { - pub attempts: u32, - message: String, -} - -/// Time to establish a TCP/TLS connection to Composio or the proxy. -const CONNECT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); -/// Whole-request ceiling. Composio actions that page a large mailbox can run -/// long, so this is generous, but it is finite. -const REQUEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(120); - -#[derive(Clone)] -pub struct ComposioClient { - http: reqwest::Client, - config: ComposioSyncConfig, -} - -#[async_trait] -pub trait ActionExecutor: Send + Sync { - async fn execute( - &self, - action: &str, - arguments: serde_json::Value, - connection_id: Option<&str>, - ) -> anyhow::Result; -} - -#[async_trait] -impl ActionExecutor for ComposioClient { - async fn execute( - &self, - action: &str, - arguments: serde_json::Value, - connection_id: Option<&str>, - ) -> anyhow::Result { - ComposioClient::execute(self, action, arguments, connection_id).await - } -} - -impl ComposioClient { - /// A client with explicit connect and request timeouts. - /// - /// `reqwest::Client::new()` has none: a hung Composio or proxy connection - /// would stall the sync task indefinitely and hold the sync-state - /// mutation window open with it. The builder is fallible only on TLS - /// backend initialisation, which cannot happen with the rustls feature this - /// crate compiles; if it ever did, an untimed fallback would silently drop - /// the guarantee, so it panics loudly instead of degrading. - pub fn new(config: ComposioSyncConfig) -> Self { - let http = reqwest::Client::builder() - .connect_timeout(CONNECT_TIMEOUT) - .timeout(REQUEST_TIMEOUT) - .build() - .unwrap_or_else(|error| { - panic!("Composio HTTP client failed to build (TLS backend unavailable): {error}") - }); - Self { http, config } - } - - pub fn with_http_client(mut self, http: reqwest::Client) -> Self { - self.http = http; - self - } - - pub async fn execute( - &self, - action: &str, - arguments: serde_json::Value, - connection_id: Option<&str>, - ) -> anyhow::Result { - let action = action.trim(); - anyhow::ensure!(!action.is_empty(), "Composio action must not be empty"); - const MAX_ATTEMPTS: u32 = 3; - for attempt in 1..=MAX_ATTEMPTS { - let result = match self.config.mode { - ComposioMode::Direct => { - self.execute_direct(action, arguments.clone(), connection_id) - .await - } - ComposioMode::Proxied => self.execute_proxied(action, arguments.clone()).await, - }; - match result { - Ok(mut response) - if response.successful - || !retryable_provider_error(response.error.as_deref()) - || attempt == MAX_ATTEMPTS => - { - response.attempts = attempt; - return Ok(response); - } - Ok(_) => tracing::warn!( - action, - attempt, - "[sync:composio] retrying provider rate limit" - ), - Err(error) if retryable_transport_error(&error) && attempt < MAX_ATTEMPTS => { - tracing::warn!(action, attempt, %error, "[sync:composio] retrying transient transport failure"); - } - Err(error) => { - return Err(ExecuteError { - attempts: attempt, - message: error.to_string(), - } - .into()) - } - } - tokio::time::sleep(std::time::Duration::from_millis( - 250 * 2u64.pow(attempt - 1), - )) - .await; - } - unreachable!("retry loop always returns") - } - - async fn execute_direct( - &self, - action: &str, - arguments: serde_json::Value, - connection_id: Option<&str>, - ) -> anyhow::Result { - let key = self - .config - .api_key - .as_ref() - .filter(|key| !key.is_empty()) - .map(|key| key.expose().to_owned()) - .filter(|key| !key.trim().is_empty()) - .ok_or_else(|| anyhow::anyhow!("Composio direct API key is not configured"))?; - let url = format!( - "{}/tools/execute/{action}", - self.config.base_url.trim_end_matches('/') - ); - let mut body = serde_json::json!({ "arguments": arguments }); - if let Some(entity_id) = self - .config - .entity_id - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - { - body["user_id"] = serde_json::json!(entity_id); - } - if let Some(connection_id) = connection_id - .map(str::trim) - .filter(|value| !value.is_empty()) - { - body["connected_account_id"] = serde_json::json!(connection_id); - } - - let response = self - .http - .post(url) - .header("x-api-key", key) - .json(&body) - .send() - .await - .map_err(|error| anyhow::anyhow!("Composio direct transport error: {error}"))?; - let status = response.status(); - if !status.is_success() { - let _ = response.bytes().await; - anyhow::bail!("Composio direct request failed with HTTP {status}"); - } - let raw: serde_json::Value = decode_response(response, "direct").await?; - Ok(decode_direct_response(raw)) - } - - async fn execute_proxied( - &self, - action: &str, - arguments: serde_json::Value, - ) -> anyhow::Result { - let bearer = self - .config - .bearer_token - .as_ref() - .filter(|token| !token.is_empty()) - .ok_or_else(|| anyhow::anyhow!("Composio proxy bearer token is not configured"))?; - let url = format!( - "{}/agent-integrations/composio/execute", - self.config.base_url.trim_end_matches('/') - ); - let response = self - .http - .post(url) - .bearer_auth(bearer.expose()) - .json(&serde_json::json!({ "tool": action, "arguments": arguments })) - .send() - .await - .map_err(|error| anyhow::anyhow!("Composio proxy transport error: {error}"))?; - let status = response.status(); - if !status.is_success() { - let _ = response.bytes().await; - anyhow::bail!("Composio proxy request failed with HTTP {status}"); - } - let raw: serde_json::Value = response - .json() - .await - .map_err(|error| anyhow::anyhow!("Composio proxy response decode failed: {error}"))?; - decode_proxy_response(raw) - } -} - -/// Shape a direct-API payload into an [`ExecuteResponse`]. -/// -/// A payload that reports an `error` is not a success, whatever the -/// `successful` flag says or omits: consumers gate document creation on the -/// flag, and an error body must never be stored as content. -fn decode_direct_response(raw: serde_json::Value) -> ExecuteResponse { - let flagged = raw - .get("successful") - .and_then(serde_json::Value::as_bool) - .or_else(|| raw.get("success").and_then(serde_json::Value::as_bool)) - .unwrap_or(true); - let error = raw - .get("error") - .and_then(serde_json::Value::as_str) - .map(str::trim) - .filter(|error| !error.is_empty()) - .map(str::to_owned); - let successful = flagged && error.is_none(); - let data = raw.get("data").cloned().unwrap_or(raw); - ExecuteResponse { - data, - successful, - error, - cost_usd: 0.0, - markdown_formatted: None, - attempts: 1, - } -} - -fn decode_proxy_response(raw: serde_json::Value) -> anyhow::Result { - let payload = if raw.get("successful").is_some() { - raw - } else { - raw.get("data").cloned().unwrap_or(raw) - }; - serde_json::from_value(payload) - .map_err(|error| anyhow::anyhow!("Composio proxy response decode failed: {error}")) -} - -fn retryable_provider_error(error: Option<&str>) -> bool { - error.is_some_and(|error| { - let lower = error.to_ascii_lowercase(); - lower.contains("ratelimit") - || lower.contains("rate limit") - || lower.contains("too many requests") - }) -} - -/// Whether a failed execute is worth retrying with backoff. -/// -/// Retryable: rate limiting and upstream unavailability (429/502/503/504), -/// and transport failures (connect/read errors, timeouts) — reported by the -/// request paths as `"… transport error: …"`. NOT retryable: any other HTTP -/// status. 400/401/403/404 are permanent — an invalid API key must fail once, -/// not storm three times — and used to be caught by a `"request failed"` -/// needle that both status-bail messages also matched. -fn retryable_transport_error(error: &anyhow::Error) -> bool { - let message = error.to_string(); - [ - "HTTP 429", - "HTTP 502", - "HTTP 503", - "HTTP 504", - "transport error", - ] - .iter() - .any(|needle| message.contains(needle)) -} - -async fn decode_response( - response: reqwest::Response, - mode: &str, -) -> anyhow::Result { - response - .json() - .await - .map_err(|error| anyhow::anyhow!("Composio {mode} response decode failed: {error}")) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// 4xx is permanent: an invalid key must fail once, not retry with - /// backoff. Only rate-limit/upstream statuses and transport failures - /// (connect/read errors, timeouts) are worth another attempt. - #[test] - fn retry_classification_is_by_status_not_by_substring() { - let retry = |m: &str| retryable_transport_error(&anyhow::anyhow!("{m}")); - assert!(retry( - "Composio direct request failed with HTTP 429 Too Many Requests" - )); - assert!(retry( - "Composio proxy request failed with HTTP 503 Service Unavailable" - )); - assert!(retry("Composio direct transport error: connection reset")); - assert!(!retry( - "Composio direct request failed with HTTP 401 Unauthorized" - )); - assert!(!retry( - "Composio proxy request failed with HTTP 404 Not Found" - )); - assert!(!retry( - "Composio direct request failed with HTTP 400 Bad Request" - )); - } - - /// An error payload is a failure even when the flag is absent or true. - #[test] - fn an_error_payload_is_never_a_success() { - let r = decode_direct_response(serde_json::json!({"error": "quota exceeded"})); - assert!(!r.successful, "missing flag + error must be a failure"); - assert_eq!(r.error.as_deref(), Some("quota exceeded")); - - let r = decode_direct_response(serde_json::json!({"successful": true, "error": " boom "})); - assert!(!r.successful, "flag=true + error must still be a failure"); - assert_eq!(r.error.as_deref(), Some("boom")); - - let r = decode_direct_response( - serde_json::json!({"successful": true, "error": " ", "data": {"x": 1}}), - ); - assert!(r.successful, "an empty error string is no error"); - assert!(r.error.is_none()); - assert_eq!(r.data["x"], 1); - } - - /// The client is built with finite timeouts; a build failure must not - /// silently degrade to an untimed client. - #[test] - fn client_builds_with_timeouts() { - let _ = ComposioClient::new(ComposioSyncConfig::default()); - assert!(CONNECT_TIMEOUT < REQUEST_TIMEOUT); - } - - #[test] - fn proxied_backend_envelope_decodes_provider_response() { - let response = decode_proxy_response(serde_json::json!({ - "success": true, - "data": { - "successful": true, - "data": {"messages": [{"messageId": "message-1"}]}, - "error": null - } - })) - .unwrap(); - - assert!(response.successful); - assert_eq!(response.data["messages"][0]["messageId"], "message-1"); - } - - #[test] - fn flat_proxy_response_remains_supported() { - let response = decode_proxy_response(serde_json::json!({ - "successful": true, - "data": {"items": [1]} - })) - .unwrap(); - - assert!(response.successful); - assert_eq!(response.data["items"], serde_json::json!([1])); - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/connect.rs b/crates/tinymemory-core/src/sync/pipelines/composio/connect.rs deleted file mode 100644 index 1ea09988..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/connect.rs +++ /dev/null @@ -1,364 +0,0 @@ -//! Composio v3 login/connect helpers. -//! -//! This module owns the reusable, host-agnostic pieces of the Composio -//! connection flow so a harness (or a server-side host) can drive an OAuth -//! login without re-deriving the wire contract: -//! -//! * a small per-integration **entity-id store** ([`EntityStore`]) that -//! remembers the `user_id` chosen for each toolkit across runs so re-runs -//! reuse the same Composio "entity" instead of orphaning connections; -//! * pure parsers for the connected-account **status** lifecycle; and -//! * thin async wrappers over the three v3 endpoints the connect flow needs. -//! -//! ## Verified v3 endpoints -//! -//! All confirmed against the Composio SDK source (the generated OpenAPI client -//! these SDKs wrap) — — and the v3 API -//! reference at : -//! -//! * `GET /api/v3/auth_configs?toolkit_slug={slug}` — list auth configs; -//! response `{ items: [ { id, toolkit: { slug } } ] }`. -//! (`ts/packages/core/src/models/AuthConfigs.ts`, `authConfigs.types.ts`.) -//! * `POST /api/v3/connected_accounts/link` — create a Composio Connect Link; -//! body `{ auth_config_id, user_id, callback_url? }`, response -//! `{ connected_account_id, redirect_url }`. -//! (`ConnectedAccounts.ts` `link()`, `connected_accounts.py` `link()`.) -//! * `GET /api/v3/connected_accounts/{nanoid}` — poll status; top-level -//! `status` in `INITIALIZING | INITIATED | ACTIVE | EXPIRED | FAILED | -//! REVOKED`. (`connected_accounts.py` `wait_for_connection`.) -//! -//! Authentication is the direct-mode `x-api-key` header, matching -//! [`super::client::ComposioClient`]. No secret is ever logged and error paths -//! discard raw response bodies (they can echo the key back verbatim). - -use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; - -use serde::{Deserialize, Serialize}; -use serde_json::Value; -use uuid::Uuid; - -/// Connection statuses that will never recover on their own — polling should -/// stop and fail. Mirrors the Composio SDK's `terminalErrorStates` -/// (`FAILED`, `EXPIRED`, `REVOKED`); `INACTIVE` is intentionally excluded -/// because it can transition back to `ACTIVE`. -const TERMINAL_STATUSES: &[&str] = &["FAILED", "EXPIRED", "REVOKED", "DELETED"]; - -/// Generate a fresh Composio entity id (`user_id`) for a new connection. -/// -/// The `tinycortex-` prefix keeps harness-created entities recognisable in the -/// Composio dashboard while the UUID guarantees uniqueness. -pub fn generate_entity_id() -> String { - format!("tinycortex-{}", Uuid::new_v4()) -} - -/// True when a connected-account status string means the account is live and -/// usable for tool execution. Case-insensitive. -pub fn status_is_active(status: &str) -> bool { - status.trim().eq_ignore_ascii_case("ACTIVE") -} - -/// True when a status string is a terminal failure that polling must give up -/// on. Case-insensitive. -pub fn status_is_terminal(status: &str) -> bool { - let status = status.trim(); - TERMINAL_STATUSES - .iter() - .any(|terminal| status.eq_ignore_ascii_case(terminal)) -} - -/// Pull the connected-account `status` out of a get-by-id response. -/// -/// Composio has shipped the status both at the top level and nested under -/// `state.val` / `connectionData.val`; probe the known shapes. -pub fn extract_status(record: &Value) -> Option { - [ - record.get("status"), - record.pointer("/state/val/status"), - record.pointer("/connectionData/val/status"), - record.pointer("/connection_data/val/status"), - ] - .into_iter() - .flatten() - .find_map(Value::as_str) - .map(str::trim) - .filter(|status| !status.is_empty()) - .map(str::to_owned) -} - -/// Extract the OAuth redirect URL from a create-link response. The v3 `/link` -/// endpoint returns a flat `redirect_url`; older shapes nested it under -/// `connectionData.val.redirectUrl`, so probe both. -pub fn extract_redirect_url(record: &Value) -> Option { - [ - record.get("redirect_url"), - record.get("redirectUrl"), - record.pointer("/connectionData/val/redirectUrl"), - record.pointer("/connection_data/val/redirect_url"), - ] - .into_iter() - .flatten() - .find_map(Value::as_str) - .map(str::trim) - .filter(|url| !url.is_empty()) - .map(str::to_owned) -} - -/// Extract the connected-account id from a create-link response. The v3 -/// `/link` endpoint returns `connected_account_id`; legacy `initiate` returned -/// a top-level `id`. -pub fn extract_account_id(record: &Value) -> Option { - ["connected_account_id", "connectedAccountId", "id", "nanoid"] - .iter() - .find_map(|key| record.get(key).and_then(Value::as_str)) - .map(str::trim) - .filter(|id| !id.is_empty()) - .map(str::to_owned) -} - -/// Resolve an auth-config id for `toolkit` from a `GET /auth_configs` response. -/// -/// Prefers an item whose `toolkit.slug` matches (case-insensitively); falls -/// back to the first listed config when the toolkit was already used as a -/// server-side filter and the slug field is shaped differently. -pub fn resolve_auth_config_id(list: &Value, toolkit: &str) -> Option { - let items = list - .pointer("/items") - .and_then(Value::as_array) - .or_else(|| list.get("data").and_then(Value::as_array)) - .or_else(|| list.as_array())?; - - let matches_toolkit = |item: &Value| { - [ - item.pointer("/toolkit/slug"), - item.pointer("/toolkit/name"), - item.get("toolkit"), - ] - .into_iter() - .flatten() - .find_map(Value::as_str) - .map(|slug| slug.trim().eq_ignore_ascii_case(toolkit)) - .unwrap_or(false) - }; - let auth_config_id = |item: &Value| { - ["id", "nanoid"] - .iter() - .find_map(|key| item.get(key).and_then(Value::as_str)) - .map(str::trim) - .filter(|id| !id.is_empty()) - .map(str::to_owned) - }; - - items - .iter() - .find(|item| matches_toolkit(item)) - .and_then(auth_config_id) - .or_else(|| items.iter().find_map(auth_config_id)) -} - -/// A newly-created Composio Connect Link. -#[derive(Debug, Clone)] -pub struct ConnectionLink { - /// The pending connected-account id to poll for `ACTIVE`. - pub connected_account_id: String, - /// The OAuth URL the user must open to complete login, when the scheme is - /// redirect-based. `None` for schemes that activate without a browser step. - pub redirect_url: Option, -} - -/// Persistent, per-toolkit map of the entity id (`user_id`) chosen for each -/// integration. -/// -/// Stored as a small JSON object on disk (e.g. `.composio-harness.json`) so a -/// re-run reuses the same Composio entity instead of creating a fresh — and -/// therefore orphaned — connection every time. Load is best-effort: a missing -/// or corrupt file yields an empty store rather than an error. -#[derive(Debug, Clone)] -pub struct EntityStore { - path: PathBuf, - entries: BTreeMap, -} - -#[derive(Default, Serialize, Deserialize)] -struct EntityStoreFile { - /// toolkit slug -> entity id (`user_id`). - #[serde(default)] - entities: BTreeMap, -} - -impl EntityStore { - /// Load the store from `path`, tolerating a missing or unreadable file. - pub fn load(path: impl Into) -> Self { - let path = path.into(); - let entries = std::fs::read_to_string(&path) - .ok() - .and_then(|raw| serde_json::from_str::(&raw).ok()) - .map(|file| file.entities) - .unwrap_or_default(); - Self { path, entries } - } - - /// The backing file path. - pub fn path(&self) -> &Path { - &self.path - } - - /// The entity id recorded for `toolkit`, if any. - pub fn get(&self, toolkit: &str) -> Option<&str> { - self.entries.get(toolkit).map(String::as_str) - } - - /// Record `entity_id` for `toolkit` in memory (call [`Self::save`] to - /// persist). - pub fn set(&mut self, toolkit: &str, entity_id: impl Into) { - self.entries.insert(toolkit.to_owned(), entity_id.into()); - } - - /// Resolve the entity id to use when connecting `toolkit`, persisting the - /// choice so future runs are stable. - /// - /// Precedence: a value already stored for this toolkit wins; otherwise an - /// explicit `override_id` (e.g. `COMPOSIO_ENTITY_ID`) is adopted; otherwise - /// a fresh id is generated. The resolved id is written back and saved. - pub fn entity_id_for( - &mut self, - toolkit: &str, - override_id: Option<&str>, - ) -> std::io::Result { - if let Some(existing) = self.get(toolkit) { - return Ok(existing.to_owned()); - } - let chosen = override_id - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) - .unwrap_or_else(generate_entity_id); - self.set(toolkit, chosen.clone()); - self.save()?; - Ok(chosen) - } - - /// Serialize the store to its backing file (pretty JSON). - pub fn save(&self) -> std::io::Result<()> { - let file = EntityStoreFile { - entities: self.entries.clone(), - }; - let json = serde_json::to_string_pretty(&file) - .map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidData, error))?; - std::fs::write(&self.path, json) - } -} - -/// `GET /api/v3/auth_configs?toolkit_slug={toolkit}` — list auth configs, -/// optionally filtered to one toolkit. Returns the decoded JSON body so the -/// caller can resolve an id via [`resolve_auth_config_id`]. -pub async fn list_auth_configs( - http: &reqwest::Client, - base_url: &str, - api_key: &str, - toolkit: Option<&str>, -) -> anyhow::Result { - let mut request = http - .get(format!("{}/auth_configs", base_url.trim_end_matches('/'))) - .header("x-api-key", api_key); - if let Some(toolkit) = toolkit { - request = request.query(&[("toolkit_slug", toolkit)]); - } - let response = request - .send() - .await - .map_err(|error| anyhow::anyhow!("auth_configs request failed: {error}"))?; - let status = response.status(); - if !status.is_success() { - // Never echo the body; it can contain the key back verbatim. - let _ = response.bytes().await; - anyhow::bail!("auth_configs returned HTTP {status}"); - } - response - .json() - .await - .map_err(|error| anyhow::anyhow!("auth_configs decode failed: {error}")) -} - -/// `POST /api/v3/connected_accounts/link` — create a Composio Connect Link for -/// `auth_config_id` scoped to `user_id`. Returns the pending account id plus an -/// optional OAuth redirect URL. -pub async fn create_connection_link( - http: &reqwest::Client, - base_url: &str, - api_key: &str, - auth_config_id: &str, - user_id: &str, - callback_url: Option<&str>, -) -> anyhow::Result { - let mut body = serde_json::json!({ - "auth_config_id": auth_config_id, - "user_id": user_id, - }); - if let Some(callback_url) = callback_url - .map(str::trim) - .filter(|value| !value.is_empty()) - { - body["callback_url"] = serde_json::json!(callback_url); - } - let response = http - .post(format!( - "{}/connected_accounts/link", - base_url.trim_end_matches('/') - )) - .header("x-api-key", api_key) - .json(&body) - .send() - .await - .map_err(|error| anyhow::anyhow!("connected_accounts/link request failed: {error}"))?; - let status = response.status(); - if !status.is_success() { - let _ = response.bytes().await; - anyhow::bail!("connected_accounts/link returned HTTP {status}"); - } - let record: Value = response - .json() - .await - .map_err(|error| anyhow::anyhow!("connected_accounts/link decode failed: {error}"))?; - let connected_account_id = extract_account_id(&record).ok_or_else(|| { - anyhow::anyhow!("connected_accounts/link response missing a connected account id") - })?; - Ok(ConnectionLink { - connected_account_id, - redirect_url: extract_redirect_url(&record), - }) -} - -/// `GET /api/v3/connected_accounts/{account_id}` — fetch the current status of -/// a (possibly pending) connected account. Returns `None` if the response had -/// no recognisable status field. -pub async fn get_connection_status( - http: &reqwest::Client, - base_url: &str, - api_key: &str, - account_id: &str, -) -> anyhow::Result> { - let response = http - .get(format!( - "{}/connected_accounts/{account_id}", - base_url.trim_end_matches('/') - )) - .header("x-api-key", api_key) - .send() - .await - .map_err(|error| anyhow::anyhow!("connected_accounts/{{id}} request failed: {error}"))?; - let status = response.status(); - if !status.is_success() { - let _ = response.bytes().await; - anyhow::bail!("connected_accounts/{{id}} returned HTTP {status}"); - } - let record: Value = response - .json() - .await - .map_err(|error| anyhow::anyhow!("connected_accounts/{{id}} decode failed: {error}"))?; - Ok(extract_status(&record)) -} - -#[cfg(test)] -#[path = "connect_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/connect_tests.rs b/crates/tinymemory-core/src/sync/pipelines/composio/connect_tests.rs deleted file mode 100644 index 7e893bcd..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/connect_tests.rs +++ /dev/null @@ -1,305 +0,0 @@ -//! Unit tests for the pure Composio connect helpers: entity-id persistence, -//! status classification, and response-field extraction. No network I/O. - -use super::*; -use serde_json::json; -use wiremock::matchers::{body_partial_json, header, method, path as url_path, query_param}; -use wiremock::{Mock, MockServer, ResponseTemplate}; - -#[test] -fn generated_entity_id_is_prefixed_and_unique() { - let a = generate_entity_id(); - let b = generate_entity_id(); - assert!(a.starts_with("tinycortex-"), "unexpected id: {a}"); - assert_ne!(a, b, "two generated ids must differ"); -} - -#[test] -fn status_active_is_case_insensitive() { - assert!(status_is_active("ACTIVE")); - assert!(status_is_active("active")); - assert!(status_is_active(" Active ")); - assert!(!status_is_active("INITIATED")); - assert!(!status_is_active("FAILED")); -} - -#[test] -fn status_terminal_matches_failure_states_only() { - for terminal in ["FAILED", "expired", "Revoked", "DELETED"] { - assert!( - status_is_terminal(terminal), - "{terminal} should be terminal" - ); - } - for live in ["ACTIVE", "INITIATED", "INITIALIZING", "INACTIVE"] { - assert!(!status_is_terminal(live), "{live} should not be terminal"); - } -} - -#[test] -fn extract_status_probes_top_level_and_nested() { - assert_eq!( - extract_status(&json!({"status": "ACTIVE"})).as_deref(), - Some("ACTIVE") - ); - assert_eq!( - extract_status(&json!({"state": {"val": {"status": "INITIATED"}}})).as_deref(), - Some("INITIATED") - ); - assert_eq!( - extract_status(&json!({"connectionData": {"val": {"status": "EXPIRED"}}})).as_deref(), - Some("EXPIRED") - ); - assert_eq!(extract_status(&json!({"other": 1})), None); -} - -#[test] -fn extract_link_fields_from_v3_shape() { - let response = json!({ - "connected_account_id": "ca_abc123", - "redirect_url": "https://backend.composio.dev/oauth/start?token=xyz", - "link_token": "lt_123", - }); - assert_eq!(extract_account_id(&response).as_deref(), Some("ca_abc123")); - assert_eq!( - extract_redirect_url(&response).as_deref(), - Some("https://backend.composio.dev/oauth/start?token=xyz") - ); -} - -#[test] -fn extract_link_fields_tolerates_legacy_shape() { - let response = json!({ - "id": "ca_legacy", - "connectionData": {"val": {"status": "INITIATED", "redirectUrl": "https://x/y"}}, - }); - assert_eq!(extract_account_id(&response).as_deref(), Some("ca_legacy")); - assert_eq!( - extract_redirect_url(&response).as_deref(), - Some("https://x/y") - ); -} - -#[test] -fn resolve_auth_config_prefers_matching_toolkit() { - let list = json!({ - "items": [ - {"id": "ac_github", "toolkit": {"slug": "github"}}, - {"id": "ac_gmail", "toolkit": {"slug": "gmail"}}, - ] - }); - assert_eq!( - resolve_auth_config_id(&list, "gmail").as_deref(), - Some("ac_gmail") - ); - assert_eq!( - resolve_auth_config_id(&list, "github").as_deref(), - Some("ac_github") - ); -} - -#[test] -fn resolve_auth_config_falls_back_to_first_when_no_slug_match() { - // Server already filtered by toolkit_slug; items may not echo a slug shape - // we recognise, so fall back to the first listed config. - let list = json!({ "items": [ {"id": "ac_only"} ] }); - assert_eq!( - resolve_auth_config_id(&list, "gmail").as_deref(), - Some("ac_only") - ); - assert_eq!(resolve_auth_config_id(&json!({"items": []}), "gmail"), None); -} - -#[test] -fn entity_store_round_trips_and_reuses_ids() { - let dir = tempfile::tempdir().unwrap(); - let path = dir.path().join(".composio-harness.json"); - - let mut store = EntityStore::load(&path); - assert_eq!(store.get("gmail"), None); - - // First resolve for gmail with an explicit override adopts + persists it. - let gmail_id = store.entity_id_for("gmail", Some("my-entity")).unwrap(); - assert_eq!(gmail_id, "my-entity"); - - // A second toolkit with no override generates a fresh id. - let github_id = store.entity_id_for("github", None).unwrap(); - assert!(github_id.starts_with("tinycortex-")); - assert_ne!(github_id, gmail_id); - - // Reload from disk: both ids survived and are stable on re-resolve. - let mut reloaded = EntityStore::load(&path); - assert_eq!(reloaded.get("gmail"), Some("my-entity")); - assert_eq!(reloaded.get("github"), Some(github_id.as_str())); - // Stored value wins even if a different override is passed on re-run. - assert_eq!( - reloaded.entity_id_for("gmail", Some("different")).unwrap(), - "my-entity" - ); -} - -#[test] -fn entity_store_load_tolerates_missing_and_corrupt_files() { - let dir = tempfile::tempdir().unwrap(); - let missing = dir.path().join("nope.json"); - assert_eq!(EntityStore::load(&missing).get("gmail"), None); - - let corrupt = dir.path().join("corrupt.json"); - std::fs::write(&corrupt, "{ not json ").unwrap(); - assert_eq!(EntityStore::load(&corrupt).get("gmail"), None); -} - -#[tokio::test] -async fn list_auth_configs_sends_filter_and_decodes_success() { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(url_path("/api/v3/auth_configs")) - .and(query_param("toolkit_slug", "gmail")) - .and(header("x-api-key", "secret")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "items": [{"id": "ac_gmail", "toolkit": {"slug": "gmail"}}] - }))) - .mount(&server) - .await; - - let value = list_auth_configs( - &reqwest::Client::new(), - &format!("{}/api/v3/", server.uri()), - "secret", - Some("gmail"), - ) - .await - .unwrap(); - - assert_eq!( - resolve_auth_config_id(&value, "gmail").as_deref(), - Some("ac_gmail") - ); -} - -#[tokio::test] -async fn list_auth_configs_redacts_error_body_and_rejects_invalid_json() { - let failure = MockServer::start().await; - Mock::given(method("GET")) - .and(url_path("/auth_configs")) - .respond_with(ResponseTemplate::new(401).set_body_string("echoed-secret")) - .mount(&failure) - .await; - let error = list_auth_configs( - &reqwest::Client::new(), - &failure.uri(), - "echoed-secret", - None, - ) - .await - .unwrap_err() - .to_string(); - assert!(error.contains("HTTP 401")); - assert!(!error.contains("echoed-secret")); - - let malformed = MockServer::start().await; - Mock::given(method("GET")) - .and(url_path("/auth_configs")) - .respond_with(ResponseTemplate::new(200).set_body_string("not-json")) - .mount(&malformed) - .await; - assert!( - list_auth_configs(&reqwest::Client::new(), &malformed.uri(), "key", None) - .await - .unwrap_err() - .to_string() - .contains("decode failed") - ); -} - -#[tokio::test] -async fn create_connection_link_sends_trimmed_callback_and_extracts_fields() { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(url_path("/connected_accounts/link")) - .and(header("x-api-key", "secret")) - .and(body_partial_json(json!({ - "auth_config_id": "ac_1", - "user_id": "user_1", - "callback_url": "https://callback" - }))) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "connected_account_id": "ca_1", - "redirect_url": "https://oauth" - }))) - .mount(&server) - .await; - - let link = create_connection_link( - &reqwest::Client::new(), - &server.uri(), - "secret", - "ac_1", - "user_1", - Some(" https://callback "), - ) - .await - .unwrap(); - assert_eq!(link.connected_account_id, "ca_1"); - assert_eq!(link.redirect_url.as_deref(), Some("https://oauth")); -} - -#[tokio::test] -async fn create_connection_link_rejects_http_decode_and_missing_id_failures() { - for (status, body, expected) in [ - (500, "server error", "HTTP 500"), - (200, "not-json", "decode failed"), - (200, "{}", "missing a connected account id"), - ] { - let server = MockServer::start().await; - Mock::given(method("POST")) - .and(url_path("/connected_accounts/link")) - .respond_with(ResponseTemplate::new(status).set_body_string(body)) - .mount(&server) - .await; - let error = create_connection_link( - &reqwest::Client::new(), - &server.uri(), - "key", - "ac", - "user", - Some(" "), - ) - .await - .unwrap_err() - .to_string(); - assert!(error.contains(expected), "unexpected error: {error}"); - } -} - -#[tokio::test] -async fn get_connection_status_handles_success_and_safe_failures() { - let active = MockServer::start().await; - Mock::given(method("GET")) - .and(url_path("/connected_accounts/ca_1")) - .and(header("x-api-key", "key")) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({"status": "ACTIVE"}))) - .mount(&active) - .await; - assert_eq!( - get_connection_status(&reqwest::Client::new(), &active.uri(), "key", "ca_1") - .await - .unwrap() - .as_deref(), - Some("ACTIVE") - ); - - for (status, body, expected) in [(403, "key", "HTTP 403"), (200, "invalid", "decode failed")] { - let server = MockServer::start().await; - Mock::given(method("GET")) - .and(url_path("/connected_accounts/ca_2")) - .respond_with(ResponseTemplate::new(status).set_body_string(body)) - .mount(&server) - .await; - let error = get_connection_status(&reqwest::Client::new(), &server.uri(), "key", "ca_2") - .await - .unwrap_err() - .to_string(); - assert!(error.contains(expected)); - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/gmail.rs b/crates/tinymemory-core/src/sync/pipelines/composio/gmail.rs deleted file mode 100644 index de04774f..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/gmail.rs +++ /dev/null @@ -1,380 +0,0 @@ -//! Incremental Gmail synchronization through Composio. - -use std::sync::Arc; - -use async_trait::async_trait; -use chrono::{DateTime, Utc}; -use serde_json::Value; - -use super::client::{ActionExecutor, ComposioClient}; -use super::orchestrator::{ - run_incremental_sync, IncrementalSource, PageFetch, SyncItem, SyncScope, -}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; -use tinymemory_sync::email_clean; -use tinymemory_sync::email_markdown::{self as email, EmailMessage, EmailThread}; - -const ACTION_FETCH_EMAILS: &str = "GMAIL_FETCH_EMAILS"; - -pub struct GmailSyncPipeline { - executor: Arc, - connection_id: String, - max_pages: usize, - page_size: usize, - query_override: Option, -} - -impl GmailSyncPipeline { - /// Sync through a plain Composio client. - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self::with_executor(Arc::new(client), connection_id) - } - - /// Sync through a caller-supplied executor. - /// - /// The seam exists for host-side response reshaping: the Gmail envelope - /// rewrite (verbose MIME payload → one slim record per message, body - /// pre-rendered into `markdown`) lives in the host, above this crate, so - /// wrapping the executor is the only way it can reach the fetched page - /// before [`document`](SyncPipeline) turns it into a stored document. - pub fn with_executor( - executor: Arc, - connection_id: impl Into, - ) -> Self { - Self { - executor, - connection_id: connection_id.into(), - max_pages: 10, - // Gmail fetches full message payloads (`include_payload: true`), so a - // large page overflows Composio's tool-response size cap with HTTP - // 413. 25 full messages/request stays comfortably under it; callers - // needing more throughput can raise it via `with_limits`. - page_size: 25, - query_override: None, - } - } - - pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { - self.max_pages = max_pages.max(1); - self.page_size = page_size.max(1); - self - } - - pub fn with_query(mut self, query: impl Into) -> Self { - self.query_override = Some(query.into()); - self - } -} - -#[async_trait] -impl SyncPipeline for GmailSyncPipeline { - fn id(&self) -> &str { - "composio:gmail" - } - - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - - async fn init(&self, _config: &PipelineConfig, _context: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - - async fn tick( - &self, - _config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync( - self, - self.executor.as_ref(), - &self.connection_id, - _config, - context, - ) - .await - } -} - -#[async_trait] -impl IncrementalSource for GmailSyncPipeline { - fn toolkit(&self) -> &'static str { - "gmail" - } - - fn action(&self) -> &'static str { - ACTION_FETCH_EMAILS - } - - fn max_pages(&self) -> usize { - self.max_pages - } - fn stop_on_empty_pending(&self) -> bool { - true - } - - fn server_side_depth(&self) -> bool { - true - } - - /// Gmail pages are capped by `max_results`, and full message payloads make - /// a page's size depend on what is *in* the mail — a handful of large - /// attachments is enough for the provider to refuse 25 messages it accepted - /// yesterday. Naming the argument lets the orchestrator halve it and retry - /// rather than leaving the source stuck. - fn page_size_arg_key(&self) -> Option<&'static str> { - Some("max_results") - } - - fn arguments( - &self, - _scope: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - let mut arguments = serde_json::json!({ - "max_results": self.page_size, - "include_payload": true, - }); - if let Some(token) = page { - arguments["page_token"] = serde_json::json!(token); - } - if let Some(query) = self.query_override.as_deref() { - arguments["query"] = Value::String(query.into()); - } else if let Some(cursor) = state.cursor.as_deref() { - arguments["query"] = serde_json::json!(format!( - "after:{}", - cursor_to_seconds(cursor).unwrap_or_default() - )); - } else if let Some(days) = config.sync_depth_days { - arguments["query"] = serde_json::json!(format!( - "after:{}", - (chrono::Utc::now() - chrono::Duration::days(days as i64)).timestamp() - )); - } - arguments - } - - fn extract_page(&self, data: &Value, _page: Option<&str>) -> PageFetch { - PageFetch { - items: extract_messages(data), - next: extract_page_token(data), - } - } - - fn dedup_key(&self, item: &Value) -> Option { - item_id(item) - } - - fn sort_cursor(&self, item: &Value) -> Option { - item_cursor(item) - } - - async fn document( - &self, - _scope: &SyncScope, - connection_id: &str, - item: SyncItem, - _executor: &dyn ActionExecutor, - _state: &mut SyncState, - ) -> anyhow::Result { - let id = item_id(&item.raw).unwrap_or_else(|| item.dedup_key.clone()); - Ok(SkillDocument { - namespace_skill_id: "gmail".into(), - connection_id: connection_id.into(), - document_id: format!("gmail:{id}"), - title: message_title(&item.raw), - content: canonical_markdown(&item.raw, &id), - toolkit: "gmail".into(), - metadata: serde_json::json!({ - "source": "composio-provider-incremental", - "taint": "external_sync", - "message_id": id, - }), - }) - } -} - -fn extract_messages(data: &Value) -> Vec { - [ - "/data/messages", - "/messages", - "/data/data/messages", - "/data/items", - "/items", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_array)) - .cloned() - .unwrap_or_default() -} - -fn extract_page_token(data: &Value) -> Option { - [ - "/data/nextPageToken", - "/nextPageToken", - "/data/data/nextPageToken", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::trim) - .filter(|token| !token.is_empty()) - .map(str::to_owned) -} - -fn item_id(message: &Value) -> Option { - ["id", "messageId", "message_id"] - .iter() - .find_map(|key| message.get(key).and_then(Value::as_str)) - .map(str::trim) - .filter(|id| !id.is_empty()) - .map(str::to_owned) -} - -fn item_cursor(message: &Value) -> Option { - ["internalDate", "internal_date", "date"] - .iter() - .find_map(|key| message.get(key).and_then(Value::as_str)) - .map(str::trim) - .filter(|cursor| !cursor.is_empty()) - .map(str::to_owned) -} - -fn message_title(message: &Value) -> String { - ["subject", "title"] - .iter() - .find_map(|key| message.get(key).and_then(Value::as_str)) - .map(str::trim) - .filter(|title| !title.is_empty()) - .unwrap_or("Gmail message") - .to_owned() -} - -/// Render one Gmail message as canonical Markdown — the same shape the memory -/// tree ingests — rather than the provider's raw JSON. -/// -/// Storing `to_string_pretty(&item.raw)` puts a MIME tree, `Received:` headers -/// and base64 part bodies into the document: the literal words of the mail are -/// either absent or split mid-token by the chunker, so recall can never match -/// them. Routing through [`email::canonicalise`] reuses the canonicaliser the -/// tree already uses — headers as a small block, body through -/// `email_clean::clean_body` (reply chains and footer boilerplate stripped). -fn canonical_markdown(message: &Value, id: &str) -> String { - let thread = email_thread(message, id); - match email::thread_markdown(thread) { - Some(markdown) => markdown, - // The thread built here always holds exactly one message, so an empty - // thread (`None`) is unreachable in practice. Degrade to the bare body - // rather than dropping the message out of memory. - None => message_body(message), - } -} - -/// Adapt one provider message into the canonicaliser's input shape. A Gmail -/// sync item is a single message, so the thread wraps exactly one. -fn email_thread(message: &Value, id: &str) -> EmailThread { - let subject = message_title(message); - EmailThread { - provider: "gmail".into(), - thread_subject: subject.clone(), - messages: vec![EmailMessage { - from: message_sender(message), - to: message_recipients(message), - cc: Vec::new(), - subject, - sent_at: message_sent_at(message), - body: message_body(message), - source_ref: Some(format!("gmail:{id}")), - list_unsubscribe: None, - }], - } -} - -/// Body text for one message, best rendering first. -/// -/// `markdown` is what the Gmail response reshaper pins onto each message (HTML -/// stripped, URLs shortened, footers removed). `messageText` is the provider's -/// own plain-text rendering, used when the reshape did not run. `snippet` is a -/// last resort: truncated, but real prose — unlike the raw payload. -fn message_body(message: &Value) -> String { - ["markdown", "markdownFormatted", "messageText", "snippet"] - .iter() - .find_map(|key| nonempty_str(message, key)) - .unwrap_or_default() -} - -/// Sender header, rendered as `From:` and used by the canonicaliser as the -/// participant key. -fn message_sender(message: &Value) -> String { - ["from", "sender"] - .iter() - .find_map(|key| nonempty_str(message, key)) - .unwrap_or_else(|| "unknown".to_owned()) -} - -/// Recipients arrive as one comma-joined header string (some responses use an -/// array); split them so the canonicaliser can render a `To:` line. -fn message_recipients(message: &Value) -> Vec { - match message.get("to") { - Some(Value::String(header)) => header - .split(',') - .map(str::trim) - .filter(|address| !address.is_empty()) - .map(str::to_owned) - .collect(), - Some(Value::Array(values)) => values - .iter() - .filter_map(Value::as_str) - .map(str::trim) - .filter(|address| !address.is_empty()) - .map(str::to_owned) - .collect(), - _ => Vec::new(), - } -} - -/// Send time, preferring the canonicaliser's own `Value`-level date parser (it -/// already knows `date`, `internalDate`, and epoch-ms-as-string) and falling -/// back to the sync cursor. The epoch is the last resort because it is -/// *deterministic*: a message the provider dated with nothing must not rewrite -/// its own content — and so re-chunk and re-embed — on every sync. -fn message_sent_at(message: &Value) -> DateTime { - email_clean::parse_message_date(message) - .or_else(|| { - item_cursor(message) - .as_deref() - .and_then(cursor_to_seconds) - .and_then(|seconds| DateTime::from_timestamp(seconds, 0)) - }) - .unwrap_or_else(|| DateTime::from_timestamp(0, 0).expect("epoch is a valid timestamp")) -} - -/// Read `key` as a trimmed, non-empty string. Unlike a plain `get(..).as_str()` -/// chain over a candidate list, a present-but-blank field falls through to the -/// next candidate instead of ending the search. -fn nonempty_str(message: &Value, key: &str) -> Option { - message - .get(key) - .and_then(Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) -} - -fn cursor_to_seconds(cursor: &str) -> Option { - if let Ok(milliseconds) = cursor.trim().parse::() { - return Some(milliseconds / 1000); - } - chrono::DateTime::parse_from_rfc3339(cursor) - .ok() - .map(|date| date.timestamp()) -} - -#[cfg(test)] -#[path = "gmail_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/gmail_tests.rs b/crates/tinymemory-core/src/sync/pipelines/composio/gmail_tests.rs deleted file mode 100644 index d4de7681..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/gmail_tests.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! Tests for the Gmail message → canonical Markdown adapter. - -use serde_json::json; - -use super::{canonical_markdown, message_body, message_recipients, message_sent_at}; - -/// One message in the shape the Gmail response reshaper emits: a slim envelope -/// whose body is pre-rendered into `markdown`. -fn slim_message() -> serde_json::Value { - json!({ - "id": "18f0abc", - "threadId": "18f0abc", - "subject": "Boulder visit", - "from": "Advising ", - "to": "me@example.com, second@example.com", - "date": "2026-05-02T09:15:00Z", - "labels": ["INBOX"], - "markdown": "The University of Colorado orientation is on May 20.\n\nOn Fri, 1 May 2026, someone wrote:\n> please ignore this quoted reply", - }) -} - -#[test] -fn canonical_markdown_renders_headers_and_cleaned_body() { - let content = canonical_markdown(&slim_message(), "18f0abc"); - - // The literal words of the mail — the thing recall has to match — are - // present as prose, and the headers are readable rather than a MIME tree. - assert!( - content.contains("The University of Colorado orientation is on May 20."), - "body text must survive canonicalisation: {content}" - ); - assert!( - content.contains("From: Advising "), - "{content}" - ); - assert!(content.contains("Subject: Boulder visit"), "{content}"); - assert!( - content.contains("To: me@example.com, second@example.com"), - "{content}" - ); - - // Canonicalisation is what strips the quoted reply chain. - assert!( - !content.contains("please ignore this quoted reply"), - "reply chain must be stripped by clean_body: {content}" - ); - - // Nothing JSON-shaped is left: this is the regression the fix exists for. - assert!( - !content.contains("\"markdown\""), - "raw JSON must not be stored: {content}" - ); - assert!( - !content.contains("threadId"), - "envelope keys must not be stored: {content}" - ); -} - -#[test] -fn body_falls_back_to_message_text_when_the_reshape_did_not_run() { - // No `markdown` field — the provider's own plain text is used instead. - let raw = json!({ - "id": "18f0def", - "subject": "Direct", - "messageText": "Plain provider text about Colorado.", - }); - assert_eq!(message_body(&raw), "Plain provider text about Colorado."); - assert!(canonical_markdown(&raw, "18f0def").contains("Plain provider text about Colorado.")); -} - -#[test] -fn body_skips_a_present_but_blank_field() { - // A blank `markdown` must not shadow a usable `messageText`: the candidate - // list falls through on emptiness, not just on absence. - let raw = json!({ "markdown": " ", "messageText": "real body" }); - assert_eq!(message_body(&raw), "real body"); -} - -#[test] -fn recipients_split_from_either_a_header_string_or_an_array() { - let joined = json!({ "to": "a@x.com, b@y.com" }); - assert_eq!(message_recipients(&joined), vec!["a@x.com", "b@y.com"]); - - let array = json!({ "to": ["a@x.com", " b@y.com "] }); - assert_eq!(message_recipients(&array), vec!["a@x.com", "b@y.com"]); - - assert!(message_recipients(&json!({})).is_empty()); -} - -#[test] -fn sent_at_reads_epoch_millis_and_is_deterministic_when_undated() { - // Gmail's `internalDate` is epoch millis as a string. - let dated = json!({ "internalDate": "1777712100000" }); - assert_eq!(message_sent_at(&dated).timestamp(), 1_777_712_100); - - // Undated messages must resolve to the same value every sync, otherwise the - // rendered `Date:` header changes and the document re-chunks forever. - let undated = json!({ "subject": "no date anywhere" }); - assert_eq!(message_sent_at(&undated), message_sent_at(&undated)); - assert_eq!(message_sent_at(&undated).timestamp(), 0); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/mod.rs b/crates/tinymemory-core/src/sync/pipelines/composio/mod.rs deleted file mode 100644 index 6c8645b6..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/mod.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! Composio sync, engine-free: HTTP client, connection lifecycle, the -//! incremental-sync orchestrator, and one pipeline per toolkit. - -pub mod client; -pub mod connect; -pub mod gmail; -pub mod orchestrator; -pub(crate) mod page_size; -pub mod providers; - -pub use client::{ActionExecutor, ComposioClient, ExecuteError, ExecuteResponse}; -pub use connect::{ - create_connection_link, generate_entity_id, get_connection_status, list_auth_configs, - resolve_auth_config_id, status_is_active, status_is_terminal, ConnectionLink, EntityStore, -}; -pub use gmail::GmailSyncPipeline; -pub use orchestrator::{run_incremental_sync, IncrementalSource, PageFetch, SyncItem, SyncScope}; -pub use providers::{ - ClickUpSyncPipeline, GitHubSyncPipeline, GoogleCalendarSyncPipeline, GoogleDocsSyncPipeline, - GoogleDriveSyncPipeline, GoogleSheetsSyncPipeline, LinearSyncPipeline, NotionSyncPipeline, - OutlookSyncPipeline, SlackSearchBackfillPipeline, SlackSyncPipeline, TodoistSyncPipeline, -}; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator.rs b/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator.rs deleted file mode 100644 index fd201371..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator.rs +++ /dev/null @@ -1,528 +0,0 @@ -//! Shared bounded incremental synchronization control flow. - -use async_trait::async_trait; -use serde_json::Value; - -use super::client::ActionExecutor; -use super::page_size::{apply_page_size, is_payload_too_large, shrink_page_size}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncEvent, SyncOutcome, SyncRunError, SyncStage, -}; - -#[derive(Debug)] -pub struct PageFetch { - pub items: Vec, - pub next: Option, -} - -#[derive(Debug)] -pub struct SyncItem { - pub dedup_key: String, - pub sort_cursor: Option, - pub raw: Value, -} - -#[derive(Clone, Debug, Default)] -pub struct SyncScope { - pub id: String, - pub label: String, - pub metadata: Value, -} - -impl SyncScope { - pub fn flat() -> Self { - Self::default() - } - - pub fn named(id: impl Into, label: impl Into) -> Self { - Self { - id: id.into(), - label: label.into(), - metadata: Value::Null, - } - } - - pub fn with_metadata(mut self, metadata: Value) -> Self { - self.metadata = metadata; - self - } -} - -#[async_trait] -pub trait IncrementalSource: Send + Sync { - fn toolkit(&self) -> &'static str; - fn action(&self) -> &'static str; - fn max_pages(&self) -> usize { - 10 - } - fn per_scope_cursors(&self) -> bool { - false - } - fn tolerate_scope_errors(&self) -> bool { - false - } - fn retain_dedup_keys(&self) -> bool { - true - } - fn stop_on_empty_pending(&self) -> bool { - false - } - fn server_side_depth(&self) -> bool { - false - } - fn depth_floor(&self, config: &PipelineConfig, state: &SyncState) -> Option { - if state.cursor.is_some() { - return None; - } - config.sync_depth_days.map(|days| { - (chrono::Utc::now() - chrono::Duration::days(days as i64)) - .format("%Y-%m-%dT%H:%M:%SZ") - .to_string() - }) - } - fn advance_scope_cursor(&self, _state: &mut SyncState, _scope: &SyncScope, _cursor: &str) {} - async fn scopes( - &self, - _executor: &dyn ActionExecutor, - _connection_id: &str, - _state: &mut SyncState, - ) -> anyhow::Result> { - Ok(vec![SyncScope::flat()]) - } - /// Name of the argument that caps how many items one page requests - /// (`max_results` for Gmail), when the action has one. - /// - /// Returning `Some` opts the source into the too-large-page retry: a page - /// the provider refuses purely for size is re-requested with the cap - /// halved, instead of failing the whole run. A source whose action has no - /// such knob returns `None` and keeps the previous behaviour. - fn page_size_arg_key(&self) -> Option<&'static str> { - None - } - fn arguments( - &self, - scope: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value; - fn extract_page(&self, data: &Value, page: Option<&str>) -> PageFetch; - fn dedup_key(&self, item: &Value) -> Option; - fn sort_cursor(&self, item: &Value) -> Option; - async fn document( - &self, - scope: &SyncScope, - connection_id: &str, - item: SyncItem, - executor: &dyn ActionExecutor, - state: &mut SyncState, - ) -> anyhow::Result; -} - -pub async fn run_incremental_sync( - source: &dyn IncrementalSource, - executor: &dyn ActionExecutor, - connection_id: &str, - config: &PipelineConfig, - context: &SyncContext, -) -> anyhow::Result { - let toolkit = source.toolkit(); - emit(context, toolkit, connection_id, SyncStage::Fetching, None).await; - tracing::debug!(toolkit, connection_id, "[sync:orchestrator] sync starting"); - - let mut state = SyncState::load(context.state.as_ref(), toolkit, connection_id).await?; - if state.budget_exhausted() { - tracing::debug!( - toolkit, - connection_id, - "[sync:orchestrator] daily budget exhausted" - ); - return Ok(SyncOutcome { - note: Some("daily request budget exhausted".into()), - ..SyncOutcome::default() - }); - } - - let result = match source.scopes(executor, connection_id, &mut state).await { - Ok(scopes) => { - run_pages( - source, - executor, - connection_id, - config, - context, - &mut state, - &scopes, - ) - .await - } - Err(error) => Err(error), - }; - state.last_sync_at_ms = Some(now_ms()); - if let Err(error) = state.save(context.state.as_ref()).await { - emit( - context, - toolkit, - connection_id, - SyncStage::Failed, - Some("sync state persistence failed".into()), - ) - .await; - return Err(error); - } - - match result { - Ok(outcome) => { - emit( - context, - toolkit, - connection_id, - SyncStage::Stored, - Some(format!("{} records", outcome.records_ingested)), - ) - .await; - emit(context, toolkit, connection_id, SyncStage::Completed, None).await; - tracing::debug!( - toolkit, - connection_id, - records = outcome.records_ingested, - more_pending = outcome.more_pending, - "[sync:orchestrator] sync completed" - ); - Ok(outcome) - } - Err(error) => { - tracing::warn!(toolkit, connection_id, %error, "[sync:orchestrator] sync failed"); - emit( - context, - toolkit, - connection_id, - SyncStage::Failed, - Some(error.to_string()), - ) - .await; - Err(SyncRunError::new( - error.to_string(), - state.run_requests, - state.run_provider_cost_usd, - ) - .into()) - } - } -} - -async fn run_pages( - source: &dyn IncrementalSource, - executor: &dyn ActionExecutor, - connection_id: &str, - config: &PipelineConfig, - context: &SyncContext, - state: &mut SyncState, - scopes: &[SyncScope], -) -> anyhow::Result { - let mut newest_cursor = state.cursor.clone(); - let mut ingested = 0u32; - // Estimated tokens of stored content this run, for `max_tokens_per_sync`. - let mut tokens_ingested: u64 = 0; - let mut more_pending = false; - let depth_floor = (!source.server_side_depth()) - .then(|| source.depth_floor(config, state)) - .flatten(); - - // Once a page proves too large for the provider, every later page of this - // run asks for the smaller size straight away rather than paying a rejected - // round-trip to rediscover the same limit. - let mut page_size_override: Option = None; - - 'scopes: for scope in scopes { - let mut page_token = None; - let mut scope_newest_cursor: Option = None; - let mut scope_failed = false; - 'pages: for page_index in 0..source.max_pages().max(1) { - if state.budget_exhausted() { - more_pending = true; - break 'scopes; - } - let mut arguments = source.arguments(scope, config, state, page_token.as_deref()); - apply_page_size( - &mut arguments, - source.page_size_arg_key(), - page_size_override, - ); - // The size a shrink most recently *tried*. Promoted to the run's - // sticky override only once the provider accepts a page at it — - // before that it names a size that may itself be refused. - let mut attempted_page_size: Option = None; - let response = loop { - let response = match executor - .execute(source.action(), arguments.clone(), Some(connection_id)) - .await - { - Ok(response) => response, - Err(error) if source.tolerate_scope_errors() => { - if let Some(execute_error) = - error.downcast_ref::() - { - state.record_requests(execute_error.attempts); - } - tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] scope fetch failed; continuing"); - scope_failed = true; - break 'pages; - } - Err(error) => { - if let Some(execute_error) = - error.downcast_ref::() - { - state.record_requests(execute_error.attempts); - } - return Err(error); - } - }; - // A completed provider round-trip is billable even when its - // envelope reports failure. Transport failures return before - // this point. - state.record_action(response.attempts, response.cost_usd); - if response.successful { - // Sticky for the rest of the run, and set HERE rather than - // at the shrink: with several halvings (25 → 12 → 6) an - // assignment per attempt leaves the last *rejected* size in - // the override on every step but the final one, and is - // correct at the end only because that step happens to be - // the accepted one. Recording the accepted size makes the - // intent independent of the retry order. - if let Some(accepted) = attempted_page_size { - page_size_override = Some(accepted); - } - break response; - } - // A page refused purely for its size is the one provider - // failure a *smaller request* can fix, so shrink and retry - // instead of failing the run. Without this a single oversized - // page stops the source dead until someone notices: on one live - // workspace Gmail sync sat broken for nine days that way. - if is_payload_too_large(response.error.as_deref()) { - if state.budget_exhausted() { - more_pending = true; - break 'scopes; - } - if let Some(reduced) = - shrink_page_size(&mut arguments, source.page_size_arg_key()) - { - attempted_page_size = Some(reduced); - tracing::warn!( - toolkit = source.toolkit(), - connection_id, - scope = %scope.label, - reduced_page_size = reduced, - "[sync:orchestrator] provider refused the page as too large; retrying with a smaller page" - ); - continue; - } - } - let error = anyhow::anyhow!( - "{} provider failure: {}", - source.toolkit(), - response - .error - .unwrap_or_else(|| "unknown provider error".into()) - ); - if source.tolerate_scope_errors() { - tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] provider rejected scope; continuing"); - scope_failed = true; - break 'pages; - } - return Err(error); - }; - - let fetched = source.extract_page(&response.data, page_token.as_deref()); - let mut reached_cursor_boundary = false; - let mut saw_unsynced_item = false; - for raw in fetched.items { - if config.max_items.is_some_and(|limit| ingested >= limit) { - more_pending = true; - break 'scopes; - } - if state.budget_exhausted() { - more_pending = true; - break 'scopes; - } - // Per-source spend caps (#18): checked with the same "stop, - // leave the rest pending" contract as `max_items`, so a - // capped run resumes from its cursor next tick. - if config - .max_cost_per_sync_usd - .is_some_and(|cap| state.run_provider_cost_usd >= cap) - { - more_pending = true; - break 'scopes; - } - if config - .max_tokens_per_sync - .is_some_and(|cap| tokens_ingested >= cap) - { - more_pending = true; - break 'scopes; - } - let Some(dedup_key) = source.dedup_key(&raw) else { - continue; - }; - if state.is_synced(&dedup_key) { - continue; - } - saw_unsynced_item = true; - let sort_cursor = source.sort_cursor(&raw); - if sort_cursor - .as_deref() - .zip(depth_floor.as_deref()) - .is_some_and(|(item_cursor, floor)| item_cursor < floor) - { - reached_cursor_boundary = true; - break; - } - if !source.per_scope_cursors() - && sort_cursor - .as_deref() - .zip(state.cursor.as_deref()) - .is_some_and(|(item_cursor, persisted_cursor)| { - item_cursor <= persisted_cursor - }) - { - tracing::debug!( - toolkit = source.toolkit(), - connection_id, - scope = %scope.label, - "[sync:orchestrator] reached persisted cursor boundary" - ); - reached_cursor_boundary = true; - break; - } - let document = match source - .document( - scope, - connection_id, - SyncItem { - dedup_key: dedup_key.clone(), - sort_cursor: sort_cursor.clone(), - raw, - }, - executor, - state, - ) - .await - { - Ok(document) => document, - Err(error) if source.tolerate_scope_errors() => { - tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] scope document conversion failed; continuing"); - scope_failed = true; - break; - } - Err(error) => return Err(error), - }; - // Same rough estimate the tree's budgeting uses (~4 chars per - // token); a cap, not an invoice. - tokens_ingested = - tokens_ingested.saturating_add((document.content.len() / 4) as u64); - if let Err(error) = context.documents.store(document).await { - if source.tolerate_scope_errors() { - tracing::warn!(toolkit = source.toolkit(), connection_id, scope = %scope.label, %error, "[sync:orchestrator] scope document store failed; continuing"); - scope_failed = true; - break; - } - return Err(error); - } - if source.retain_dedup_keys() { - state.mark_synced(dedup_key); - } - if let Some(cursor) = sort_cursor { - let target = if source.per_scope_cursors() { - &mut scope_newest_cursor - } else { - &mut newest_cursor - }; - if target - .as_deref() - .is_none_or(|current| cursor.as_str() > current) - { - *target = Some(cursor); - } - } - ingested = ingested.saturating_add(1); - if config.max_items.is_some_and(|limit| ingested >= limit) { - more_pending = true; - break 'scopes; - } - } - - page_token = fetched.next; - if source.stop_on_empty_pending() && !saw_unsynced_item { - tracing::debug!( - toolkit = source.toolkit(), - connection_id, - scope = %scope.label, - "[sync:orchestrator] stopping after all-deduplicated page" - ); - break; - } - if reached_cursor_boundary { - break; - } - if page_token.is_none() { - break; - } - if page_index + 1 == source.max_pages().max(1) { - more_pending = true; - } - } - if source.per_scope_cursors() && !scope_failed && !more_pending { - if let Some(cursor) = scope_newest_cursor.as_deref() { - source.advance_scope_cursor(state, scope, cursor); - state.save(context.state.as_ref()).await?; - } - } - } - - if !source.per_scope_cursors() && !more_pending { - if let Some(cursor) = newest_cursor { - state.advance_cursor(cursor); - } - } - Ok(SyncOutcome { - records_ingested: ingested, - more_pending, - actions_called: state.run_requests, - provider_cost_usd: state.run_provider_cost_usd, - note: None, - }) -} - -async fn emit( - context: &SyncContext, - toolkit: &str, - connection_id: &str, - stage: SyncStage, - message: Option, -) { - let _ = context - .events - .emit(SyncEvent { - source_id: format!("composio:{toolkit}:{connection_id}"), - toolkit: toolkit.into(), - connection_id: Some(connection_id.into()), - stage, - message, - }) - .await; -} - -fn now_ms() -> u64 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap_or_default() - .as_millis() as u64 -} - -#[cfg(test)] -#[path = "orchestrator_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator_tests.rs b/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator_tests.rs deleted file mode 100644 index ba3f46af..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/orchestrator_tests.rs +++ /dev/null @@ -1,236 +0,0 @@ -//! Tests for the too-large-page retry. - -use std::sync::{Arc, Mutex}; - -use serde_json::json; - -use super::*; -use crate::sync::composio::providers::sync_state::SyncStateStore; -use crate::sync::pipelines::composio::client::ExecuteResponse; -use crate::sync::pipelines::traits::{SkillDocSink, SkillDocument, SyncEvent, SyncEventSink}; - -/// Executor that mimics a provider with a response-size ceiling: it refuses any -/// page asking for more than `accepts` items and records every size it was -/// asked for. -struct SizeLimitedExecutor { - accepts: u64, - requested: Mutex>, -} - -impl SizeLimitedExecutor { - fn new(accepts: u64) -> Self { - Self { - accepts, - requested: Mutex::new(Vec::new()), - } - } - - fn requested_sizes(&self) -> Vec { - self.requested.lock().unwrap().clone() - } -} - -#[async_trait] -impl ActionExecutor for SizeLimitedExecutor { - async fn execute( - &self, - _action: &str, - arguments: Value, - _connection_id: Option<&str>, - ) -> anyhow::Result { - let requested = arguments - .get("max_results") - .and_then(Value::as_u64) - .unwrap_or(0); - self.requested.lock().unwrap().push(requested); - - let mut response: ExecuteResponse = serde_json::from_value(json!({})).unwrap(); - if requested > self.accepts { - response.successful = false; - response.error = Some( - "413 {\"error\":{\"message\":\"The tool response payload is too large.\",\ - \"code\":1613,\"slug\":\"Upstream_PayloadTooLarge\"}}" - .to_string(), - ); - return Ok(response); - } - response.successful = true; - response.data = json!({ - "messages": [{ "id": format!("m{requested}"), "date": "1700000000000" }], - }); - Ok(response) - } -} - -/// Minimal paged source: one page, one item, page size declared as -/// `max_results` unless `declare_page_size` is off. -struct StubSource { - declare_page_size: bool, - initial_page_size: u64, -} - -#[async_trait] -impl IncrementalSource for StubSource { - fn toolkit(&self) -> &'static str { - "stub" - } - fn action(&self) -> &'static str { - "STUB_FETCH" - } - fn max_pages(&self) -> usize { - 1 - } - fn page_size_arg_key(&self) -> Option<&'static str> { - self.declare_page_size.then_some("max_results") - } - fn arguments( - &self, - _scope: &SyncScope, - _config: &PipelineConfig, - _state: &SyncState, - _page: Option<&str>, - ) -> Value { - json!({ "max_results": self.initial_page_size }) - } - fn extract_page(&self, data: &Value, _page: Option<&str>) -> PageFetch { - PageFetch { - items: data - .get("messages") - .and_then(Value::as_array) - .cloned() - .unwrap_or_default(), - next: None, - } - } - fn dedup_key(&self, item: &Value) -> Option { - item.get("id").and_then(Value::as_str).map(str::to_string) - } - fn sort_cursor(&self, item: &Value) -> Option { - item.get("date").and_then(Value::as_str).map(str::to_string) - } - async fn document( - &self, - _scope: &SyncScope, - connection_id: &str, - item: SyncItem, - _executor: &dyn ActionExecutor, - _state: &mut SyncState, - ) -> anyhow::Result { - Ok(SkillDocument { - namespace_skill_id: "stub".into(), - connection_id: connection_id.into(), - document_id: item.dedup_key, - title: "stub".into(), - content: "stub".into(), - toolkit: "stub".into(), - metadata: Value::Null, - }) - } -} - -#[derive(Default)] -struct NoopHost(Mutex>); - -#[async_trait] -impl SkillDocSink for NoopHost { - async fn store(&self, _: SkillDocument) -> anyhow::Result<()> { - Ok(()) - } - async fn delete(&self, _: &str, _: &str) -> anyhow::Result<()> { - Ok(()) - } -} - -#[async_trait] -impl SyncEventSink for NoopHost { - async fn emit(&self, _: SyncEvent) -> anyhow::Result<()> { - Ok(()) - } -} - -#[async_trait] -impl SyncStateStore for NoopHost { - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - Ok(self - .0 - .lock() - .unwrap() - .get(&format!("{namespace}:{key}")) - .cloned()) - } - async fn set(&self, namespace: &str, key: &str, value: &Value) -> anyhow::Result<()> { - self.0 - .lock() - .unwrap() - .insert(format!("{namespace}:{key}"), value.clone()); - Ok(()) - } -} - -fn context() -> SyncContext { - let host = Arc::new(NoopHost::default()); - SyncContext { - events: host.clone(), - documents: host.clone(), - state: host, - } -} - -#[tokio::test] -async fn an_oversized_page_is_halved_until_the_provider_accepts_it() { - // The provider takes 6 at a time; the source asks for 25. - let executor = SizeLimitedExecutor::new(6); - let source = StubSource { - declare_page_size: true, - initial_page_size: 25, - }; - - let outcome = run_incremental_sync( - &source, - &executor, - "conn-1", - &PipelineConfig::default(), - &context(), - ) - .await - .expect("a too-large page must not fail the run"); - - assert_eq!( - outcome.records_ingested, 1, - "the page is ingested after the retry" - ); - assert_eq!( - executor.requested_sizes(), - vec![25, 12, 6], - "each rejection halves the request until it fits" - ); -} - -#[tokio::test] -async fn a_source_without_a_page_size_argument_still_fails_fast() { - // No `page_size_arg_key` — there is nothing to shrink, so the old - // behaviour (surface the provider failure) must be preserved rather than - // looping on a request that can never change. - let executor = SizeLimitedExecutor::new(6); - let source = StubSource { - declare_page_size: false, - initial_page_size: 25, - }; - - let error = run_incremental_sync( - &source, - &executor, - "conn-1", - &PipelineConfig::default(), - &context(), - ) - .await - .expect_err("an unshrinkable too-large page is still a failure"); - - assert!(error.to_string().contains("provider failure"), "{error}"); - assert_eq!( - executor.requested_sizes(), - vec![25], - "no pointless retry of an identical request" - ); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/page_size.rs b/crates/tinymemory-core/src/sync/pipelines/composio/page_size.rs deleted file mode 100644 index d970493b..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/page_size.rs +++ /dev/null @@ -1,79 +0,0 @@ -//! Page-size retry for a provider that refuses a page for its size. -//! -//! A page rejected purely because the response is too big is the one provider -//! failure a *smaller request* can fix. The orchestrator halves the page-size -//! argument and retries rather than failing the source, so a single oversized -//! page cannot stop a sync dead — on one live workspace a Gmail sync sat broken -//! for nine days that way. - -use serde_json::Value; - -/// Smallest page a shrink will ask for. One item is the point past which a -/// too-large response is about that single item, not the batch size. -pub(super) const MIN_PAGE_SIZE: u64 = 1; - -/// Whether the provider refused a page for its *size* rather than for anything -/// about the request's content — the only failure a smaller page can fix. -/// -/// Matched on the error text because that is all the envelope carries: Composio -/// reports it as HTTP 413 with a `Upstream_PayloadTooLarge` slug, and other -/// backends phrase it as "payload too large" / "response too large". -pub(super) fn is_payload_too_large(error: Option<&str>) -> bool { - error.is_some_and(|error| { - let lower = error.to_ascii_lowercase(); - lower.contains("payloadtoolarge") - || lower.contains("payload_too_large") - || mentions_status_413(&lower) - || (lower.contains("too large") - && (lower.contains("payload") || lower.contains("response"))) - }) -} - -/// Whether `text` names HTTP 413 as a status code. -/// -/// The digits have to stand alone. An unanchored `contains("413")` also matches -/// a message id, an amount, or a timestamp that merely contains those three -/// digits, and every such match costs a shrink-and-retry cycle before the real -/// error is finally surfaced — on a failure that a smaller page was never going -/// to fix. -fn mentions_status_413(lower: &str) -> bool { - lower.match_indices("413").any(|(at, _)| { - let before_is_digit = lower[..at] - .chars() - .next_back() - .is_some_and(|c| c.is_ascii_digit()); - let after_is_digit = lower[at + 3..] - .chars() - .next() - .is_some_and(|c| c.is_ascii_digit()); - !before_is_digit && !after_is_digit - }) -} - -/// Pin the page-size argument to `size`, if the source declared one. -pub(super) fn apply_page_size(arguments: &mut Value, key: Option<&str>, size: Option) { - if let (Some(key), Some(size)) = (key, size) { - if let Some(slot) = arguments.get_mut(key) { - *slot = Value::from(size); - } - } -} - -/// Halve the page-size argument in place, returning the new value. -/// -/// `None` means retrying is pointless — the source declares no page-size -/// argument, this request does not carry it, or it is already at the floor. -pub(super) fn shrink_page_size(arguments: &mut Value, key: Option<&str>) -> Option { - let key = key?; - let current = arguments.get(key)?.as_u64()?; - if current <= MIN_PAGE_SIZE { - return None; - } - let reduced = (current / 2).max(MIN_PAGE_SIZE); - arguments[key] = Value::from(reduced); - Some(reduced) -} - -#[cfg(test)] -#[path = "page_size_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/page_size_tests.rs b/crates/tinymemory-core/src/sync/pipelines/composio/page_size_tests.rs deleted file mode 100644 index 6328c7b0..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/page_size_tests.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Tests for the page-size retry helpers. - -use serde_json::json; - -use super::*; - -#[test] -fn payload_too_large_is_told_apart_from_other_provider_errors() { - assert!(is_payload_too_large(Some( - "413 {\"slug\":\"Upstream_PayloadTooLarge\"}" - ))); - assert!(is_payload_too_large(Some("Response too large for tool"))); - assert!(!is_payload_too_large(Some("rate limit exceeded"))); - assert!(!is_payload_too_large(Some("invalid grant"))); - assert!(!is_payload_too_large(None)); -} - -#[test] -fn shrinking_stops_at_the_floor() { - let mut arguments = json!({ "max_results": 3 }); - assert_eq!( - shrink_page_size(&mut arguments, Some("max_results")), - Some(1) - ); - assert_eq!(arguments["max_results"], json!(1)); - // At one item per page there is nothing left to halve. - assert_eq!(shrink_page_size(&mut arguments, Some("max_results")), None); - // A source that declares no key, or a request that lacks it, cannot shrink. - assert_eq!(shrink_page_size(&mut arguments, None), None); - assert_eq!( - shrink_page_size(&mut json!({ "other": 10 }), Some("max_results")), - None - ); -} - -/// The digits have to stand alone. Every false positive here costs a -/// shrink-and-retry cycle on a failure a smaller page was never going to fix, -/// and delays the real error reaching the caller. -#[test] -fn a_number_that_merely_contains_413_is_not_a_status_code() { - assert!(is_payload_too_large(Some("HTTP 413 Payload Too Large"))); - assert!(is_payload_too_large(Some("upstream returned 413."))); - assert!(is_payload_too_large(Some("(413)"))); - - assert!(!is_payload_too_large(Some("message id 4130 not found"))); - assert!(!is_payload_too_large(Some("amount 1413 exceeds the cap"))); - assert!(!is_payload_too_large(Some("thread 94137 is archived"))); - assert!(!is_payload_too_large(Some( - "at 1782891413 the token expired" - ))); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/clickup.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/clickup.rs deleted file mode 100644 index d16176ec..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/clickup.rs +++ /dev/null @@ -1,196 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_USER: &str = "CLICKUP_GET_AUTHORIZED_USER"; -const ACTION_WORKSPACES: &str = "CLICKUP_GET_AUTHORIZED_TEAMS_WORKSPACES"; -const ACTION_TASKS: &str = "CLICKUP_GET_FILTERED_TEAM_TASKS"; - -pub struct ClickUpSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl ClickUpSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 20, - page_size: 50, - } - } -} - -#[async_trait] -impl SyncPipeline for ClickUpSyncPipeline { - fn id(&self) -> &str { - "composio:clickup" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for ClickUpSyncPipeline { - fn toolkit(&self) -> &'static str { - "clickup" - } - fn action(&self) -> &'static str { - ACTION_TASKS - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn depth_floor(&self, config: &PipelineConfig, state: &SyncState) -> Option { - if state.cursor.is_some() { - return None; - } - config.sync_depth_days.map(|days| { - (chrono::Utc::now() - chrono::Duration::days(days as i64)) - .timestamp_millis() - .to_string() - }) - } - async fn scopes( - &self, - executor: &dyn ActionExecutor, - connection_id: &str, - state: &mut SyncState, - ) -> anyhow::Result> { - let user_response = checked_execute( - executor, - ACTION_USER, - serde_json::json!({}), - connection_id, - state, - ) - .await?; - let user_id = ["/user/id", "/data/user/id", "/id", "/data/id"] - .iter() - .find_map(|path| user_response.data.pointer(path)) - .and_then(value_string) - .ok_or_else(|| anyhow::anyhow!("{ACTION_USER} returned no user id"))?; - if state.budget_exhausted() { - return Ok(Vec::new()); - } - let workspace_response = checked_execute( - executor, - ACTION_WORKSPACES, - serde_json::json!({}), - connection_id, - state, - ) - .await?; - let workspaces = first_array( - &workspace_response.data, - &["/teams", "/data/teams", "/workspaces", "/data/workspaces"], - ); - Ok(workspaces - .into_iter() - .filter_map(|workspace| pick_str(&workspace, &["id", "team_id", "workspace_id"])) - .map(|id| { - SyncScope::named(id.clone(), format!("workspace:{id}")) - .with_metadata(serde_json::json!({"user_id": user_id})) - }) - .collect()) - } - fn arguments( - &self, - scope: &SyncScope, - _: &PipelineConfig, - _: &SyncState, - page: Option<&str>, - ) -> Value { - serde_json::json!({"team_id": scope.id, "assignees": [scope.metadata.get("user_id").and_then(Value::as_str).unwrap_or_default()], "order_by": "updated", "reverse": true, "page": page.and_then(|value| value.parse::().ok()).unwrap_or(0), "page_size": self.page_size, "subtasks": true}) - } - fn extract_page(&self, data: &Value, page: Option<&str>) -> PageFetch { - let items = first_array( - data, - &[ - "/data/tasks", - "/tasks", - "/data/data/tasks", - "/data/results", - "/results", - "/data/items", - "/items", - ], - ); - let page_number = page - .and_then(|value| value.parse::().ok()) - .unwrap_or(0); - let next = (items.len() == self.page_size).then(|| (page_number + 1).to_string()); - PageFetch { items, next } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "task_id", "data.task_id"])?; - Some(match self.sort_cursor(item) { - Some(updated) => format!("{id}@{updated}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &[ - "date_updated", - "data.date_updated", - "updated_at", - "data.updated_at", - "dateUpdated", - "data.dateUpdated", - ], - ) - } - async fn document( - &self, - scope: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "task_id", "data.task_id"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str(&item.raw, &["name", "data.name", "title", "data.title"]) - .unwrap_or_else(|| format!("ClickUp task {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - let mut result = document("clickup", connection_id, &id, title, content, item.raw); - result.metadata["workspace_id"] = Value::String(scope.id.clone()); - Ok(result) - } -} - -fn value_string(value: &Value) -> Option { - match value { - Value::String(value) => Some(value.clone()), - Value::Number(value) => Some(value.to_string()), - _ => None, - } - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/common.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/common.rs deleted file mode 100644 index a76fa519..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/common.rs +++ /dev/null @@ -1,110 +0,0 @@ -use serde_json::Value; - -use crate::sync::pipelines::traits::SkillDocument; - -/// Walk a JSON document by dotted path and return the first non-empty scalar. -/// -/// # Not interchangeable with [`normalize::helpers::pick_str`] -/// -/// A second `pick_str` lives in [`normalize::helpers`], and the two differ. -/// This one resolves paths with [`Value::pointer`] (so a numeric segment -/// indexes into an array) and **coerces `Number` to its string form**; that -/// one walks with [`Value::get`] (objects only) and returns `None` for any -/// non-string leaf. Swapping one for the other changes what normalisers emit -/// for numeric fields. Keep them separate. -/// -/// [`normalize::helpers`]: tinymemory_sync::helpers -/// [`normalize::helpers::pick_str`]: tinymemory_sync::helpers::pick_str -pub fn pick_str(value: &Value, paths: &[&str]) -> Option { - paths.iter().find_map(|path| { - let pointer = format!("/{}", path.replace('.', "/")); - value - .pointer(&pointer) - .and_then(|value| match value { - Value::String(value) => Some(value.clone()), - Value::Number(value) => Some(value.to_string()), - _ => None, - }) - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) - }) -} - -pub fn first_array(data: &Value, pointers: &[&str]) -> Vec { - pointers - .iter() - .find_map(|pointer| data.pointer(pointer).and_then(Value::as_array)) - .cloned() - .unwrap_or_default() -} - -/// Reads a Google-style `nextPageToken` from the common Composio response -/// envelopes (single- and double-`data`-wrapped), trimming and dropping empty -/// tokens. Shared by the Google provider pipelines to avoid drift. -pub fn next_page_token(data: &Value) -> Option { - [ - "/data/nextPageToken", - "/nextPageToken", - "/data/data/nextPageToken", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::trim) - .filter(|token| !token.is_empty()) - .map(str::to_owned) -} - -pub fn document( - toolkit: &str, - connection_id: &str, - id: &str, - title: String, - content: String, - raw: Value, -) -> SkillDocument { - SkillDocument { - namespace_skill_id: toolkit.into(), - connection_id: connection_id.into(), - document_id: format!("{toolkit}:{id}"), - title, - content, - toolkit: toolkit.into(), - metadata: serde_json::json!({ - "source": "composio-provider-incremental", - "taint": "external_sync", - "provider_id": id, - "raw": raw, - }), - } -} - -pub async fn checked_execute( - executor: &dyn super::super::client::ActionExecutor, - action: &str, - arguments: Value, - connection_id: &str, - state: &mut crate::sync::composio::providers::sync_state::SyncState, -) -> anyhow::Result { - let response = match executor - .execute(action, arguments, Some(connection_id)) - .await - { - Ok(response) => response, - Err(error) => { - if let Some(error) = error.downcast_ref::() { - state.record_requests(error.attempts); - } - return Err(error); - } - }; - state.record_action(response.attempts, response.cost_usd); - anyhow::ensure!( - response.successful, - "{action} provider failure: {}", - response - .error - .as_deref() - .unwrap_or("unknown provider error") - ); - Ok(response) -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/github.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/github.rs deleted file mode 100644 index 032935ae..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/github.rs +++ /dev/null @@ -1,178 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_USER: &str = "GITHUB_GET_THE_AUTHENTICATED_USER"; -const ACTION_SEARCH: &str = "GITHUB_SEARCH_ISSUES_AND_PULL_REQUESTS"; - -pub struct GitHubSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl GitHubSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 20, - page_size: 50, - } - } -} - -#[async_trait] -impl SyncPipeline for GitHubSyncPipeline { - fn id(&self) -> &str { - "composio:github" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for GitHubSyncPipeline { - fn toolkit(&self) -> &'static str { - "github" - } - fn action(&self) -> &'static str { - ACTION_SEARCH - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn server_side_depth(&self) -> bool { - true - } - async fn scopes( - &self, - executor: &dyn ActionExecutor, - connection_id: &str, - state: &mut SyncState, - ) -> anyhow::Result> { - let response = checked_execute( - executor, - ACTION_USER, - serde_json::json!({}), - connection_id, - state, - ) - .await?; - let login = pick_str(&response.data, &["login", "data.login"]) - .ok_or_else(|| anyhow::anyhow!("{ACTION_USER} returned no login"))?; - Ok(vec![SyncScope::named( - login.clone(), - format!("involves:{login}"), - )]) - } - fn arguments( - &self, - scope: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - let mut query = format!("involves:{}", scope.id); - if let Some(cursor) = state.cursor.as_deref() { - query.push_str(&format!(" updated:>{cursor}")); - } else if let Some(days) = config.sync_depth_days { - let floor = chrono::Utc::now() - chrono::Duration::days(days as i64); - query.push_str(&format!(" updated:>{}", floor.format("%Y-%m-%dT%H:%M:%SZ"))); - } - serde_json::json!({ "q": query, "sort": "updated", "order": "desc", "per_page": self.page_size, "page": page.and_then(|value| value.parse::().ok()).unwrap_or(1) }) - } - fn extract_page(&self, data: &Value, page: Option<&str>) -> PageFetch { - let items = first_array( - data, - &[ - "/data/items", - "/items", - "/data/data/items", - "/data/results", - "/results", - ], - ); - let page_number = page - .and_then(|value| value.parse::().ok()) - .unwrap_or(1); - let next = (items.len() == self.page_size).then(|| (page_number + 1).to_string()); - PageFetch { items, next } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = issue_id(item)?; - Some(match self.sort_cursor(item) { - Some(updated) => format!("{id}@{updated}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &[ - "updated_at", - "data.updated_at", - "updatedAt", - "data.updatedAt", - ], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = issue_id(&item.raw).unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str(&item.raw, &["title", "data.title"]) - .unwrap_or_else(|| format!("GitHub issue {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - Ok(document( - "github", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} - -fn issue_id(item: &Value) -> Option { - pick_str(item, &["id", "data.id"]).or_else(|| { - let url = pick_str(item, &["html_url", "data.html_url", "url", "data.url"])?; - let parts: Vec<_> = url.trim_end_matches('/').split('/').collect(); - (parts.len() >= 7).then(|| { - format!( - "{}/{}#{}", - parts[parts.len() - 4], - parts[parts.len() - 3], - parts[parts.len() - 1] - ) - }) - }) -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_calendar.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_calendar.rs deleted file mode 100644 index 6e0a1f3e..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_calendar.rs +++ /dev/null @@ -1,173 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{document, first_array, next_page_token, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_EVENTS_LIST: &str = "GOOGLECALENDAR_EVENTS_LIST"; - -/// Incremental Google Calendar synchronization through Composio. -/// -/// Events are self-contained records (stable id + `updated` timestamp), so this -/// follows the document-shaped pattern (`LinearSyncPipeline`) rather than the -/// message-shaped one: a single list action, client-visible `updated` cursor, -/// content taken directly from the event payload with no secondary fetch. -pub struct GoogleCalendarSyncPipeline { - client: ComposioClient, - connection_id: String, - calendar_id: String, - max_pages: usize, - page_size: usize, -} - -impl GoogleCalendarSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - calendar_id: "primary".into(), - max_pages: 10, - page_size: 50, - } - } - - pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { - self.max_pages = max_pages.max(1); - // Google Calendar caps `maxResults` at 2500; stay well under it. - self.page_size = page_size.clamp(1, 2500); - self - } -} - -#[async_trait] -impl SyncPipeline for GoogleCalendarSyncPipeline { - fn id(&self) -> &str { - "composio:googlecalendar" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for GoogleCalendarSyncPipeline { - fn toolkit(&self) -> &'static str { - "googlecalendar" - } - fn action(&self) -> &'static str { - ACTION_EVENTS_LIST - } - fn max_pages(&self) -> usize { - self.max_pages - } - // NB: `stop_on_empty_pending` is left at its default (false). The cursor only - // advances on a *complete* sync, so a run capped by `max_pages`/budget leaves - // it unadvanced; stopping early on an all-deduplicated first page would then - // permanently skip the still-unsynced tail. The persisted-cursor boundary - // already halts incremental runs at the right point. - fn server_side_depth(&self) -> bool { - true - } - fn arguments( - &self, - _: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - // `single_events` expands recurring series into concrete instances so - // each carries a stable id; `order_by: "updated"` sorts ascending by - // modification time (oldest change first), matching the `updated` cursor. - let mut args = serde_json::json!({ - "calendar_id": self.calendar_id, - "max_results": self.page_size, - "single_events": true, - "order_by": "updated", - }); - if let Some(page) = page { - args["page_token"] = serde_json::json!(page); - } - // The cursor is a modification time (the item `updated` field), so it - // belongs on `updated_min` (last-modified lower bound) — NOT `time_min`, - // which filters by event *start* time and would drop recently-edited - // past events. `time_min` is only the start-time horizon for the first, - // cursorless backfill; once a cursor exists, `updated_min` fully bounds - // the incremental window. - if let Some(cursor) = state.cursor.as_deref() { - args["updated_min"] = serde_json::json!(cursor); - } else if let Some(days) = config.sync_depth_days { - args["time_min"] = serde_json::json!((chrono::Utc::now() - - chrono::Duration::days(days as i64)) - .to_rfc3339()); - } - args - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/items", - "/items", - "/data/data/items", - "/data/events", - "/events", - ], - ), - next: next_page_token(data), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "iCalUID", "data.iCalUID"])?; - Some(match self.sort_cursor(item) { - Some(updated) => format!("{id}@{updated}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str(item, &["updated", "data.updated"]) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "iCalUID", "data.iCalUID"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str( - &item.raw, - &["summary", "data.summary", "title", "data.title"], - ) - .unwrap_or_else(|| format!("Calendar event {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - Ok(document( - "googlecalendar", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_docs.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_docs.rs deleted file mode 100644 index e1809744..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_docs.rs +++ /dev/null @@ -1,216 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_SEARCH: &str = "GOOGLEDOCS_SEARCH_DOCUMENTS"; -const ACTION_PLAINTEXT: &str = "GOOGLEDOCS_GET_DOCUMENT_PLAINTEXT"; - -/// Incremental Google Docs synchronization through Composio. -/// -/// Two-step, document-shaped (like `NotionSyncPipeline`): `GOOGLEDOCS_SEARCH_DOCUMENTS` -/// enumerates accessible documents, then `GOOGLEDOCS_GET_DOCUMENT_PLAINTEXT` fetches the -/// body for each item inside [`IncrementalSource::document`]. -pub struct GoogleDocsSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl GoogleDocsSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - // NOTE: SEARCH_DOCUMENTS' page-token arg name is not pinned by the - // curated catalog, so we do a single-page-per-tick fetch (no page - // token emitted) rather than guessing a pagination scheme. Capped at - // 1 page: since `arguments()` never advances the token, a >1 cap - // would re-fire the identical page-1 request and burn budget slots - // for silently-deduplicated items. - max_pages: 1, - page_size: 25, - } - } -} - -#[async_trait] -impl SyncPipeline for GoogleDocsSyncPipeline { - fn id(&self) -> &str { - "composio:googledocs" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for GoogleDocsSyncPipeline { - fn toolkit(&self) -> &'static str { - "googledocs" - } - fn action(&self) -> &'static str { - ACTION_SEARCH - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn arguments( - &self, - _: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - _page: Option<&str>, - ) -> Value { - // `GOOGLEDOCS_SEARCH_DOCUMENTS` fronts Drive's `files.list`, so it takes - // the same server-side controls Drive does. Order deterministically by - // modification time and bound the window with a `q` clause, so each - // tick fetches what changed since the cursor rather than the same - // first batch forever. Without this the action returned the identical - // page every tick and documents past `max_results` were unreachable. - let mut args = serde_json::json!({ - "query": "", - "max_results": self.page_size, - "order_by": "modifiedTime desc", - }); - // Prefer the last-synced cursor, else the configured horizon. The - // cursor is validated as RFC 3339 before it is interpolated into `q`, - // so a malformed persisted value can never inject into the query — on - // a bad value the depth filter is simply omitted (full scan). - let floor = state - .cursor - .as_deref() - .filter(|cursor| chrono::DateTime::parse_from_rfc3339(cursor).is_ok()) - .map(str::to_owned) - .or_else(|| { - config.sync_depth_days.map(|days| { - (chrono::Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339() - }) - }); - if let Some(floor) = floor { - args["q"] = serde_json::json!(format!("modifiedTime > '{floor}'")); - } - args - } - fn server_side_depth(&self) -> bool { - // The `q` floor above bounds depth on the server, so the orchestrator - // must not additionally treat the cursor as a client-side stop. - true - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/documents", - "/documents", - "/data/files", - "/files", - "/data/results", - "/results", - "/data/items", - "/items", - ], - ), - // Bounded fetch: no page token consumed (see `max_pages`). The - // pointers are read defensively should Composio surface one. - next: [ - "/data/nextPageToken", - "/nextPageToken", - "/data/next_page_token", - "/next_page_token", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::trim) - .filter(|token| !token.is_empty()) - .map(str::to_owned), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "documentId", "data.documentId"])?; - Some(match self.sort_cursor(item) { - Some(modified) => format!("{id}@{modified}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &[ - "modifiedTime", - "data.modifiedTime", - "modified_time", - "updatedTime", - ], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - executor: &dyn ActionExecutor, - state: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str( - &item.raw, - &["id", "data.id", "documentId", "data.documentId"], - ) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str(&item.raw, &["title", "data.title", "name", "data.name"]) - .unwrap_or_else(|| format!("Google Doc {id}")); - // NOTE: GET_DOCUMENT_PLAINTEXT identifies the doc by an id argument; - // Composio commonly keys this as "id" (or "document_id"). We send "id". - let response = checked_execute( - executor, - ACTION_PLAINTEXT, - serde_json::json!({"id": id}), - connection_id, - state, - ) - .await?; - let content = [ - "/data/text", - "/text", - "/data/plaintext", - "/plaintext", - "/data/content", - "/content", - "/data/response_data/text", - ] - .iter() - .find_map(|path| response.data.pointer(path).and_then(Value::as_str)) - .filter(|value| !value.trim().is_empty()) - .map(str::to_owned) - .unwrap_or(serde_json::to_string_pretty(&item.raw)?); - Ok(document( - "googledocs", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_drive.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_drive.rs deleted file mode 100644 index d619aa2b..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_drive.rs +++ /dev/null @@ -1,179 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{document, first_array, next_page_token, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -// Composio deprecated `GOOGLEDRIVE_LIST_FILES` (2026-03-28) in favour of -// `GOOGLEDRIVE_FIND_FILE`, which is the current `files.list`-backed listing -// action (same paging/ordering/`q` filter surface). -const ACTION_FIND_FILE: &str = "GOOGLEDRIVE_FIND_FILE"; - -/// Incremental Google Drive synchronization through Composio. -/// -/// File-shaped: each Drive file is a record with a stable id and a -/// `modifiedTime`. This indexes file *metadata* only — it never downloads -/// binary bodies (which may be arbitrarily large and are not memory-shaped); -/// the document content is the file's structured metadata. -pub struct GoogleDriveSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl GoogleDriveSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 10, - page_size: 50, - } - } - - pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { - self.max_pages = max_pages.max(1); - // Google Drive caps `pageSize` at 1000. - self.page_size = page_size.clamp(1, 1000); - self - } -} - -#[async_trait] -impl SyncPipeline for GoogleDriveSyncPipeline { - fn id(&self) -> &str { - "composio:googledrive" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for GoogleDriveSyncPipeline { - fn toolkit(&self) -> &'static str { - "googledrive" - } - fn action(&self) -> &'static str { - ACTION_FIND_FILE - } - fn max_pages(&self) -> usize { - self.max_pages - } - // NB: `stop_on_empty_pending` stays at its default (false) — see the note on - // the Calendar pipeline. The cursor only advances on a complete sync, so a - // capped run must not stop early on an all-deduplicated first page or it - // would permanently skip the unsynced tail. - fn server_side_depth(&self) -> bool { - true - } - fn arguments( - &self, - _: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - let mut args = serde_json::json!({ - "page_size": self.page_size, - "order_by": "modifiedTime desc", - // Guarantee the fields the cursor/title/dedup depend on come back, - // regardless of the action's default projection. - "fields": "files(id,name,mimeType,modifiedTime),nextPageToken", - }); - if let Some(page) = page { - args["page_token"] = serde_json::json!(page); - } - // Depth window via a Drive `q` clause on modification time. Prefer the - // last-synced cursor, else the configured horizon. The cursor is - // validated as an RFC3339 timestamp before being interpolated into the - // query so a malformed persisted value can never inject into the `q` - // clause — on a bad value we simply omit the depth filter (full scan). - let floor = state - .cursor - .as_deref() - .filter(|cursor| chrono::DateTime::parse_from_rfc3339(cursor).is_ok()) - .map(str::to_owned) - .or_else(|| { - config.sync_depth_days.map(|days| { - (chrono::Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339() - }) - }); - if let Some(floor) = floor { - // `GOOGLEDRIVE_FIND_FILE` names the Drive query parameter `q` (the - // native `files.list` name), not `query` — an unrecognised key would - // be ignored and defeat server-side depth bounding. - args["q"] = serde_json::json!(format!("modifiedTime > '{floor}'")); - } - args - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/files", - "/files", - "/data/data/files", - "/data/items", - "/items", - ], - ), - next: next_page_token(data), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "fileId", "data.fileId"])?; - Some(match self.sort_cursor(item) { - Some(modified) => format!("{id}@{modified}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &["modifiedTime", "data.modifiedTime", "modified_time"], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "fileId", "data.fileId"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str(&item.raw, &["name", "data.name", "title", "data.title"]) - .unwrap_or_else(|| format!("Drive file {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - Ok(document( - "googledrive", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_sheets.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_sheets.rs deleted file mode 100644 index fbc26657..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/google_sheets.rs +++ /dev/null @@ -1,186 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_SEARCH: &str = "GOOGLESHEETS_SEARCH_SPREADSHEETS"; -const ACTION_INFO: &str = "GOOGLESHEETS_GET_SPREADSHEET_INFO"; - -/// Incremental Google Sheets synchronization through Composio. -/// -/// Two-step, document-shaped (like `NotionSyncPipeline`): `GOOGLESHEETS_SEARCH_SPREADSHEETS` -/// enumerates accessible spreadsheets, then `GOOGLESHEETS_GET_SPREADSHEET_INFO` fetches the -/// spreadsheet metadata for each item inside [`IncrementalSource::document`]. -pub struct GoogleSheetsSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl GoogleSheetsSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - // NOTE: SEARCH_SPREADSHEETS' page-token arg name is not pinned by the - // curated catalog, so we do a single-page-per-tick fetch (no page - // token emitted) rather than guessing a pagination scheme. Capped at - // 1 page: since `arguments()` never advances the token, a >1 cap - // would re-fire the identical page-1 request and burn budget slots - // for silently-deduplicated items. - max_pages: 1, - page_size: 25, - } - } -} - -#[async_trait] -impl SyncPipeline for GoogleSheetsSyncPipeline { - fn id(&self) -> &str { - "composio:googlesheets" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for GoogleSheetsSyncPipeline { - fn toolkit(&self) -> &'static str { - "googlesheets" - } - fn action(&self) -> &'static str { - ACTION_SEARCH - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn arguments( - &self, - _: &SyncScope, - _: &PipelineConfig, - _: &SyncState, - _page: Option<&str>, - ) -> Value { - // NOTE: an empty/broad `query` enumerates every accessible spreadsheet; - // `max_results` bounds the batch. Both mirror the underlying Drive - // search parameters. No page token is emitted (see `max_pages`). - serde_json::json!({"query": "", "max_results": self.page_size}) - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/spreadsheets", - "/spreadsheets", - "/data/files", - "/files", - "/data/results", - "/results", - "/data/items", - "/items", - ], - ), - // Bounded fetch: no page token consumed (see `max_pages`). The - // pointers are read defensively should Composio surface one. - next: [ - "/data/nextPageToken", - "/nextPageToken", - "/data/next_page_token", - "/next_page_token", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::trim) - .filter(|token| !token.is_empty()) - .map(str::to_owned), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str( - item, - &["id", "data.id", "spreadsheetId", "data.spreadsheetId"], - )?; - Some(match self.sort_cursor(item) { - Some(modified) => format!("{id}@{modified}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &["modifiedTime", "data.modifiedTime", "modified_time"], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - executor: &dyn ActionExecutor, - state: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str( - &item.raw, - &["id", "data.id", "spreadsheetId", "data.spreadsheetId"], - ) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str( - &item.raw, - &[ - "title", - "data.title", - "properties.title", - "data.properties.title", - "name", - ], - ) - .unwrap_or_else(|| format!("Google Sheet {id}")); - // NOTE: GET_SPREADSHEET_INFO identifies the spreadsheet by a - // "spreadsheet_id" argument (Google's canonical parameter name). - let response = checked_execute( - executor, - ACTION_INFO, - serde_json::json!({"spreadsheet_id": id}), - connection_id, - state, - ) - .await?; - // `response.data` is the already-unwrapped payload; the pointers catch - // any additional Composio wrapping, else we serialize the payload root. - let info = ["/data", "/data/data"] - .iter() - .find_map(|path| response.data.pointer(path)) - .unwrap_or(&response.data); - let content = serde_json::to_string_pretty(info)?; - Ok(document( - "googlesheets", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/linear.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/linear.rs deleted file mode 100644 index bfa08f7c..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/linear.rs +++ /dev/null @@ -1,193 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_USERS: &str = "LINEAR_LIST_LINEAR_USERS"; -const ACTION_ISSUES: &str = "LINEAR_LIST_LINEAR_ISSUES"; - -pub struct LinearSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl LinearSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 20, - page_size: 50, - } - } -} - -#[async_trait] -impl SyncPipeline for LinearSyncPipeline { - fn id(&self) -> &str { - "composio:linear" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for LinearSyncPipeline { - fn toolkit(&self) -> &'static str { - "linear" - } - fn action(&self) -> &'static str { - ACTION_ISSUES - } - fn max_pages(&self) -> usize { - self.max_pages - } - async fn scopes( - &self, - executor: &dyn ActionExecutor, - connection_id: &str, - state: &mut SyncState, - ) -> anyhow::Result> { - let response = checked_execute( - executor, - ACTION_USERS, - serde_json::json!({"isMe": true}), - connection_id, - state, - ) - .await?; - let users = first_array( - &response.data, - &[ - "/data/nodes", - "/nodes", - "/data/data/nodes", - "/data/users/nodes", - ], - ); - let viewer = users.first().unwrap_or(&response.data); - let id = pick_str(viewer, &["id", "data.id"]) - .ok_or_else(|| anyhow::anyhow!("{ACTION_USERS} returned no viewer id"))?; - Ok(vec![SyncScope::named(id, "assignee:me")]) - } - fn arguments( - &self, - scope: &SyncScope, - _: &PipelineConfig, - _: &SyncState, - page: Option<&str>, - ) -> Value { - let mut args = serde_json::json!({"assigneeId": scope.id, "first": self.page_size, "orderBy": "updatedAt"}); - if let Some(page) = page { - args["after"] = serde_json::json!(page); - } - args - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - let items = first_array( - data, - &[ - "/data/nodes", - "/nodes", - "/data/data/nodes", - "/data/issues/nodes", - "/data/results", - "/results", - "/data/items", - "/items", - ], - ); - let page_info = [ - "/data/pageInfo", - "/pageInfo", - "/data/data/pageInfo", - "/data/issues/pageInfo", - ] - .iter() - .find_map(|path| data.pointer(path)); - let next = page_info - .filter(|info| { - info.get("hasNextPage") - .and_then(Value::as_bool) - .unwrap_or(false) - }) - .and_then(|info| info.get("endCursor").and_then(Value::as_str)) - .map(str::to_owned); - PageFetch { items, next } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "identifier", "data.identifier"])?; - Some(match self.sort_cursor(item) { - Some(updated) => format!("{id}@{updated}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &[ - "updatedAt", - "data.updatedAt", - "updated_at", - "data.updated_at", - ], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str( - &item.raw, - &["id", "data.id", "identifier", "data.identifier"], - ) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str( - &item.raw, - &[ - "title", - "data.title", - "name", - "data.name", - "identifier", - "data.identifier", - ], - ) - .unwrap_or_else(|| format!("Linear issue {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - Ok(document( - "linear", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/mod.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/mod.rs deleted file mode 100644 index dc86b2b5..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/mod.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! One pipeline per Composio toolkit. Normalisation lives in -//! `tinymemory-sync`; these drive fetch, budget, and the write path. - -mod clickup; -mod common; -mod github; -mod google_calendar; -mod google_docs; -mod google_drive; -mod google_sheets; -mod linear; -mod notion; -mod outlook; -mod slack; -mod slack_parse; -mod todoist; - -pub use clickup::ClickUpSyncPipeline; -pub use github::GitHubSyncPipeline; -pub use google_calendar::GoogleCalendarSyncPipeline; -pub use google_docs::GoogleDocsSyncPipeline; -pub use google_drive::GoogleDriveSyncPipeline; -pub use google_sheets::GoogleSheetsSyncPipeline; -pub use linear::LinearSyncPipeline; -pub use notion::NotionSyncPipeline; -pub use outlook::OutlookSyncPipeline; -pub use slack::{SlackSearchBackfillPipeline, SlackSyncPipeline}; -pub use todoist::TodoistSyncPipeline; diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/notion.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/notion.rs deleted file mode 100644 index 265f4475..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/notion.rs +++ /dev/null @@ -1,193 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_FETCH: &str = "NOTION_FETCH_DATA"; -const ACTION_MARKDOWN: &str = "NOTION_GET_PAGE_MARKDOWN"; - -pub struct NotionSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl NotionSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 20, - page_size: 25, - } - } -} - -#[async_trait] -impl SyncPipeline for NotionSyncPipeline { - fn id(&self) -> &str { - "composio:notion" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for NotionSyncPipeline { - fn toolkit(&self) -> &'static str { - "notion" - } - fn action(&self) -> &'static str { - ACTION_FETCH - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn arguments( - &self, - _: &SyncScope, - _: &PipelineConfig, - _: &SyncState, - page: Option<&str>, - ) -> Value { - let mut args = serde_json::json!({"page_size": self.page_size, "filter": {"value": "page", "property": "object"}, "sort": {"direction": "descending", "timestamp": "last_edited_time"}}); - if let Some(page) = page { - args["start_cursor"] = serde_json::json!(page); - } - args - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/results", - "/results", - "/data/data/results", - "/data/items", - "/items", - ], - ), - next: [ - "/data/next_cursor", - "/next_cursor", - "/data/data/next_cursor", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::to_owned), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "pageId", "data.pageId"])?; - Some(match self.sort_cursor(item) { - Some(edited) => format!("{id}@{edited}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - pick_str( - item, - &[ - "last_edited_time", - "data.last_edited_time", - "lastEditedTime", - "data.lastEditedTime", - ], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - executor: &dyn ActionExecutor, - state: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "pageId", "data.pageId"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = notion_title(&item.raw).unwrap_or_else(|| format!("Notion page {id}")); - let response = checked_execute( - executor, - ACTION_MARKDOWN, - serde_json::json!({"page_id": id}), - connection_id, - state, - ) - .await?; - let content = [ - "/markdown", - "/data/markdown", - "/data/response_data/markdown", - "/response_data/markdown", - "/data/content", - "/content", - "/text", - "/data/text", - ] - .iter() - .find_map(|path| response.data.pointer(path).and_then(Value::as_str)) - .filter(|value| !value.trim().is_empty()) - .map(str::to_owned) - .unwrap_or(serde_json::to_string_pretty(&item.raw)?); - Ok(document( - "notion", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} - -fn notion_title(page: &Value) -> Option { - let properties = page - .get("properties") - .or_else(|| page.pointer("/data/properties")); - properties - .and_then(Value::as_object) - .and_then(|props| { - props.values().find_map(|property| { - (property.get("type").and_then(Value::as_str) == Some("title")) - .then(|| { - property - .get("title") - .and_then(Value::as_array) - .map(|parts| { - parts - .iter() - .filter_map(|part| { - part.get("plain_text").and_then(Value::as_str) - }) - .collect::>() - .join("") - }) - }) - .flatten() - .filter(|title| !title.is_empty()) - }) - }) - .or_else(|| pick_str(page, &["title", "data.title", "name", "data.name"])) -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/outlook.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/outlook.rs deleted file mode 100644 index bfe0780b..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/outlook.rs +++ /dev/null @@ -1,205 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_LIST_MESSAGES: &str = "OUTLOOK_LIST_MESSAGES"; - -/// Incremental Microsoft Outlook mail synchronization through Composio. -/// -/// Outlook messages carry a stable `id` and a `receivedDateTime` timestamp, so -/// this follows the message-shaped pattern (`GmailSyncPipeline`): a single list -/// action ordered newest-first, a client-visible `receivedDateTime` cursor, and -/// content taken directly from the message payload with no secondary fetch. -pub struct OutlookSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, -} - -impl OutlookSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 10, - page_size: 25, - } - } - - pub fn with_limits(mut self, max_pages: usize, page_size: usize) -> Self { - self.max_pages = max_pages.max(1); - self.page_size = page_size.max(1); - self - } -} - -#[async_trait] -impl SyncPipeline for OutlookSyncPipeline { - fn id(&self) -> &str { - "composio:outlook" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for OutlookSyncPipeline { - fn toolkit(&self) -> &'static str { - "outlook" - } - fn action(&self) -> &'static str { - ACTION_LIST_MESSAGES - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn stop_on_empty_pending(&self) -> bool { - true - } - fn server_side_depth(&self) -> bool { - true - } - fn arguments( - &self, - _: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - // Microsoft Graph list-messages params passed through Composio: `top` - // bounds the page size, `orderby` sorts newest-first by receive time. - let mut args = serde_json::json!({ - "top": self.page_size, - "orderby": "receivedDateTime desc", - }); - if let Some(page) = page { - // Graph paginates via a `$skiptoken`; `extract_page` has already - // reduced the `@odata.nextLink` URL to the bare token. The exact - // Composio arg name for feeding it back is not fully certain — we - // send `skip_token` (the Graph-native name), so a mislabel here - // surfaces as a single-page fetch, not silent data loss. - args["skip_token"] = serde_json::json!(page); - } - // Depth window: prefer the last-synced cursor over the configured - // horizon (same precedence as the Gmail/Calendar pipelines). Graph - // filters server-side via `$filter` on `receivedDateTime`. - if let Some(cursor) = state.cursor.as_deref() { - args["filter"] = serde_json::json!(format!("receivedDateTime ge {cursor}")); - } else if let Some(days) = config.sync_depth_days { - let horizon = (chrono::Utc::now() - chrono::Duration::days(days as i64)).to_rfc3339(); - args["filter"] = serde_json::json!(format!("receivedDateTime ge {horizon}")); - } - args - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &[ - "/data/value", - "/value", - "/data/messages", - "/messages", - "/data/data/value", - "/data/items", - "/items", - ], - ), - next: [ - "/data/@odata.nextLink", - "/@odata.nextLink", - "/data/nextPageToken", - "/nextPageToken", - "/data/skip_token", - "/skip_token", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::trim) - .filter(|token| !token.is_empty()) - .map(normalize_skip_token), - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "messageId", "data.messageId"])?; - Some(match self.sort_cursor(item) { - Some(received) => format!("{id}@{received}"), - None => id, - }) - } - fn sort_cursor(&self, item: &Value) -> Option { - // Only `receivedDateTime` — the same field the `$filter` depth window - // keys on. A `lastModifiedDateTime` fallback would store a cursor in a - // different field than the filter compares, so on the next sync the - // `receivedDateTime ge ` window could skip valid messages. - pick_str( - item, - &[ - "receivedDateTime", - "data.receivedDateTime", - "received_date_time", - ], - ) - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "messageId", "data.messageId"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str(&item.raw, &["subject", "data.subject", "title"]) - .unwrap_or_else(|| format!("Outlook message {id}")); - let content = serde_json::to_string_pretty(&item.raw)?; - Ok(document( - "outlook", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} - -/// Reduce a Graph paging token to the bare `$skiptoken` value. -/// -/// Graph returns `@odata.nextLink` as a full URL -/// (`https://graph.microsoft.com/v1.0/me/messages?$skiptoken=ABC...`). Feeding -/// that whole URL back as the paging arg would not resume pagination, so when -/// the token looks like a URL we extract just the `skiptoken` query value; -/// otherwise (Composio may already surface the bare token) we pass it through. -fn normalize_skip_token(token: &str) -> String { - let lower = token.to_ascii_lowercase(); - if let Some(pos) = lower.find("skiptoken=") { - let value = &token[pos + "skiptoken=".len()..]; - let end = value.find('&').unwrap_or(value.len()); - return value[..end].to_string(); - } - token.to_string() -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack.rs deleted file mode 100644 index 3953d6fb..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack.rs +++ /dev/null @@ -1,467 +0,0 @@ -use std::collections::HashMap; - -use async_trait::async_trait; -use chrono::Utc; -use serde_json::Value; - -use super::common::{checked_execute, document, first_array, pick_str}; -use super::slack_parse::{ - decode_cursors, next_cursor, parse_ts, replace_mentions, search_matches, search_total_pages, -}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_CHANNELS: &str = "SLACK_LIST_CONVERSATIONS"; -const ACTION_HISTORY: &str = "SLACK_FETCH_CONVERSATION_HISTORY"; -const ACTION_SEARCH: &str = "SLACK_SEARCH_MESSAGES"; - -pub struct SlackSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, - page_size: usize, - backfill_days: i64, -} - -pub struct SlackSearchBackfillPipeline { - client: ComposioClient, - connection_id: String, - backfill_days: i64, - max_pages: u32, -} - -impl SlackSearchBackfillPipeline { - pub fn new( - client: ComposioClient, - connection_id: impl Into, - backfill_days: i64, - ) -> Self { - Self { - client, - connection_id: connection_id.into(), - backfill_days: backfill_days.max(1), - max_pages: 50, - } - } -} - -#[async_trait] -impl SyncPipeline for SlackSearchBackfillPipeline { - fn id(&self) -> &str { - "composio:slack:search-backfill" - } - - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - - async fn tick(&self, _: &PipelineConfig, context: &SyncContext) -> anyhow::Result { - let mut state = - SyncState::load(context.state.as_ref(), "slack", &self.connection_id).await?; - if state.budget_exhausted() { - return Ok(SyncOutcome { - note: Some("slack search-backfill skipped: daily budget exhausted".into()), - ..SyncOutcome::default() - }); - } - - // Run the body, then save the state on BOTH paths. `checked_execute` - // records billable requests and provider cost into `state` before it - // returns an error; propagating that error before the save would lose - // the accounting and leave the daily budget unadvanced, so a backfill - // that fails repeatedly could keep calling the search action unbudgeted. - // Same contract `run_incremental_sync` keeps. - let result = self.run_backfill(&mut state, context).await; - state.last_sync_at_ms = Some(Utc::now().timestamp_millis() as u64); - state.save(context.state.as_ref()).await?; - result - } -} - -impl SlackSearchBackfillPipeline { - async fn run_backfill( - &self, - state: &mut SyncState, - context: &SyncContext, - ) -> anyhow::Result { - let directory = SlackSyncPipeline::new(self.client.clone(), self.connection_id.clone()); - let scopes = directory - .scopes(&self.client, &self.connection_id, state) - .await?; - let channels: HashMap<_, _> = scopes - .into_iter() - .map(|scope| (scope.id.clone(), scope)) - .collect(); - let users = channels - .values() - .find_map(|scope| scope.metadata.get("users").and_then(Value::as_object)); - let after = (Utc::now() - chrono::Duration::days(self.backfill_days)) - .format("%Y-%m-%d") - .to_string(); - let mut page = 1u32; - let mut total_pages = 1u32; - let mut stored = 0u32; - - loop { - if state.budget_exhausted() || page > self.max_pages { - break; - } - let response = checked_execute( - &self.client, - ACTION_SEARCH, - serde_json::json!({ - "query": format!("after:{after}"), - "count": 100, - "sort": "timestamp", - "sort_dir": "asc", - "page": page, - }), - &self.connection_id, - state, - ) - .await?; - if page == 1 { - total_pages = search_total_pages(&response.data).min(self.max_pages); - } - let matches = search_matches(&response.data); - let fetched = matches.len(); - for raw in matches { - let Some(ts) = pick_str(&raw, &["ts"]) else { - continue; - }; - if parse_ts(&ts).is_none() { - continue; - } - let Some(text) = pick_str(&raw, &["text"]).filter(|text| !text.trim().is_empty()) - else { - continue; - }; - let Some(channel_id) = pick_str(&raw, &["channel.id", "channel_id"]) else { - continue; - }; - let Some(scope) = channels.get(&channel_id) else { - tracing::warn!(channel_id, "[sync:slack-search] unknown channel skipped"); - continue; - }; - let author_id = - pick_str(&raw, &["user", "bot_id"]).unwrap_or_else(|| "unknown".into()); - let author = users - .and_then(|users| users.get(&author_id)) - .and_then(Value::as_str) - .unwrap_or(&author_id); - let text = replace_mentions(&text, users); - let mut doc = document( - "slack", - &self.connection_id, - &format!("{channel_id}:{ts}"), - format!("Slack {} from {author}", scope.label), - format!("[{ts}] {author}: {text}"), - raw, - ); - doc.metadata["channel_id"] = Value::String(channel_id); - doc.metadata["channel_label"] = Value::String(scope.label.clone()); - context.documents.store(doc).await?; - stored = stored.saturating_add(1); - } - if fetched == 0 || page >= total_pages { - break; - } - page = page.saturating_add(1); - } - - Ok(SyncOutcome { - records_ingested: stored, - more_pending: page < total_pages, - actions_called: state.run_requests, - provider_cost_usd: state.run_provider_cost_usd, - note: Some(format!( - "slack search-backfill: pages={page} records={stored}" - )), - }) - } -} - -impl SlackSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 20, - page_size: 200, - backfill_days: 30, - } - } -} - -#[async_trait] -impl SyncPipeline for SlackSyncPipeline { - fn id(&self) -> &str { - "composio:slack" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for SlackSyncPipeline { - fn toolkit(&self) -> &'static str { - "slack" - } - fn action(&self) -> &'static str { - ACTION_HISTORY - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn per_scope_cursors(&self) -> bool { - true - } - fn server_side_depth(&self) -> bool { - true - } - fn tolerate_scope_errors(&self) -> bool { - true - } - fn retain_dedup_keys(&self) -> bool { - true - } - - fn advance_scope_cursor(&self, state: &mut SyncState, scope: &SyncScope, cursor: &str) { - let mut cursors = decode_cursors(state.cursor.as_deref()); - cursors.insert(scope.id.clone(), cursor.into()); - state.cursor = serde_json::to_string(&cursors).ok(); - } - - async fn scopes( - &self, - executor: &dyn ActionExecutor, - connection_id: &str, - state: &mut SyncState, - ) -> anyhow::Result> { - let users = fetch_users(executor, connection_id, state).await; - let mut cursor: Option = None; - let mut channels = Vec::new(); - for _ in 0..20 { - if state.budget_exhausted() { - break; - } - let mut args = serde_json::json!({"limit": 200, "types": "public_channel,private_channel", "exclude_archived": true}); - if let Some(cursor) = cursor.as_deref() { - args["cursor"] = Value::String(cursor.into()); - } - let response = - checked_execute(executor, ACTION_CHANNELS, args, connection_id, state).await?; - channels.extend(first_array( - &response.data, - &["/data/channels", "/channels", "/data/data/channels"], - )); - cursor = next_cursor(&response.data); - if cursor.is_none() { - break; - } - } - Ok(channels - .into_iter() - .filter_map(|channel| { - let id = pick_str(&channel, &["id", "data.id"])?; - let name = pick_str(&channel, &["name", "data.name"]).unwrap_or_else(|| id.clone()); - let private = channel - .get("is_private") - .and_then(Value::as_bool) - .unwrap_or(false); - let label = if private { - format!("private:{name}") - } else { - format!("#{name}") - }; - Some( - SyncScope::named(id, label).with_metadata(serde_json::json!({ - "channel": channel, - "users": users, - })), - ) - }) - .collect()) - } - - fn arguments( - &self, - scope: &SyncScope, - config: &PipelineConfig, - state: &SyncState, - page: Option<&str>, - ) -> Value { - let cursors = decode_cursors(state.cursor.as_deref()); - let oldest = cursors.get(&scope.id).cloned().unwrap_or_else(|| { - format!( - "{}.000000", - (Utc::now() - - chrono::Duration::days( - config - .sync_depth_days - .map(i64::from) - .unwrap_or(self.backfill_days) - )) - .timestamp() - ) - }); - let mut args = serde_json::json!({"channel": scope.id, "oldest": oldest, "inclusive": false, "limit": self.page_size}); - if let Some(page) = page { - args["cursor"] = Value::String(page.into()); - } - args - } - - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - PageFetch { - items: first_array( - data, - &["/data/messages", "/messages", "/data/data/messages"], - ), - next: next_cursor(data), - } - } - - fn dedup_key(&self, item: &Value) -> Option { - let ts = pick_str(item, &["ts", "data.ts"])?; - parse_ts(&ts)?; - let text = pick_str(item, &["text", "data.text"])?; - (!text.trim().is_empty()).then_some(ts) - } - - fn sort_cursor(&self, item: &Value) -> Option { - pick_str(item, &["ts", "data.ts"]) - } - - async fn document( - &self, - scope: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let ts = pick_str(&item.raw, &["ts", "data.ts"]).unwrap_or(item.dedup_key); - let raw_text = pick_str(&item.raw, &["text", "data.text"]).unwrap_or_default(); - let author_id = pick_str( - &item.raw, - &["user", "data.user", "username", "data.username"], - ) - .unwrap_or_else(|| "unknown".into()); - let users = scope.metadata.get("users").and_then(Value::as_object); - let author = users - .and_then(|users| users.get(&author_id)) - .and_then(Value::as_str) - .unwrap_or(&author_id) - .to_owned(); - let text = replace_mentions(&raw_text, users); - let title = format!("Slack {} from {}", scope.label, author); - let content = format!("[{ts}] {author}: {text}"); - let mut result = document( - "slack", - connection_id, - &format!("{}:{ts}", scope.id), - title, - content, - item.raw, - ); - result.metadata["channel_id"] = Value::String(scope.id.clone()); - result.metadata["channel_label"] = Value::String(scope.label.clone()); - Ok(result) - } -} - -async fn fetch_users( - executor: &dyn ActionExecutor, - connection_id: &str, - state: &mut SyncState, -) -> HashMap { - let mut users = HashMap::new(); - let mut cursor: Option = None; - for page in 0..20 { - if state.budget_exhausted() { - break; - } - let mut arguments = serde_json::json!({"limit": 200}); - if let Some(cursor) = cursor.as_deref() { - arguments["cursor"] = Value::String(cursor.into()); - } - let response = match executor - .execute("SLACK_LIST_ALL_USERS", arguments, Some(connection_id)) - .await - { - Ok(response) => response, - Err(error) => { - if let Some(error) = error.downcast_ref::() { - state.record_requests(error.attempts); - } - tracing::warn!(page, %error, "[sync:slack] user directory fetch failed; using collected users"); - break; - } - }; - state.record_action(response.attempts, response.cost_usd); - if !response.successful { - tracing::warn!( - page, - error = response.error.as_deref().unwrap_or("provider failure"), - "[sync:slack] user directory rejected; using collected users" - ); - break; - } - let members = first_array( - &response.data, - &[ - "/data/members", - "/members", - "/data/users", - "/users", - "/data/data/members", - ], - ); - for member in members { - let Some(id) = pick_str(&member, &["id"]) else { - continue; - }; - if let Some(name) = [ - "profile.display_name", - "profile.real_name", - "real_name", - "name", - "profile.display_name_normalized", - "profile.real_name_normalized", - ] - .iter() - .find_map(|path| pick_str(&member, &[*path])) - { - users.insert(id, name); - } - } - cursor = next_cursor(&response.data); - if cursor.is_none() { - break; - } - } - users -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack_parse.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack_parse.rs deleted file mode 100644 index 2419f60f..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/slack_parse.rs +++ /dev/null @@ -1,91 +0,0 @@ -//! Slack response cursor, mention, and timestamp parsing. - -use std::collections::BTreeMap; -use std::sync::OnceLock; - -use serde_json::Value; - -use super::common::first_array; - -/// Return the cached matcher for Slack `<@USERID>` mentions. -pub(super) fn mention_regex() -> &'static regex::Regex { - static REGEX: OnceLock = OnceLock::new(); - REGEX.get_or_init(|| regex::Regex::new(r"<@(U[A-Z0-9]+)>").expect("Slack mention regex")) -} - -/// Replace Slack mention tokens with resolved display names, falling back to -/// the raw user id when the optional user map has no match. -pub(super) fn replace_mentions( - text: &str, - users: Option<&serde_json::Map>, -) -> String { - mention_regex() - .replace_all(text, |captures: ®ex::Captures<'_>| { - let id = &captures[1]; - let resolved = users - .and_then(|users| users.get(id)) - .and_then(Value::as_str) - .unwrap_or(id); - format!("@{resolved}") - }) - .into_owned() -} - -/// Read the first non-blank next cursor across supported response envelopes. -pub(super) fn next_cursor(data: &Value) -> Option { - [ - "/data/response_metadata/next_cursor", - "/response_metadata/next_cursor", - "/data/next_cursor", - "/next_cursor", - "/data/data/response_metadata/next_cursor", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_str)) - .map(str::trim) - .filter(|cursor| !cursor.is_empty()) - .map(str::to_owned) -} - -/// Extract Slack search matches across legacy and nested response envelopes. -pub(super) fn search_matches(data: &Value) -> Vec { - first_array( - data, - &[ - "/data/messages/matches", - "/messages/matches", - "/data/data/messages/matches", - "/messages", - ], - ) -} - -/// Extract the search page count, defaulting to one when paging is absent. -pub(super) fn search_total_pages(data: &Value) -> u32 { - [ - "/data/messages/paging/pages", - "/messages/paging/pages", - "/data/data/messages/paging/pages", - "/pages", - ] - .iter() - .find_map(|path| data.pointer(path).and_then(Value::as_u64)) - .unwrap_or(1) as u32 -} - -/// Decode persisted per-scope cursors, returning an empty map for absent or -/// malformed JSON so synchronization can restart safely. -pub(super) fn decode_cursors(raw: Option<&str>) -> BTreeMap { - raw.and_then(|raw| serde_json::from_str(raw).ok()) - .unwrap_or_default() -} - -/// Parse Slack's `seconds.fraction` timestamp into numeric components. -/// Missing fractions become zero; malformed numeric components return `None`. -pub(super) fn parse_ts(ts: &str) -> Option<(i64, u64)> { - let mut parts = ts.splitn(2, '.'); - Some(( - parts.next()?.parse().ok()?, - parts.next().unwrap_or("0").parse().ok()?, - )) -} diff --git a/crates/tinymemory-core/src/sync/pipelines/composio/providers/todoist.rs b/crates/tinymemory-core/src/sync/pipelines/composio/providers/todoist.rs deleted file mode 100644 index cbbedc60..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/composio/providers/todoist.rs +++ /dev/null @@ -1,226 +0,0 @@ -use async_trait::async_trait; -use serde_json::Value; - -use super::common::{document, first_array, pick_str}; -use crate::sync::composio::providers::sync_state::SyncState; -use crate::sync::pipelines::composio::{ - run_incremental_sync, ActionExecutor, ComposioClient, IncrementalSource, PageFetch, SyncItem, - SyncScope, -}; -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{ - SkillDocument, SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind, -}; - -const ACTION_GET_ALL_TASKS: &str = "TODOIST_GET_ALL_TASKS"; - -/// Incremental Todoist synchronization through Composio. -/// -/// Todoist tasks are self-contained records (stable id + `created_at` -/// timestamp), so this follows the document-shaped pattern -/// (`LinearSyncPipeline`) rather than the message-shaped one: a single list -/// action, content taken directly from the task payload with no secondary -/// fetch. Todoist's active-tasks endpoint returns a plain array and is not -/// paginated, so there is no server-side incremental filter, and a task carries -/// no modification timestamp. Incremental behavior is therefore driven by the -/// orchestrator's client-side dedup (`synced_ids`) keyed on a payload -/// fingerprint (see [`dedup_key`](Self::dedup_key)) — an unchanged task is -/// skipped, while any edit re-ingests. -pub struct TodoistSyncPipeline { - client: ComposioClient, - connection_id: String, - max_pages: usize, -} - -impl TodoistSyncPipeline { - pub fn new(client: ComposioClient, connection_id: impl Into) -> Self { - Self { - client, - connection_id: connection_id.into(), - max_pages: 1, - } - } - - pub fn with_limits(mut self, max_pages: usize, _page_size: usize) -> Self { - self.max_pages = max_pages.max(1); - // Todoist active-tasks is unpaginated; the sibling `page_size` argument - // is accepted for signature parity but has no effect. - self - } -} - -#[async_trait] -impl SyncPipeline for TodoistSyncPipeline { - fn id(&self) -> &str { - "composio:todoist" - } - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Composio - } - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - run_incremental_sync(self, &self.client, &self.connection_id, config, context).await - } -} - -#[async_trait] -impl IncrementalSource for TodoistSyncPipeline { - fn toolkit(&self) -> &'static str { - "todoist" - } - fn action(&self) -> &'static str { - ACTION_GET_ALL_TASKS - } - fn max_pages(&self) -> usize { - self.max_pages - } - fn stop_on_empty_pending(&self) -> bool { - true - } - fn server_side_depth(&self) -> bool { - false - } - fn arguments( - &self, - _: &SyncScope, - _: &PipelineConfig, - _: &SyncState, - _page: Option<&str>, - ) -> Value { - // Todoist "get all active tasks" needs no required arguments and ignores - // pagination; do not invent a page token. - serde_json::json!({}) - } - fn extract_page(&self, data: &Value, _: Option<&str>) -> PageFetch { - // Todoist's active-tasks response is sometimes the bare task array - // (already unwrapped from the Composio `data` envelope by the client) - // and sometimes wrapped under `tasks`/`items`. Handle the top-level - // array first, then the wrapped shapes. - let items = data.as_array().cloned().unwrap_or_else(|| { - first_array( - data, - &[ - "/data/tasks", - "/tasks", - "/data/items", - "/items", - "/data/data", - ], - ) - }); - PageFetch { - items, - // Todoist active tasks are returned as a single unpaginated array. - next: None, - } - } - fn dedup_key(&self, item: &Value) -> Option { - let id = pick_str(item, &["id", "data.id", "task_id", "data.task_id"])?; - // Todoist tasks have no modification timestamp, so `created_at` (which is - // immutable) would never change and edited tasks would never re-ingest. - // Key on a fingerprint of the task payload instead: any change to - // content/due/project yields a new key and re-ingests, while an - // unchanged task keeps its key and is deduped. - Some(format!("{id}@{}", payload_fingerprint(item))) - } - fn sort_cursor(&self, _item: &Value) -> Option { - // Todoist active tasks have no modification timestamp and the endpoint - // is unpaginated, so there is no meaningful sort cursor. Returning None - // is deliberate: the orchestrator's cursor-boundary short-circuit keys - // on `sort_cursor`, and using the immutable `created_at` would halt the - // scan (and skip re-ingest) for an edited task created before the - // persisted cursor. Freshness is handled entirely by `dedup_key`. - None - } - async fn document( - &self, - _: &SyncScope, - connection_id: &str, - item: SyncItem, - _: &dyn ActionExecutor, - _: &mut SyncState, - ) -> anyhow::Result { - let id = pick_str(&item.raw, &["id", "data.id", "task_id", "data.task_id"]) - .unwrap_or_else(|| item.dedup_key.clone()); - let title = pick_str( - &item.raw, - &["content", "data.content", "title", "data.title"], - ) - .unwrap_or_else(|| format!("Todoist task {id}")); - // A Todoist task's meaningful text is its `content` (title line) plus an - // optional `description`; store that as the document body so retrieval - // embeds the task text, not JSON syntax. Fall back to the raw payload - // only when the task carries no content field. - let content = match pick_str(&item.raw, &["content", "data.content"]) { - Some(text) => match pick_str(&item.raw, &["description", "data.description"]) { - Some(desc) if !desc.trim().is_empty() => format!("{text}\n\n{desc}"), - _ => text, - }, - None => serde_json::to_string_pretty(&item.raw)?, - }; - Ok(document( - "todoist", - connection_id, - &id, - title, - content, - item.raw, - )) - } -} - -/// Stable content fingerprint of a task payload, used as the freshness half of -/// the dedup key. Computed as FNV-1a over a canonical serialization (object keys -/// sorted recursively). The key is **persisted** in `SyncState`, so the hash -/// must be stable across Rust toolchains and independent of `serde_json` map -/// ordering — `DefaultHasher` guarantees neither, and an unstable value would -/// silently re-ingest every task on a toolchain bump. -fn payload_fingerprint(item: &Value) -> u64 { - let mut canonical = String::new(); - write_canonical(item, &mut canonical); - // FNV-1a 64-bit — a fixed, specified algorithm. - let mut hash: u64 = 0xcbf2_9ce4_8422_2325; - for byte in canonical.as_bytes() { - hash ^= u64::from(*byte); - hash = hash.wrapping_mul(0x0000_0100_0000_01b3); - } - hash -} - -/// Serialize `value` with object keys sorted recursively so the byte stream is -/// canonical regardless of map insertion order. -fn write_canonical(value: &Value, out: &mut String) { - match value { - Value::Object(map) => { - out.push('{'); - let mut keys: Vec<&String> = map.keys().collect(); - keys.sort_unstable(); - for (index, key) in keys.iter().enumerate() { - if index > 0 { - out.push(','); - } - out.push_str(&serde_json::to_string(key).unwrap_or_default()); - out.push(':'); - write_canonical(&map[*key], out); - } - out.push('}'); - } - Value::Array(items) => { - out.push('['); - for (index, item) in items.iter().enumerate() { - if index > 0 { - out.push(','); - } - write_canonical(item, out); - } - out.push(']'); - } - other => out.push_str(&other.to_string()), - } -} From 9bb97357b19b3511dcf434f91d56e0fcef5d96ad Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:40:23 +0300 Subject: [PATCH 02/16] feat(sources): return Option from reader_for instead of a composio stub Remove the composio module and change reader_for to return Option>, returning None for SourceKind::Composio. The composio source kind is retained for stored records, but reading requires OAuth credentials that belong in the tinyconnectors crate, not here. Returning Option forces callers to handle the absent reader explicitly rather than discovering a runtime error from a stub. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/sources/readers/mod.rs | 32 +++++++++++++------ 1 file changed, 22 insertions(+), 10 deletions(-) diff --git a/crates/tinymemory-core/src/sources/readers/mod.rs b/crates/tinymemory-core/src/sources/readers/mod.rs index 0836d768..c3ebe1ab 100644 --- a/crates/tinymemory-core/src/sources/readers/mod.rs +++ b/crates/tinymemory-core/src/sources/readers/mod.rs @@ -1,6 +1,5 @@ //! Source reader trait and per-kind implementations. -pub mod composio; pub mod conversation; pub mod folder; pub mod github; @@ -30,15 +29,28 @@ pub trait SourceReader: Send + Sync { ) -> Result; } -/// Get the reader for a given source kind. -pub fn reader_for(kind: &SourceKind) -> Box { +/// Get the reader for a given source kind, if this crate has one. +/// +/// `None` for [`SourceKind::Composio`]. The kind itself stays — records +/// synced from a connected account are still stored, still queried, and still +/// forgotten under it, and removing it would orphan every row already written. +/// What left is the *reading*: an OAuth connector is reached with a credential +/// this crate does not hold and must not, so the host fetches through +/// `tinyconnectors` and hands the records here through +/// [`crate::provider::MemorySourceSink`]. +/// +/// Returning `Option` rather than a stub reader that always errors is +/// deliberate: a caller has to decide what to do about a kind it cannot read, +/// and a stub would let it call and discover the same thing at runtime, once +/// per item. +pub fn reader_for(kind: &SourceKind) -> Option> { match kind { - SourceKind::Composio => Box::new(composio::ComposioReader), - SourceKind::Conversation => Box::new(conversation::ConversationReader), - SourceKind::Folder => Box::new(folder::FolderReader), - SourceKind::GithubRepo => Box::new(github::GithubReader), - SourceKind::TwitterQuery => Box::new(twitter::TwitterReader), - SourceKind::RssFeed => Box::new(rss::RssReader::new()), - SourceKind::WebPage => Box::new(web_page::WebPageReader), + SourceKind::Composio => None, + SourceKind::Conversation => Some(Box::new(conversation::ConversationReader)), + SourceKind::Folder => Some(Box::new(folder::FolderReader)), + SourceKind::GithubRepo => Some(Box::new(github::GithubReader)), + SourceKind::TwitterQuery => Some(Box::new(twitter::TwitterReader)), + SourceKind::RssFeed => Some(Box::new(rss::RssReader::new())), + SourceKind::WebPage => Some(Box::new(web_page::WebPageReader)), } } From c7d610aea0c2ba621e5ce9cfc16699acc8648476 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:40:36 +0300 Subject: [PATCH 03/16] chore: files changed crates/tinymemory-api/src/host/mod.rs,crates/tinymemory-core/src/lib.rs,crates/ Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinymemory-api/src/host/mod.rs | 1 - crates/tinymemory-core/src/lib.rs | 1 - crates/tinymemory-core/src/sync/mod.rs | 1 - crates/tinymemory-core/src/sync/pipelines/mod.rs | 1 - 4 files changed, 4 deletions(-) diff --git a/crates/tinymemory-api/src/host/mod.rs b/crates/tinymemory-api/src/host/mod.rs index e3b713fc..67a38b32 100644 --- a/crates/tinymemory-api/src/host/mod.rs +++ b/crates/tinymemory-api/src/host/mod.rs @@ -41,7 +41,6 @@ //! them and they can go home. pub mod cloud_providers; -pub mod composio; pub mod local_ai; pub mod scheduler_gate; pub mod storage_memory; diff --git a/crates/tinymemory-core/src/lib.rs b/crates/tinymemory-core/src/lib.rs index 885e8df0..d9a428d3 100644 --- a/crates/tinymemory-core/src/lib.rs +++ b/crates/tinymemory-core/src/lib.rs @@ -33,7 +33,6 @@ pub type Config = dyn tinymemory_api::host::MemoryHostConfig; pub mod chat; pub mod chat_host; -pub mod composio_host; pub mod config_loader; pub mod conversations; pub mod diff; diff --git a/crates/tinymemory-core/src/sync/mod.rs b/crates/tinymemory-core/src/sync/mod.rs index 089b03cb..fcef0044 100644 --- a/crates/tinymemory-core/src/sync/mod.rs +++ b/crates/tinymemory-core/src/sync/mod.rs @@ -27,7 +27,6 @@ //! single shape to call; everything else stays local. pub mod audit; -pub mod composio; pub mod mcp; pub mod pipelines; pub mod sync_status; diff --git a/crates/tinymemory-core/src/sync/pipelines/mod.rs b/crates/tinymemory-core/src/sync/pipelines/mod.rs index dcd526f6..a06ca74a 100644 --- a/crates/tinymemory-core/src/sync/pipelines/mod.rs +++ b/crates/tinymemory-core/src/sync/pipelines/mod.rs @@ -11,7 +11,6 @@ //! design). Sources of kind `Composio` route here; tree-coupled source kinds //! still route through the engine seam. -pub mod composio; pub mod dispatcher; pub mod host; pub mod traits; From ad4b8157856cf38286da4b60aecc048de5bc78b0 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:41:35 +0300 Subject: [PATCH 04/16] chore: files changed crates/tinymemory-core/src/store/entities.rs,crates/tinymemory-core/src/store/m Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinymemory-core/src/store/entities.rs | 2 +- crates/tinymemory-core/src/store/identity.rs | 165 +++++++++++++++++++ crates/tinymemory-core/src/store/mod.rs | 1 + 3 files changed, 167 insertions(+), 1 deletion(-) create mode 100644 crates/tinymemory-core/src/store/identity.rs diff --git a/crates/tinymemory-core/src/store/entities.rs b/crates/tinymemory-core/src/store/entities.rs index 5cd375a5..25532a32 100644 --- a/crates/tinymemory-core/src/store/entities.rs +++ b/crates/tinymemory-core/src/store/entities.rs @@ -8,7 +8,7 @@ use crate::engine::backend::store::entity_index::{ use anyhow::Result; use crate::engine::memory_config_from; -use crate::sync::composio::providers::profile::{is_self_identity_any_toolkit, IdentityKind}; +use crate::store::identity::{is_self_identity_any_toolkit, IdentityKind}; use crate::Config; /// Aggregate entity-index row for capability providers. diff --git a/crates/tinymemory-core/src/store/identity.rs b/crates/tinymemory-core/src/store/identity.rs new file mode 100644 index 00000000..5142b236 --- /dev/null +++ b/crates/tinymemory-core/src/store/identity.rs @@ -0,0 +1,165 @@ +//! Which identities on a stored row are the user's own. +//! +//! # Why this is here and not with the connector +//! +//! It reads *this crate's* profile store — the `skill::: +//! ` rows written when a connected account's profile is ingested — and +//! answers a question the memory tree asks while building entity rows: is this +//! email address the user themselves? +//! +//! It lived under the Composio sync tree only because that is what wrote the +//! rows. Writing them is the connector module's job now; reading them was +//! never anything but memory's, so it stays. + +use serde::{Deserialize, Serialize}; + +/// The facet kinds a connected account's profile is stored as. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum IdentityKind { + /// Platform-canonical immutable id — Slack `U123ABC`, Notion UUID. + UserId, + Email, + /// `@`-style screen name, canonicalised without the leading `@`. + Handle, + /// E.164 phone number. + Phone, + /// Human display label. Weak signal — never auto-promotes to is_self. + DisplayName, + /// Not for matching; kept for UI / prompt rendering. + AvatarUrl, + /// Not for matching; kept for UI / prompt rendering. + ProfileUrl, +} + +impl IdentityKind { + pub fn as_str(self) -> &'static str { + match self { + Self::UserId => "user_id", + Self::Email => "email", + Self::Handle => "handle", + Self::Phone => "phone", + Self::DisplayName => "display_name", + Self::AvatarUrl => "avatar_url", + Self::ProfileUrl => "profile_url", + } + } + + pub fn parse(s: &str) -> Option { + Some(match s { + "user_id" => Self::UserId, + "email" => Self::Email, + "handle" => Self::Handle, + "phone" => Self::Phone, + "display_name" => Self::DisplayName, + "avatar_url" => Self::AvatarUrl, + "profile_url" => Self::ProfileUrl, + _ => return None, + }) + } + + /// Confidence the matcher records on the row. Hard kinds auto-promote + /// a chunk to `is_self`; weak kinds require corroboration. + pub fn confidence(self) -> f64 { + match self { + Self::UserId | Self::Phone => 1.00, + Self::Email => 0.95, + Self::Handle => 0.70, + Self::DisplayName => 0.40, + Self::AvatarUrl | Self::ProfileUrl => 0.50, + } + } + + /// True if this kind is a real identity signal worth running through + /// the matcher (vs. UI-only fields). + pub fn is_matchable(self) -> bool { + matches!( + self, + Self::UserId | Self::Email | Self::Handle | Self::Phone | Self::DisplayName + ) + } +} + +/// Canonicalize a raw value for storage and lookup. The same routine runs +/// on the entity side at match time, so equality of canonical forms is the +/// matcher's only test — no `COLLATE NOCASE`, no per-call lowercasing. +pub fn canonicalize(kind: IdentityKind, raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + Some(match kind { + IdentityKind::Email => trimmed.to_lowercase(), + IdentityKind::Handle => trimmed.trim_start_matches('@').to_lowercase(), + IdentityKind::Phone => trimmed + .chars() + .filter(|c| c.is_ascii_digit() || *c == '+') + .collect(), + IdentityKind::DisplayName => trimmed.split_whitespace().collect::>().join(" "), + IdentityKind::UserId | IdentityKind::AvatarUrl | IdentityKind::ProfileUrl => { + trimmed.to_string() + } + }) +} + +pub fn canonicalize(kind: IdentityKind, raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + Some(match kind { + IdentityKind::Email => trimmed.to_lowercase(), + IdentityKind::Handle => trimmed.trim_start_matches('@').to_lowercase(), + IdentityKind::Phone => trimmed + .chars() + .filter(|c| c.is_ascii_digit() || *c == '+') + .collect(), + IdentityKind::DisplayName => trimmed.split_whitespace().collect::>().join(" "), + IdentityKind::UserId | IdentityKind::AvatarUrl | IdentityKind::ProfileUrl => { + trimmed.to_string() + } + }) +} + +/// Cross-toolkit variant — matches against every connected provider's +/// rows of this kind. Used for marking memory-tree entity rows: an email +/// in a Slack message that matches the user's Gmail address is still +/// "me," regardless of which source produced the chunk. +pub fn is_self_identity_any_toolkit(kind: IdentityKind, raw_value: &str) -> bool { + if !kind.is_matchable() { + return false; + } + let Some(canonical) = canonicalize(kind, raw_value) else { + return false; + }; + let Some(client) = crate::global::client_if_ready() else { + return false; + }; + let key_pattern = format!("skill:%:%:{}", kind.as_str()); + client + .profile_store() + .skill_identity_matches(&key_pattern, &canonical) +} + +/// Render a compact section for prompt injection. Skips `user_id` (not +/// human-readable), prefixes `handle` with `@`. + +/// Fold a token to the shape used in a profile-store key. +pub fn normalize_token(raw: &str) -> String { + let mut out = String::with_capacity(raw.len()); + for ch in raw.chars() { + let lower = ch.to_ascii_lowercase(); + if lower.is_ascii_alphanumeric() || lower == '-' || lower == '_' { + out.push(lower); + } else { + out.push('_'); + } + } + out.trim_matches('_').to_string() +} + +/// Fold a connection identifier the same way. +#[must_use] +pub fn normalize_connection_identifier(raw: &str) -> String { + normalize_token(raw) +} diff --git a/crates/tinymemory-core/src/store/mod.rs b/crates/tinymemory-core/src/store/mod.rs index 50580c52..23cce60b 100644 --- a/crates/tinymemory-core/src/store/mod.rs +++ b/crates/tinymemory-core/src/store/mod.rs @@ -27,6 +27,7 @@ pub mod chunks; pub mod content; pub mod entities; +pub mod identity; pub mod kinds; pub mod kv; pub mod namespace_store; From c80e3fb7e6036ee09eaf7be2f452f8aaf1e81f9b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:42:04 +0300 Subject: [PATCH 05/16] chore: files changed crates/tinymemory-core/src/sync/mod.rs,crates/tinymemory-core/src/sync/pipeline Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinymemory-core/src/sync/mod.rs | 1 - .../src/sync/pipelines/dispatcher.rs | 123 --- .../src/sync/pipelines/dispatcher_tests.rs | 206 ----- .../src/sync/pipelines/host.rs | 856 ------------------ .../tinymemory-core/src/sync/pipelines/mod.rs | 16 - .../src/sync/pipelines/traits.rs | 196 ---- .../composio_gmail_non_tinycortex_e2e.rs | 233 ----- 7 files changed, 1631 deletions(-) delete mode 100644 crates/tinymemory-core/src/sync/pipelines/dispatcher.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/dispatcher_tests.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/host.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/mod.rs delete mode 100644 crates/tinymemory-core/src/sync/pipelines/traits.rs delete mode 100644 crates/tinymemory-core/tests/composio_gmail_non_tinycortex_e2e.rs diff --git a/crates/tinymemory-core/src/sync/mod.rs b/crates/tinymemory-core/src/sync/mod.rs index fcef0044..59ea17ac 100644 --- a/crates/tinymemory-core/src/sync/mod.rs +++ b/crates/tinymemory-core/src/sync/mod.rs @@ -28,6 +28,5 @@ pub mod audit; pub mod mcp; -pub mod pipelines; pub mod sync_status; pub mod workspace; diff --git a/crates/tinymemory-core/src/sync/pipelines/dispatcher.rs b/crates/tinymemory-core/src/sync/pipelines/dispatcher.rs deleted file mode 100644 index 4e40554d..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/dispatcher.rs +++ /dev/null @@ -1,123 +0,0 @@ -//! Pipeline registry and fault-isolated synchronization dispatcher. - -use std::collections::BTreeMap; -use std::sync::Arc; - -use serde::{Deserialize, Serialize}; - -use crate::sync::pipelines::traits::PipelineConfig; -use crate::sync::pipelines::traits::{SyncContext, SyncOutcome, SyncPipeline, SyncPipelineKind}; - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SyncRunResult { - pub pipeline_id: String, - pub kind: SyncPipelineKind, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub outcome: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub error: Option, -} - -#[derive(Default)] -pub struct SyncDispatcher { - pipelines: BTreeMap>, -} - -impl SyncDispatcher { - pub fn new() -> Self { - Self::default() - } - - pub fn register(&mut self, pipeline: Arc) -> anyhow::Result<()> { - let id = pipeline.id().trim(); - anyhow::ensure!(!id.is_empty(), "sync pipeline id must not be empty"); - anyhow::ensure!( - !self.pipelines.contains_key(id), - "sync pipeline already registered: {id}" - ); - tracing::debug!( - pipeline_id = id, - kind = pipeline.kind().as_str(), - "[memory_sync:dispatcher] registering pipeline" - ); - self.pipelines.insert(id.to_owned(), pipeline); - Ok(()) - } - - pub fn ids(&self) -> Vec<&str> { - self.pipelines.keys().map(String::as_str).collect() - } - - pub async fn init_all( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> Vec { - let mut results = Vec::with_capacity(self.pipelines.len()); - for (id, pipeline) in &self.pipelines { - tracing::debug!( - pipeline_id = id, - "[memory_sync:dispatcher] initializing pipeline" - ); - let result = pipeline.init(config, context).await; - results.push(SyncRunResult { - pipeline_id: id.clone(), - kind: pipeline.kind(), - outcome: result.as_ref().ok().map(|_| SyncOutcome::default()), - error: result.err().map(|error| error.to_string()), - }); - } - results - } - - pub async fn tick( - &self, - pipeline_id: &str, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result { - let pipeline = self - .pipelines - .get(pipeline_id) - .ok_or_else(|| anyhow::anyhow!("unknown sync pipeline: {pipeline_id}"))?; - tracing::debug!( - pipeline_id, - "[memory_sync:dispatcher] pipeline tick starting" - ); - let outcome = pipeline.tick(config, context).await; - match &outcome { - Ok(outcome) => tracing::debug!( - pipeline_id, - records = outcome.records_ingested, - more_pending = outcome.more_pending, - "[memory_sync:dispatcher] pipeline tick completed" - ), - Err(error) => { - tracing::warn!(pipeline_id, %error, "[memory_sync:dispatcher] pipeline tick failed") - } - } - outcome - } - - pub async fn tick_all( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> Vec { - let mut results = Vec::with_capacity(self.pipelines.len()); - for (id, pipeline) in &self.pipelines { - let result = pipeline.tick(config, context).await; - results.push(SyncRunResult { - pipeline_id: id.clone(), - kind: pipeline.kind(), - outcome: result.as_ref().ok().cloned(), - error: result.err().map(|error| error.to_string()), - }); - } - results - } -} - -#[cfg(test)] -#[path = "dispatcher_tests.rs"] -mod tests; diff --git a/crates/tinymemory-core/src/sync/pipelines/dispatcher_tests.rs b/crates/tinymemory-core/src/sync/pipelines/dispatcher_tests.rs deleted file mode 100644 index a5b40d85..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/dispatcher_tests.rs +++ /dev/null @@ -1,206 +0,0 @@ -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; - -use async_trait::async_trait; - -use super::*; -use crate::sync::composio::providers::sync_state::SyncStateStore; -use crate::sync::pipelines::traits::{SkillDocSink, SkillDocument, SyncEvent, SyncEventSink}; - -struct FakePipeline { - id: &'static str, - fail: bool, - init_fail: bool, -} - -#[async_trait] -impl SyncPipeline for FakePipeline { - fn id(&self) -> &str { - self.id - } - - fn kind(&self) -> SyncPipelineKind { - SyncPipelineKind::Workspace - } - - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - if self.init_fail { - anyhow::bail!("expected init failure") - } - Ok(()) - } - - async fn tick(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result { - if self.fail { - anyhow::bail!("expected failure") - } - Ok(SyncOutcome { - records_ingested: 3, - more_pending: false, - actions_called: 0, - provider_cost_usd: 0.0, - note: None, - }) - } -} - -#[derive(Default)] -struct NoopHost(Mutex>); - -#[async_trait] -impl SkillDocSink for NoopHost { - async fn store(&self, _: SkillDocument) -> anyhow::Result<()> { - Ok(()) - } - - async fn delete(&self, _: &str, _: &str) -> anyhow::Result<()> { - Ok(()) - } -} - -#[async_trait] -impl SyncEventSink for NoopHost { - async fn emit(&self, _: SyncEvent) -> anyhow::Result<()> { - Ok(()) - } -} - -#[async_trait] -impl SyncStateStore for NoopHost { - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - Ok(self - .0 - .lock() - .unwrap() - .get(&format!("{namespace}:{key}")) - .cloned()) - } - - async fn set( - &self, - namespace: &str, - key: &str, - value: &serde_json::Value, - ) -> anyhow::Result<()> { - self.0 - .lock() - .unwrap() - .insert(format!("{namespace}:{key}"), value.clone()); - Ok(()) - } -} - -fn context() -> SyncContext { - let host = Arc::new(NoopHost::default()); - SyncContext { - events: host.clone(), - documents: host.clone(), - state: host, - } -} - -#[tokio::test] -async fn tick_all_is_deterministic_and_isolates_failures() { - let mut dispatcher = SyncDispatcher::new(); - dispatcher - .register(Arc::new(FakePipeline { - id: "z-fail", - fail: true, - init_fail: false, - })) - .unwrap(); - dispatcher - .register(Arc::new(FakePipeline { - id: "a-ok", - fail: false, - init_fail: false, - })) - .unwrap(); - assert_eq!(dispatcher.ids(), vec!["a-ok", "z-fail"]); - assert!(dispatcher - .register(Arc::new(FakePipeline { - id: "a-ok", - fail: false, - init_fail: false, - })) - .is_err()); - let results = dispatcher - .tick_all(&PipelineConfig::default(), &context()) - .await; - assert_eq!(results.len(), 2); - assert_eq!(results[0].outcome.as_ref().unwrap().records_ingested, 3); - assert!(results[1] - .error - .as_deref() - .unwrap() - .contains("expected failure")); -} - -#[tokio::test] -async fn register_rejects_blank_ids_and_tick_reports_unknown_pipeline() { - let mut dispatcher = SyncDispatcher::new(); - assert!(dispatcher - .register(Arc::new(FakePipeline { - id: " ", - fail: false, - init_fail: false, - })) - .is_err()); - let error = dispatcher - .tick("missing", &PipelineConfig::default(), &context()) - .await - .unwrap_err(); - assert!(error.to_string().contains("unknown sync pipeline")); -} - -#[tokio::test] -async fn init_all_and_individual_tick_preserve_success_and_failure_details() { - let mut dispatcher = SyncDispatcher::new(); - dispatcher - .register(Arc::new(FakePipeline { - id: "a-init-fails", - fail: false, - init_fail: true, - })) - .unwrap(); - dispatcher - .register(Arc::new(FakePipeline { - id: "b-ok", - fail: false, - init_fail: false, - })) - .unwrap(); - dispatcher - .register(Arc::new(FakePipeline { - id: "c-tick-fails", - fail: true, - init_fail: false, - })) - .unwrap(); - let config = PipelineConfig::default(); - let context = context(); - - let initialized = dispatcher.init_all(&config, &context).await; - assert!(initialized[0] - .error - .as_deref() - .unwrap() - .contains("expected init failure")); - assert!(initialized[1].outcome.is_some()); - assert_eq!( - dispatcher - .tick("b-ok", &config, &context) - .await - .unwrap() - .records_ingested, - 3 - ); - assert!(dispatcher - .tick("c-tick-fails", &config, &context) - .await - .is_err()); - - let encoded = serde_json::to_value(&initialized[0]).unwrap(); - assert_eq!(encoded["pipeline_id"], "a-init-fails"); - assert!(encoded.get("outcome").is_none()); -} diff --git a/crates/tinymemory-core/src/sync/pipelines/host.rs b/crates/tinymemory-core/src/sync/pipelines/host.rs deleted file mode 100644 index 56c013a8..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/host.rs +++ /dev/null @@ -1,856 +0,0 @@ -//! The host side of the engine-free pipelines (#18 §B1): the sink adapter -//! over [`MemoryClient`](crate::store::MemoryClient), the Composio settings -//! mapping, and the runners the -//! rest of `core/src/sync/` calls. -//! -//! This is the piece §B5's acceptance rests on: a pipeline sees three -//! capabilities — events, documents, state — and every one resolves through -//! [`MemoryClient`](crate::store::MemoryClient), so whatever driver the host -//! bound serves the sync. - -use std::collections::HashMap; -use std::sync::{Arc, Mutex, OnceLock}; - -use async_trait::async_trait; -use tokio::sync::OwnedMutexGuard; - -use crate::store::MemoryClientRef; -use crate::sync::composio::providers::sync_state::SyncStateStore; -use crate::sync::pipelines::composio::{ - ClickUpSyncPipeline, ComposioClient, GitHubSyncPipeline, GmailSyncPipeline, LinearSyncPipeline, - NotionSyncPipeline, SlackSearchBackfillPipeline, SlackSyncPipeline, -}; -use crate::sync::pipelines::dispatcher::SyncDispatcher; -use crate::sync::pipelines::traits::{ - ComposioMode, ComposioSyncConfig, PipelineConfig, SecretString, SkillDocSink, SkillDocument, - SyncContext, SyncEvent, SyncEventSink, SyncOutcome, SyncPipeline, SyncRunError, -}; -use crate::Config; - -/// A failed pipeline run, with whatever usage it burned before failing. -#[derive(Debug)] -pub struct PipelineFailure { - pub message: String, - pub actions_called: u32, - pub provider_cost_usd: f64, -} - -impl std::fmt::Display for PipelineFailure { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(&self.message) - } -} - -impl std::error::Error for PipelineFailure {} - -impl PipelineFailure { - pub fn without_usage(message: impl Into) -> Self { - Self { - message: message.into(), - actions_called: 0, - provider_cost_usd: 0.0, - } - } -} - -/// Adapter giving the pipelines their three capabilities over the bound -/// memory client. The engine's `HostSyncAdapter` remains for the engine's own -/// pipelines; this one exists so a Composio sync never needs the engine. -pub struct PipelineHost { - memory: MemoryClientRef, - config: Option>, -} - -impl PipelineHost { - /// An adapter that also feeds the memory tree after each stored document - /// (parity with the engine adapter's #5473 behaviour). - pub fn new(memory: MemoryClientRef, config: Arc) -> Self { - Self { - memory, - config: Some(config), - } - } - - /// An adapter with no host config: documents are stored, tree ingest is - /// skipped. This is the shape a non-TinyCortex host uses. - pub fn without_tree_ingest(memory: MemoryClientRef) -> Self { - Self { - memory, - config: None, - } - } - - /// The pipeline context over this adapter. - pub fn context(self: &Arc) -> SyncContext { - SyncContext { - events: self.clone(), - documents: self.clone(), - state: self.clone(), - } - } -} - -#[async_trait] -impl SkillDocSink for PipelineHost { - async fn store(&self, document: SkillDocument) -> anyhow::Result<()> { - tracing::debug!( - toolkit = %document.toolkit, - connection_id = %document.connection_id, - document_id = %document.document_id, - "[memory_sync] storing synchronized document" - ); - self.memory - .store_skill_sync( - &document.namespace_skill_id, - &document.connection_id, - &document.title, - &document.content, - Some("tinycortex-sync".into()), - Some(document.metadata.clone()), - Some("medium".into()), - None, - None, - Some(document.document_id.clone()), - ) - .await - .map_err(anyhow::Error::msg)?; - - // #5473: additively reconnect the synced item to the memory tree — a - // best-effort secondary index; the skill store above is the source of - // truth and has committed. A failure here must NOT abort the sync (one - // poisonous item would stall the connection and re-buy the page on - // every retry). The config-less adapter skips tree ingest entirely. - if let Some(config) = self.config.as_deref() { - if let Err(error) = ingest_into_tree(config, &document).await { - tracing::warn!( - %error, - document_id = %document.document_id, - "[memory_sync] tree ingest failed; skill store remains authoritative" - ); - } - } - Ok(()) - } - - async fn delete(&self, namespace_skill_id: &str, document_id: &str) -> anyhow::Result<()> { - let namespace = format!("skill-{}", namespace_skill_id.trim()); - tracing::debug!( - namespace, - document_id, - "[memory_sync] deleting synchronized document" - ); - self.memory - .delete_document(&namespace, document_id) - .await - .map(|_| ()) - .map_err(anyhow::Error::msg) - } -} - -/// Mirror of the engine adapter's tree reconnect (`engine::sync`'s -/// `ingest_document_into_memory_tree`): route the stored document through -/// core's ingest funnel under the same addressing scheme. -/// -/// # The scheme is the contract, not an implementation detail -/// -/// The tree scope a chunk seals under is `path_scope`, falling back to -/// `source_id`. Retrieval selects source trees by that scope and classifies -/// them by their **platform prefix** — `gmail:` is email, `slack:` is chat. -/// So the scope has to be `"{toolkit}:{connection_id}"`: one tree per -/// connection, named by a prefix retrieval knows. -/// -/// Passing no `path_scope` is not a smaller version of that. It makes each -/// item's own `source_id` the scope, which is a *tree per document*, named by -/// a prefix that matches no platform — the items are stored and then -/// unreachable, which is the #5473 defect this reconnect exists to fix. The -/// `source_id LIKE` prefix the memory-source status and diff snapshots query -/// by is keyed on this same scheme — see `sources::status::source_id_prefix`. -/// -/// Tags are a deliberate superset of the engine adapter's: it tags the toolkit -/// alone, this also tags `composio_sync`. Tags feed scoring and filtering, not -/// addressing, so the extra one costs nothing and marks the ingest path. -async fn ingest_into_tree(config: &Config, document: &SkillDocument) -> anyhow::Result<()> { - let toolkit = document.toolkit.trim().to_ascii_lowercase(); - let connection_id = document.connection_id.trim(); - // A blank toolkit or connection would yield a scope with no platform - // prefix (`":conn"`, `"gmail:"`), which no retrieval kind matches; skip - // rather than write an unreachable tree. The skill store still holds the - // item. - if toolkit.is_empty() || connection_id.is_empty() { - tracing::debug!( - document_id = %document.document_id, - "[memory_sync] skipping memory-tree ingest: item has no toolkit/connection scope" - ); - return Ok(()); - } - let tree_scope = format!("{toolkit}:{connection_id}"); - let source_id = format!("{tree_scope}:{}", document.document_id); - let owner = format!("{toolkit}-sync:{connection_id}"); - let doc = crate::ingest_pipeline::IngestDocumentInput { - provider: format!("composio:{toolkit}"), - title: document.title.clone(), - body: document.content.clone(), - modified_at: chrono::Utc::now(), - source_ref: Some(document.document_id.clone()), - }; - let tags = vec!["composio_sync".to_string(), toolkit]; - crate::ingest_pipeline::ingest_document_with_scope( - config, - &source_id, - &owner, - tags, - doc, - Some(tree_scope), - ) - .await - .map(|_| ()) - .map_err(|error| anyhow::anyhow!("memory-tree ingest failed for source `{source_id}`: {error}")) -} - -#[async_trait] -impl SyncEventSink for PipelineHost { - async fn emit(&self, event: SyncEvent) -> anyhow::Result<()> { - crate::events::publish(crate::events::MemoryEvent::SyncStageChanged { - trigger: "tinycortex".into(), - stage: super::traits::stage_name(event.stage).into(), - provider: Some(event.toolkit), - connection_id: event.connection_id, - detail: event.message, - source_id: Some(event.source_id), - }); - Ok(()) - } -} - -#[async_trait] -impl SyncStateStore for PipelineHost { - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - self.memory - .kv_get(Some(namespace), key) - .await - .map_err(anyhow::Error::msg) - } - - async fn set( - &self, - namespace: &str, - key: &str, - value: &serde_json::Value, - ) -> anyhow::Result<()> { - self.memory - .kv_set(Some(namespace), key, value) - .await - .map_err(anyhow::Error::msg) - } -} - -/// The Composio connection settings from the host's config — the same -/// resolution the engine seam performs, onto the local types. -pub fn composio_config(config: &Config) -> Result { - if config.composio().mode.eq_ignore_ascii_case("direct") { - let api_key = crate::composio_host::api_key(config) - .or_else(|| config.composio().api_key.clone()) - .ok_or_else(|| "Composio direct API key is not configured".to_string())?; - Ok(ComposioSyncConfig { - mode: ComposioMode::Direct, - base_url: "https://backend.composio.dev/api/v3".into(), - api_key: Some(SecretString::new(api_key)), - bearer_token: None, - entity_id: Some(config.composio().entity_id.clone()), - }) - } else { - let bearer = config - .session_token()? - .ok_or_else(|| "OpenHuman backend bearer token is not configured".to_string())?; - Ok(ComposioSyncConfig { - mode: ComposioMode::Proxied, - base_url: config.effective_backend_api_url(), - api_key: None, - bearer_token: Some(SecretString::new(bearer)), - entity_id: Some(config.composio().entity_id.clone()), - }) - } -} - -/// The toolkits with a native pipeline here. Kept identical to the engine -/// seam's list; `sync_status` advertising draws from the provider registry. -pub fn syncable_composio_toolkits() -> &'static [&'static str] { - &["clickup", "github", "gmail", "linear", "notion", "slack"] -} - -/// Whether `toolkit` has a native pipeline (case-insensitive). -pub fn is_composio_toolkit_syncable(toolkit: &str) -> bool { - let slug = toolkit.trim().to_ascii_lowercase(); - syncable_composio_toolkits().contains(&slug.as_str()) -} - -fn build_composio_pipeline( - toolkit: &str, - connection_id: &str, - composio: ComposioSyncConfig, -) -> Result, String> { - // Fail closed before resolving credentials for any toolkit without a - // native pipeline (#4957) — the gate stays a single testable list. - // - // Normalise once and match on the normalised slug: the gate accepts - // `" Gmail "` (trim + lowercase), so matching on the raw input would let a - // padded or mixed-case toolkit through the gate and into `unreachable!`. - let slug = toolkit.trim().to_ascii_lowercase(); - if !syncable_composio_toolkits().contains(&slug.as_str()) { - return Err(format!("memory sync does not support toolkit '{toolkit}'")); - } - let client = ComposioClient::new(composio); - Ok(match slug.as_str() { - "gmail" => Arc::new(GmailSyncPipeline::new(client, connection_id)), - "github" => Arc::new(GitHubSyncPipeline::new(client, connection_id)), - "notion" => Arc::new(NotionSyncPipeline::new(client, connection_id)), - "linear" => Arc::new(LinearSyncPipeline::new(client, connection_id)), - "clickup" => Arc::new(ClickUpSyncPipeline::new(client, connection_id)), - "slack" => Arc::new(SlackSyncPipeline::new(client, connection_id)), - _ => unreachable!("gated by is_composio_toolkit_syncable"), - }) -} - -/// Run one Composio connection through the engine-free pipelines. -pub async fn run_composio_connection( - toolkit: &str, - connection_id: &str, - config: &Config, - max_items: Option, - sync_depth_days: Option, -) -> Result { - run_composio_connection_with_caps( - toolkit, - connection_id, - config, - SourceCaps { - max_items, - sync_depth_days, - ..SourceCaps::default() - }, - ) - .await -} - -/// The per-source limits a run honours. All `None` = the source's defaults. -#[derive(Clone, Copy, Debug, Default)] -pub struct SourceCaps { - pub max_items: Option, - pub sync_depth_days: Option, - pub max_tokens_per_sync: Option, - pub max_cost_per_sync_usd: Option, -} - -impl SourceCaps { - /// The caps a registry entry carries. - pub fn from_source(source: &tinymemory_sources::MemorySourceEntry) -> Self { - Self { - max_items: source.max_items, - sync_depth_days: source.sync_depth_days, - max_tokens_per_sync: source.max_tokens_per_sync, - max_cost_per_sync_usd: source.max_cost_per_sync_usd, - } - } -} - -/// Run one Composio connection through the engine-free pipelines, honouring -/// every per-source cap. -pub async fn run_composio_connection_with_caps( - toolkit: &str, - connection_id: &str, - config: &Config, - caps: SourceCaps, -) -> Result { - let memory = crate::global::client_if_ready() - .ok_or_else(|| PipelineFailure::without_usage("memory client is not ready"))?; - let composio = composio_config(config).map_err(PipelineFailure::without_usage)?; - let pipeline = build_composio_pipeline(toolkit, connection_id, composio) - .map_err(PipelineFailure::without_usage)?; - let pipeline_config = PipelineConfig { - composio: None, // the client already holds the connection settings - sync_depth_days: caps.sync_depth_days, - max_items: caps.max_items, - max_tokens_per_sync: caps.max_tokens_per_sync, - max_cost_per_sync_usd: caps.max_cost_per_sync_usd, - }; - let host = Arc::new(PipelineHost::new(memory, config.to_arc())); - run_pipeline( - pipeline, - toolkit, - connection_id, - &pipeline_config, - &host.context(), - ) - .await -} - -/// Run a bounded Gmail backfill through the engine-free pipelines. -pub async fn run_gmail_backfill( - connection_id: &str, - query: &str, - max_pages: usize, - page_size: usize, - config: &Config, -) -> Result { - let memory = crate::global::client_if_ready() - .ok_or_else(|| PipelineFailure::without_usage("memory client is not ready"))?; - let composio = composio_config(config).map_err(PipelineFailure::without_usage)?; - let pipeline: Arc = Arc::new( - GmailSyncPipeline::new(ComposioClient::new(composio), connection_id) - .with_limits(max_pages, page_size) - .with_query(query), - ); - let host = Arc::new(PipelineHost::new(memory, config.to_arc())); - // The backfill drives the Gmail pipeline, which keys its `SyncState` on - // `"gmail"`; naming the same toolkit here puts it behind the same guard as - // a periodic or RPC Gmail sync of this connection. - run_pipeline( - pipeline, - "gmail", - connection_id, - &PipelineConfig::default(), - &host.context(), - ) - .await -} - -/// Run the Slack search backfill through the engine-free pipelines. -pub async fn run_slack_search_backfill( - connection_id: &str, - backfill_days: i64, - config: &Config, -) -> Result { - let memory = crate::global::client_if_ready() - .ok_or_else(|| PipelineFailure::without_usage("memory client is not ready"))?; - let composio = composio_config(config).map_err(PipelineFailure::without_usage)?; - let client = ComposioClient::new(composio); - let pipeline: Arc = Arc::new(SlackSearchBackfillPipeline::new( - client, - connection_id, - backfill_days, - )); - let host = Arc::new(PipelineHost::new(memory, config.to_arc())); - // `SlackSearchBackfillPipeline` loads and saves the same - // `("slack", connection_id)` state the Slack sync pipeline does, so the two - // must share one guard or they clobber each other's cursor and budget. - run_pipeline( - pipeline, - "slack", - connection_id, - &PipelineConfig::default(), - &host.context(), - ) - .await -} - -/// The note a run carries when another run already holds its connection. -/// -/// Callers that distinguish "nothing to sync" from "did not sync" match on -/// this rather than on a message they would have to keep in step by hand. -pub const SYNC_ALREADY_RUNNING: &str = "sync already running for this connection"; - -/// One guard per connection, so two runs cannot clobber each other's state. -type ConnectionLock = Arc>; - -/// The process-wide guard table. -/// -/// `run_incremental_sync` loads the connection's `SyncState` once, mutates it -/// in memory for the whole run, and saves at the end; the Slack search -/// backfill does the same over the same `("slack", connection_id)` record. Two -/// runs of one connection therefore race on the cursor, the dedup set and the -/// daily budget, and whichever saves last wins — losing either the dedup set -/// (re-fetch, re-spend) or the budget count (overspend past the cap). The -/// periodic loop, the sync RPC and a trigger can each fire the same -/// connection, so the race is reachable as the code stands. -/// -/// This is the single-process answer, which is how the loop and the RPC paths -/// actually run. An optimistic version stamp on the KV record is what a -/// multi-process host would need instead. -/// -/// The table only ever grows, bounded by the number of connections the host -/// has seen — the same shape, and the same bound, as the periodic scheduler's -/// last-fired map. An entry is one `Arc` and an unlocked mutex. -fn connection_locks() -> &'static Mutex> { - static LOCKS: OnceLock>> = OnceLock::new(); - LOCKS.get_or_init(|| Mutex::new(HashMap::new())) -} - -/// The guard key for a connection. -/// -/// Normalised exactly as [`build_composio_pipeline`] normalises the toolkit -/// gate, so `" Gmail "` and `gmail` name one connection rather than two — and -/// so the Slack sync pipeline and the Slack search backfill, which share one -/// `SyncState` record, share one guard. -fn connection_key(toolkit: &str, connection_id: &str) -> (String, String) { - ( - toolkit.trim().to_ascii_lowercase(), - connection_id.trim().to_owned(), - ) -} - -/// Take the guard for one connection, or `None` if a run already holds it. -/// -/// Deliberately non-blocking. Queueing behind the running sync would stall the -/// periodic loop's whole tick — it walks connections sequentially — and then -/// run a second sync of a connection that has just been synced, which is the -/// Composio spend this guard exists to avoid. -fn try_hold_connection(toolkit: &str, connection_id: &str) -> Option> { - let lock = { - // A panic inside a run cannot corrupt the table: it holds `Arc`s, and - // the async guard is released by its own `Drop`. Recovering from the - // poison keeps one panicking sync from disabling every later one. - let mut locks = connection_locks() - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - Arc::clone( - locks - .entry(connection_key(toolkit, connection_id)) - .or_default(), - ) - }; - lock.try_lock_owned().ok() -} - -async fn run_pipeline( - pipeline: Arc, - toolkit: &str, - connection_id: &str, - config: &PipelineConfig, - context: &SyncContext, -) -> Result { - let Some(_connection) = try_hold_connection(toolkit, connection_id) else { - tracing::debug!( - toolkit, - connection_id, - "[memory_sync] a sync of this connection is already running; skipping" - ); - return Ok(SyncOutcome { - note: Some(SYNC_ALREADY_RUNNING.to_owned()), - ..SyncOutcome::default() - }); - }; - let pipeline_id = pipeline.id().to_owned(); - let mut dispatcher = SyncDispatcher::new(); - dispatcher - .register(pipeline) - .map_err(|error| PipelineFailure::without_usage(error.to_string()))?; - dispatcher - .tick(&pipeline_id, config, context) - .await - .map_err(|error| { - let usage = error.downcast_ref::(); - PipelineFailure { - message: error.to_string(), - actions_called: usage.map_or(0, |error| error.actions_called), - provider_cost_usd: usage.map_or(0.0, |error| error.provider_cost_usd), - } - }) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// #4957: an unsupported toolkit is rejected *before* credentials are - /// resolved — moved here with the gate itself from the engine seam. - #[test] - fn unsupported_toolkit_is_rejected_before_resolving_credentials() { - let err = - build_composio_pipeline("googlecalendar", "conn-1", ComposioSyncConfig::default()) - .err() - .expect("unsupported toolkit must be rejected"); - assert!( - err.contains("does not support toolkit 'googlecalendar'"), - "got: {err}" - ); - } - - #[test] - fn the_syncable_set_is_exactly_the_native_pipelines() { - for toolkit in syncable_composio_toolkits() { - assert!( - build_composio_pipeline(toolkit, "conn-1", ComposioSyncConfig::default()).is_ok(), - "advertised toolkit '{toolkit}' must build" - ); - } - assert!(!is_composio_toolkit_syncable("googlecalendar")); - assert!(is_composio_toolkit_syncable(" Gmail ")); - } - - /// The gate normalises; the build must match on the same normalised - /// slug, or a padded/mixed-case toolkit passes the gate and panics. - #[test] - fn a_padded_or_mixed_case_toolkit_builds_rather_than_panicking() { - for toolkit in [" Gmail ", "GMAIL", "gmail\t", " Slack"] { - assert!( - build_composio_pipeline(toolkit, "conn-1", ComposioSyncConfig::default()).is_ok(), - "{toolkit:?} passes the gate and must build" - ); - } - } - - /// The guard table is process-global and shared by every test in this - /// binary, so each test names connections nothing else touches. - #[test] - fn one_connection_admits_one_run_at_a_time() { - let held = - try_hold_connection("gmail", "guard-single").expect("the first run takes the guard"); - assert!( - try_hold_connection("gmail", "guard-single").is_none(), - "a second run of the same connection must be refused, not queued" - ); - drop(held); - assert!( - try_hold_connection("gmail", "guard-single").is_some(), - "the guard must be released when the run ends" - ); - } - - /// The guard is per connection, not per toolkit: one slow Gmail sync must - /// not stop every other Gmail connection from syncing. - #[test] - fn different_connections_hold_independent_guards() { - let first = try_hold_connection("gmail", "guard-independent-a") - .expect("the first connection takes its guard"); - let second = try_hold_connection("gmail", "guard-independent-b") - .expect("a different connection has its own guard"); - drop((first, second)); - } - - /// `build_composio_pipeline` accepts `" Gmail "` by normalising it. The - /// guard key must normalise identically, or a padded toolkit syncs the - /// same connection concurrently with an unpadded one and they clobber each - /// other's state — the defect the guard exists to prevent. - #[test] - fn the_guard_key_normalises_the_toolkit_like_the_gate() { - assert_eq!( - connection_key(" Gmail ", " conn-1 "), - connection_key("gmail", "conn-1") - ); - let held = try_hold_connection("gmail", "guard-normalised") - .expect("the first run takes the guard"); - assert!( - try_hold_connection(" GMAIL\t", "guard-normalised").is_none(), - "a padded, mixed-case toolkit names the same connection" - ); - drop(held); - } - - /// The Slack sync pipeline and the Slack search backfill load and save the - /// same `("slack", connection_id)` state, so they must contend. - #[test] - fn the_slack_backfill_shares_the_slack_sync_guard() { - assert_eq!( - connection_key("slack", "guard-slack"), - connection_key("Slack", "guard-slack") - ); - let held = try_hold_connection("slack", "guard-slack").expect("the sync takes the guard"); - assert!( - try_hold_connection("slack", "guard-slack").is_none(), - "the backfill must not run while a Slack sync of this connection is running" - ); - drop(held); - } - - /// The engine adapter's tree reconnect has this test - /// (`engine::sync`'s `composio_sync_document_reaches_memory_tree`); the - /// engine-free host that replaced it on the live path did not, and drifted - /// — it wrote a `composio:`-prefixed source id and no `path_scope`, so - /// every synced item became its own tree under a scope no platform prefix - /// matches. Chunks existed, recall could not reach them. Asserting the - /// addressing, not merely the row count, is what catches that. - #[tokio::test] - async fn a_synced_document_is_keyed_by_its_connection_scope() { - use tinymemory_api::host::test_support::TestHostConfig; - use tinymemory_api::host::MemoryHostConfig; - - crate::test_seams::init(); - let workspace = tempfile::tempdir().expect("workspace"); - let workspace_dir = workspace.path().join("workspace"); - let mut host_config = TestHostConfig::default(); - host_config.workspace_dir = workspace_dir.clone(); - let config = host_config.to_arc(); - let client: MemoryClientRef = Arc::new( - crate::store::MemoryClient::from_workspace_dir(workspace_dir) - .expect("memory client initialises against a fresh workspace"), - ); - let host = PipelineHost::new(client, config.clone()); - - // A fresh tree is empty, so a non-zero count after the store is - // attributable to this sync rather than to pre-existing state. - assert_eq!( - crate::store::chunks::store::count_chunks(&*config).expect("count chunks"), - 0, - "fresh workspace must start with an empty memory tree" - ); - - host.store(SkillDocument { - namespace_skill_id: "gmail".into(), - connection_id: "conn-1".into(), - document_id: "gmail:msg-1".into(), - title: "Quarterly planning".into(), - content: "Let's finalise the Q3 roadmap and align on the launch date.".into(), - toolkit: "gmail".into(), - metadata: serde_json::json!({ "source": "composio-provider-incremental" }), - }) - .await - .expect("storing a synced document must also ingest it into the memory tree"); - - let scoped = crate::store::chunks::store::list_chunks( - &*config, - &crate::store::chunks::store::ListChunksQuery { - source_id: Some("gmail:conn-1:gmail:msg-1".into()), - limit: Some(8), - ..Default::default() - }, - ) - .expect("list chunks by source id"); - assert!( - !scoped.is_empty(), - "ingested chunks must be keyed by `{{toolkit}}:{{connection_id}}:{{document_id}}` — \ - the scheme the memory-source status and diff snapshots query by" - ); - assert!( - scoped - .iter() - .all(|chunk| chunk.metadata.path_scope.as_deref() == Some("gmail:conn-1")), - "connector chunks must carry the `{{toolkit}}:{{connection_id}}` tree scope so \ - query_source resolves them (gmail → email)" - ); - assert!( - scoped - .iter() - .all(|chunk| chunk.metadata.owner == "gmail-sync:conn-1"), - "connector chunks must be owned by the connection that synced them" - ); - } - - /// A blank toolkit or connection cannot produce a scope any retrieval kind - /// matches, so the tree half is skipped rather than writing an unreachable - /// tree. The skill store, which committed first, still holds the item. - #[tokio::test] - async fn an_item_without_a_connection_scope_skips_the_tree_but_not_the_store() { - use tinymemory_api::host::test_support::TestHostConfig; - use tinymemory_api::host::MemoryHostConfig; - - crate::test_seams::init(); - let workspace = tempfile::tempdir().expect("workspace"); - let workspace_dir = workspace.path().join("workspace"); - let mut host_config = TestHostConfig::default(); - host_config.workspace_dir = workspace_dir.clone(); - let config = host_config.to_arc(); - let client: MemoryClientRef = Arc::new( - crate::store::MemoryClient::from_workspace_dir(workspace_dir) - .expect("memory client initialises against a fresh workspace"), - ); - let host = PipelineHost::new(client.clone(), config.clone()); - - host.store(SkillDocument { - namespace_skill_id: "gmail".into(), - connection_id: " ".into(), - document_id: "gmail:msg-2".into(), - title: "No connection".into(), - content: "This item has no connection scope.".into(), - toolkit: "gmail".into(), - metadata: serde_json::Value::Null, - }) - .await - .expect("a scopeless item must not fail the sync"); - - assert_eq!( - crate::store::chunks::store::count_chunks(&*config).expect("count chunks"), - 0, - "a scopeless item must not write a tree no retrieval can reach" - ); - let stored = client - .list_documents(Some("skill-gmail")) - .await - .expect("list skill documents"); - let documents = stored - .get("documents") - .and_then(|value| value.as_array()) - .cloned() - .unwrap_or_default(); - assert_eq!( - documents.len(), - 1, - "the skill store is the source of truth and must still hold the item" - ); - } - - /// A pipeline that records whether it was ticked, so the refusal path can - /// be shown to skip the run rather than to run and discard the result. - struct RecordingPipeline(Arc); - - #[async_trait] - impl SyncPipeline for RecordingPipeline { - fn id(&self) -> &str { - "test:recording" - } - - fn kind(&self) -> crate::sync::pipelines::traits::SyncPipelineKind { - crate::sync::pipelines::traits::SyncPipelineKind::Composio - } - - async fn init(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result<()> { - Ok(()) - } - - async fn tick(&self, _: &PipelineConfig, _: &SyncContext) -> anyhow::Result { - self.0.store(true, std::sync::atomic::Ordering::SeqCst); - Ok(SyncOutcome { - records_ingested: 7, - ..SyncOutcome::default() - }) - } - } - - /// End to end: with the connection held, `run_pipeline` returns the note - /// without ticking the pipeline — no fetch, no Composio spend, and no - /// second writer of the connection's `SyncState`. - #[tokio::test] - async fn a_held_connection_short_circuits_the_run() { - crate::test_seams::init(); - let workspace = tempfile::tempdir().expect("workspace"); - let client: MemoryClientRef = Arc::new( - crate::store::MemoryClient::from_workspace_dir(workspace.path().join("store")) - .expect("memory client initialises against a fresh workspace"), - ); - let host = Arc::new(PipelineHost::without_tree_ingest(client)); - let ticked = Arc::new(std::sync::atomic::AtomicBool::new(false)); - - let held = try_hold_connection("gmail", "guard-short-circuit") - .expect("the first run takes the guard"); - let outcome = run_pipeline( - Arc::new(RecordingPipeline(ticked.clone())), - "gmail", - "guard-short-circuit", - &PipelineConfig::default(), - &host.context(), - ) - .await - .expect("a refused run is not a failure"); - - assert_eq!(outcome.note.as_deref(), Some(SYNC_ALREADY_RUNNING)); - assert_eq!(outcome.records_ingested, 0); - assert!( - !ticked.load(std::sync::atomic::Ordering::SeqCst), - "the refused run must not tick the pipeline" - ); - - // Released, the same call runs normally — the guard skips a concurrent - // run, it does not disable the connection. - drop(held); - let outcome = run_pipeline( - Arc::new(RecordingPipeline(ticked.clone())), - "gmail", - "guard-short-circuit", - &PipelineConfig::default(), - &host.context(), - ) - .await - .expect("the run succeeds once the guard is free"); - assert_eq!(outcome.records_ingested, 7); - assert!(ticked.load(std::sync::atomic::Ordering::SeqCst)); - } -} diff --git a/crates/tinymemory-core/src/sync/pipelines/mod.rs b/crates/tinymemory-core/src/sync/pipelines/mod.rs deleted file mode 100644 index a06ca74a..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/mod.rs +++ /dev/null @@ -1,16 +0,0 @@ -//! Engine-neutral sync pipelines (#18 §B1). -//! -//! The Composio orchestration that used to run inside the engine: fetch pages -//! within budget, normalise through `tinymemory-sync`, and write through the -//! [`traits::SyncContext`] sinks. A pipeline sees three capabilities — events, -//! documents, state — and whatever provider the host bound serves them, which -//! is the property §B5's acceptance test needs. -//! -//! The engine keeps its own copies for its internal pipelines (workspace -//! watcher, tree rebuild, repo summarisation — engine-tree features by -//! design). Sources of kind `Composio` route here; tree-coupled source kinds -//! still route through the engine seam. - -pub mod dispatcher; -pub mod host; -pub mod traits; diff --git a/crates/tinymemory-core/src/sync/pipelines/traits.rs b/crates/tinymemory-core/src/sync/pipelines/traits.rs deleted file mode 100644 index a3b76d30..00000000 --- a/crates/tinymemory-core/src/sync/pipelines/traits.rs +++ /dev/null @@ -1,196 +0,0 @@ -//! Host seams and pipeline contracts for live synchronization. - -use std::sync::Arc; - -use async_trait::async_trait; -use serde::{Deserialize, Serialize}; - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SyncPipelineKind { - Composio, - Workspace, - Mcp, -} - -impl SyncPipelineKind { - pub fn as_str(self) -> &'static str { - match self { - Self::Composio => "composio", - Self::Workspace => "workspace", - Self::Mcp => "mcp", - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SyncStage { - Requested, - Fetching, - Stored, - Ingesting, - Completed, - Failed, -} - -/// Stable wire name for each stage, shared by every event adapter. -pub fn stage_name(stage: SyncStage) -> &'static str { - match stage { - SyncStage::Requested => "requested", - SyncStage::Fetching => "fetching", - SyncStage::Stored => "stored", - SyncStage::Ingesting => "ingesting", - SyncStage::Completed => "completed", - SyncStage::Failed => "failed", - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SyncEvent { - pub source_id: String, - pub toolkit: String, - pub connection_id: Option, - pub stage: SyncStage, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub message: Option, -} - -#[async_trait] -pub trait SyncEventSink: Send + Sync { - async fn emit(&self, event: SyncEvent) -> anyhow::Result<()>; -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct SkillDocument { - pub namespace_skill_id: String, - pub connection_id: String, - pub document_id: String, - pub title: String, - pub content: String, - pub toolkit: String, - #[serde(default)] - pub metadata: serde_json::Value, -} - -#[async_trait] -pub trait SkillDocSink: Send + Sync { - async fn store(&self, document: SkillDocument) -> anyhow::Result<()>; - async fn delete(&self, namespace_skill_id: &str, document_id: &str) -> anyhow::Result<()>; -} - -/// How the Composio client reaches the API: straight at it, or through the -/// backend proxy. The engine's enum, ported with the client — distinct from -/// `tinymemory_api::host::ComposioMode`, which is the *host seam's* -/// string-typed setting; the seam converts. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] -pub enum ComposioMode { - /// Call api.composio.dev with the host's own key. - Direct, - /// Route through the backend proxy. - #[default] - Proxied, -} - -/// The Composio client's connection settings, owned here so the pipelines -/// take no engine config type (#18 §B1). The engine keeps its own copy for -/// its internal pipelines; the host constructs this one from its own config. -#[derive(Clone, Debug, Default)] -pub struct ComposioSyncConfig { - pub mode: ComposioMode, - pub base_url: String, - pub api_key: Option, - pub bearer_token: Option, - pub entity_id: Option, -} - -/// A string whose `Debug` never prints the value. -#[derive(Clone, Default, PartialEq, Eq)] -pub struct SecretString(String); - -impl SecretString { - pub fn new(value: impl Into) -> Self { - Self(value.into()) - } - - pub fn expose(&self) -> &str { - &self.0 - } - - pub fn is_empty(&self) -> bool { - self.0.trim().is_empty() - } -} - -impl std::fmt::Debug for SecretString { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("SecretString(redacted)") - } -} - -/// What a pipeline may read of the host's configuration: the Composio -/// connection settings and the sync-depth budget. Deliberately not the -/// host's whole config — a pipeline that needs more must argue for the -/// field here. -#[derive(Clone, Debug, Default)] -pub struct PipelineConfig { - pub composio: Option, - pub sync_depth_days: Option, - pub max_items: Option, - /// Stop the run once this many tokens (estimated from stored content) - /// have been ingested. `None` = unbounded. - pub max_tokens_per_sync: Option, - /// Stop the run once the provider has charged this much. `None` = - /// unbounded. - pub max_cost_per_sync_usd: Option, -} - -/// Host capabilities required by sync pipelines. -#[derive(Clone)] -pub struct SyncContext { - pub events: Arc, - pub documents: Arc, - pub state: Arc, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct SyncOutcome { - pub records_ingested: u32, - pub more_pending: bool, - #[serde(default)] - pub actions_called: u32, - #[serde(default)] - pub provider_cost_usd: f64, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub note: Option, -} - -#[derive(Debug, thiserror::Error)] -#[error("{message}")] -pub struct SyncRunError { - pub actions_called: u32, - pub provider_cost_usd: f64, - message: String, -} - -impl SyncRunError { - pub fn new(message: impl Into, actions_called: u32, provider_cost_usd: f64) -> Self { - Self { - actions_called, - provider_cost_usd, - message: message.into(), - } - } -} - -#[async_trait] -pub trait SyncPipeline: Send + Sync { - fn id(&self) -> &str; - fn kind(&self) -> SyncPipelineKind; - async fn init(&self, config: &PipelineConfig, context: &SyncContext) -> anyhow::Result<()>; - async fn tick( - &self, - config: &PipelineConfig, - context: &SyncContext, - ) -> anyhow::Result; -} diff --git a/crates/tinymemory-core/tests/composio_gmail_non_tinycortex_e2e.rs b/crates/tinymemory-core/tests/composio_gmail_non_tinycortex_e2e.rs deleted file mode 100644 index 4488a62a..00000000 --- a/crates/tinymemory-core/tests/composio_gmail_non_tinycortex_e2e.rs +++ /dev/null @@ -1,233 +0,0 @@ -//! Issue #18 §B5 / §E4 — the acceptance test for the sync section: -//! **Composio Gmail sync completes end to end against a driver that is not -//! TinyCortex.** -//! -//! The pieces under test, and what each proves: -//! -//! - A mock Composio (wiremock, loopback-only) serves two pages of -//! `GMAIL_FETCH_EMAILS` — pagination, cursor advance and dedup are real. -//! - The pipeline is `sync::pipelines::composio::GmailSyncPipeline`, run -//! through the real `SyncDispatcher` — the exact production path. -//! - The host is `PipelineHost::without_tree_ingest` over a `MemoryClient` -//! bound to the **namespace store** — the driver #42 (§A3) registers as its -//! own non-TinyCortex class. No engine is initialised, no tree exists, and -//! the pipeline code under `core/src/sync/` names no engine module. The -//! engine's `KvStore` appears only as a storage *library* inside the -//! namespace store's SQLite file — it is not the bound driver, and nothing -//! in the pipeline knows it is there. -//! -//! Offline by construction: the only socket is wiremock's 127.0.0.1 listener. - -use std::sync::Arc; - -use serde_json::json; -use wiremock::matchers::{body_partial_json, method, path}; -use wiremock::{Match, Mock, MockServer, Request, ResponseTemplate}; - -/// Matches the *first* fetch only: an execute body whose arguments carry no -/// `page_token`. `body_partial_json` cannot express absence, and without this -/// the page-1 mount also matches the page-2 request (which still contains -/// `max_results`), serving page 1 twice — dedup then eats the repeats and the -/// test fails honestly but confusingly. -struct NoPageToken; - -impl Match for NoPageToken { - fn matches(&self, request: &Request) -> bool { - serde_json::from_slice::(&request.body) - .map(|body| body["arguments"].get("page_token").is_none()) - .unwrap_or(false) - } -} - -use tinymemory_core::store::MemoryClient; - -/// The one piece of host wiring `MemoryClient` requires: an embedding host. -/// Noop — this test is about the sync path, and recall is not asserted. -#[derive(Debug)] -struct NoopEmbeddingHost; - -impl tinymemory_api::host::EmbeddingHost for NoopEmbeddingHost { - fn resolve_api_key(&self, _provider: &str) -> Option { - None - } - - fn ollama_base_url(&self) -> String { - "http://127.0.0.1:1".into() - } - - fn default_embedding_provider( - &self, - ) -> std::sync::Arc { - std::sync::Arc::new(tinymemory_api::host::NoopEmbedding) - } - - fn create_embedding_provider_with_credentials( - &self, - _provider: &str, - _model: &str, - _dims: usize, - _api_key: &str, - _custom_endpoint: Option<&str>, - ) -> Result, String> { - Ok(Box::new(tinymemory_api::host::NoopEmbedding)) - } - - fn model_supports_dimensions(&self, _model: &str) -> bool { - false - } - - fn cloud_embedding_provider( - &self, - _model: &str, - _dims: usize, - ) -> Result, String> { - Ok(Box::new(tinymemory_api::host::NoopEmbedding)) - } - - fn default_cloud_embedding_model(&self) -> &str { - "noop" - } - - fn default_cloud_embedding_dimensions(&self) -> usize { - 8 - } - - fn ollama_embedding_provider( - &self, - _base_url: &str, - _model: &str, - _dims: usize, - ) -> Result, String> { - Ok(Box::new(tinymemory_api::host::NoopEmbedding)) - } -} -use tinymemory_core::sync::composio::providers::sync_state::{SyncState, KV_NAMESPACE}; -use tinymemory_core::sync::pipelines::composio::ComposioClient; -use tinymemory_core::sync::pipelines::composio::GmailSyncPipeline; -use tinymemory_core::sync::pipelines::dispatcher::SyncDispatcher; -use tinymemory_core::sync::pipelines::host::PipelineHost; -use tinymemory_core::sync::pipelines::traits::{ - ComposioMode, ComposioSyncConfig, PipelineConfig, SecretString, SyncPipeline, -}; - -fn message(id: &str, subject: &str, body_md: &str) -> serde_json::Value { - json!({ - "id": id, - "subject": subject, - "from": "sender@example.com", - "markdown": body_md, - "messageTimestamp": "2026-01-02T03:04:05Z", - }) -} - -#[tokio::test(flavor = "multi_thread")] -async fn composio_gmail_sync_completes_against_the_namespace_driver() { - // ── The mock Composio ──────────────────────────────────────────────── - let server = MockServer::start().await; - - // Page 1: two messages and a cursor. Matched on the *absence* of a page - // token in the arguments, so retries stay deterministic. - Mock::given(method("POST")) - .and(path("/tools/execute/GMAIL_FETCH_EMAILS")) - .and(body_partial_json(json!({"arguments": {"max_results": 25}}))) - .and(NoPageToken) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "successful": true, - "data": { - "messages": [ - message("m1", "First", "hello one"), - message("m2", "Second", "hello two"), - ], - "nextPageToken": "page-2", - } - }))) - .mount(&server) - .await; - - // Page 2: one message, no cursor — the sync must stop here. - Mock::given(method("POST")) - .and(path("/tools/execute/GMAIL_FETCH_EMAILS")) - .and(body_partial_json( - json!({"arguments": {"page_token": "page-2"}}), - )) - .respond_with(ResponseTemplate::new(200).set_body_json(json!({ - "successful": true, - "data": { - "messages": [message("m3", "Third", "hello three")], - } - }))) - .mount(&server) - .await; - - // ── The non-TinyCortex driver ──────────────────────────────────────── - tinymemory_core::embedding_host::set_embedding_host(Arc::new(NoopEmbeddingHost)); - let workspace = tempfile::tempdir().expect("workspace"); - let memory = Arc::new( - MemoryClient::from_workspace_dir(workspace.path().to_path_buf()) - .expect("bind the namespace store"), - ); - - // ── The engine-free pipeline, on the production dispatcher ─────────── - let composio = ComposioSyncConfig { - mode: ComposioMode::Direct, - base_url: server.uri(), - api_key: Some(SecretString::new("test-key")), - bearer_token: None, - entity_id: Some("entity-1".into()), - }; - let pipeline = Arc::new(GmailSyncPipeline::new( - ComposioClient::new(composio), - "conn-1", - )); - let pipeline_id = pipeline.id().to_owned(); - - let host = Arc::new(PipelineHost::without_tree_ingest(memory.clone())); - let mut dispatcher = SyncDispatcher::new(); - dispatcher.register(pipeline).expect("register pipeline"); - let outcome = dispatcher - .tick(&pipeline_id, &PipelineConfig::default(), &host.context()) - .await - .expect("gmail sync must complete"); - - // ── End to end: the outcome ────────────────────────────────────────── - assert_eq!( - outcome.records_ingested, 3, - "all three messages ingest; outcome={outcome:?}" - ); - assert!(!outcome.more_pending, "page 2 carried no cursor"); - - // ── End to end: the documents landed in the bound store ────────────── - let docs = memory - .list_documents(Some("skill-gmail")) - .await - .expect("list synced documents"); - let listed = docs - .as_array() - .or_else(|| docs.get("documents").and_then(|d| d.as_array())) - .map(|a| a.len()) - .unwrap_or_default(); - assert_eq!(listed, 3, "three documents in skill-gmail: {docs}"); - - // ── End to end: the canonical markdown, not raw JSON ───────────────── - let doc = memory - .get_document("skill-gmail", "gmail:m1") - .await - .expect("read gmail:m1") - .expect("gmail:m1 stored"); - assert!( - doc.content.contains("From: sender@example.com") && doc.content.contains("hello one"), - "canonical markdown stored, got: {}", - doc.content - ); - - // ── End to end: cursor + dedup state persisted through the KV seam ─── - let state = SyncState::load(&*host, "gmail", "conn-1") - .await - .expect("load persisted state"); - assert!(state.is_synced("m1") && state.is_synced("m3"), "dedup ids"); - let raw = memory - .kv_get(Some(KV_NAMESPACE), "gmail:conn-1") - .await - .expect("kv read"); - assert!(raw.is_some(), "sync state persisted under {KV_NAMESPACE}"); -} From a5ba9b6576d7ad2ffae82dfec99e1eae1b02c42c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:42:27 +0300 Subject: [PATCH 06/16] refactor(engine): remove composio pipeline and delegate to connector module Composio sources are now handled by the connector module, which holds the credentials needed to reach connected accounts. The engine pipeline no longer attempts to run composio connections directly; instead it returns an error directing callers to the connector path. The standalone composio helper functions and the slack search backfill wrapper have been removed as their logic has moved to the engine-free pipelines. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinymemory-core/src/engine/sync.rs | 162 ++-------------------- 1 file changed, 12 insertions(+), 150 deletions(-) diff --git a/crates/tinymemory-core/src/engine/sync.rs b/crates/tinymemory-core/src/engine/sync.rs index ebe6a3f9..2cc23986 100644 --- a/crates/tinymemory-core/src/engine/sync.rs +++ b/crates/tinymemory-core/src/engine/sync.rs @@ -309,45 +309,19 @@ pub async fn run_source_pipeline( source: &MemorySourceEntry, config: &Config, ) -> Result { - // Composio sources run on the engine-free pipelines (#18 §B1); this seam - // keeps only the tree-coupled kinds (folder/repo/rss/web — they summarise - // into the engine tree by design) and converts at the boundary for its - // OpenHuman-facing callers. + // Composio sources are read by the connector module, not here: reaching a + // connected account needs a credential this crate does not hold and must + // not. The host fetches through `tinyconnectors` and hands the records + // back through `MemorySourceSink::accept_source_items`. + // + // Refused rather than skipped. A pipeline that answered "0 records, no + // error" for a source it never read would advance the caller's cursor past + // items nobody looked at, and report a healthy sync while the user's mail + // stopped arriving. if source.kind == SourceKind::Composio { - let toolkit = source - .toolkit - .as_deref() - .map(str::trim) - .filter(|toolkit| !toolkit.is_empty()) - .ok_or_else(|| SourcePipelineFailure::without_usage("composio source missing toolkit"))? - .to_ascii_lowercase(); - let connection_id = source - .connection_id - .as_deref() - .map(str::trim) - .filter(|connection_id| !connection_id.is_empty()) - .ok_or_else(|| { - SourcePipelineFailure::without_usage("composio source missing connection_id") - })?; - let outcome = crate::sync::pipelines::host::run_composio_connection_with_caps( - &toolkit, - connection_id, - config, - crate::sync::pipelines::host::SourceCaps::from_source(source), - ) - .await - .map_err(|failure| SourcePipelineFailure { - message: failure.message, - actions_called: failure.actions_called, - provider_cost_usd: failure.provider_cost_usd, - })?; - return Ok(SyncOutcome { - records_ingested: outcome.records_ingested, - more_pending: outcome.more_pending, - actions_called: outcome.actions_called, - provider_cost_usd: outcome.provider_cost_usd, - note: outcome.note, - }); + return Err(SourcePipelineFailure::without_usage( + "composio sources are synced through the connector module, not this pipeline", + )); } let memory = crate::global::client_if_ready() @@ -383,121 +357,9 @@ pub async fn run_source_pipeline( }) } -/// Run a Composio connection through tinycortex, preserving any source-level -/// budgets already configured in OpenHuman's registry. -pub async fn run_composio_connection( - toolkit: &str, - connection_id: &str, - config: &Config, -) -> Result { - run_composio_connection_with_budgets(toolkit, connection_id, config, None, None).await -} - -/// Run a Composio connection with request-scoped budget overrides. -/// -/// Provider RPCs carry these values in `ProviderContext`, before a source has -/// necessarily been persisted in the registry. Explicit values therefore take -/// precedence, while `None` preserves the registered/default source budget. -pub async fn run_composio_connection_with_budgets( - toolkit: &str, - connection_id: &str, - config: &Config, - max_items: Option, - sync_depth_days: Option, -) -> Result { - let mut source = crate::sources::decode_memory_sources(config) - .iter() - .find(|source| { - source.kind == SourceKind::Composio - && source.connection_id.as_deref() == Some(connection_id) - }) - .cloned() - .unwrap_or_else(|| { - let (max_items, sync_depth_days) = - crate::sources::memory_sync_defaults_for_toolkit(toolkit); - MemorySourceEntry { - id: format!("composio:{toolkit}:{connection_id}"), - kind: SourceKind::Composio, - label: format!("{toolkit} connection"), - enabled: true, - toolkit: Some(toolkit.to_ascii_lowercase()), - connection_id: Some(connection_id.to_string()), - path: None, - glob: None, - url: None, - branch: None, - paths: Vec::new(), - max_commits: None, - max_issues: None, - max_prs: None, - query: None, - since_days: None, - max_items, - selector: None, - max_tokens_per_sync: None, - max_cost_per_sync_usd: None, - sync_depth_days, - } - }); - - source.max_items = max_items; - source.sync_depth_days = sync_depth_days; - tracing::debug!( - toolkit, - connection_id, - source_id = %source.id, - max_items = ?source.max_items, - sync_depth_days = ?source.sync_depth_days, - "[tinycortex:sync] dispatching Composio connection" - ); - run_source_pipeline(&source, config).await -} -/// Load the persisted Composio sync state, in core's own vocabulary. -/// -/// Was typed with the engine's `SyncState`; the copies share one serde shape -/// and one KV namespace (pinned by tests in -/// `sync::composio::providers::sync_state`), so the retype changes no bytes. -/// Kept in the engine module only because OpenHuman reaches it through the -/// engine shim path. -pub async fn load_composio_sync_state( - toolkit: &str, - connection_id: &str, -) -> anyhow::Result { - let memory = crate::global::client_if_ready() - .ok_or_else(|| anyhow::anyhow!("memory client is not ready"))?; - let host = crate::sync::pipelines::host::PipelineHost::without_tree_ingest(memory); - crate::sync::composio::providers::sync_state::SyncState::load(&host, toolkit, connection_id) - .await -} -pub async fn run_slack_search_backfill( - connection_id: &str, - backfill_days: i64, - config: &Config, -) -> Result { - // Delegates to the engine-free pipelines (#18 §B1); kept here because - // OpenHuman reaches this function through the engine shim path. - let outcome = crate::sync::pipelines::host::run_slack_search_backfill( - connection_id, - backfill_days, - config, - ) - .await - .map_err(|failure| SourcePipelineFailure { - message: failure.message, - actions_called: failure.actions_called, - provider_cost_usd: failure.provider_cost_usd, - })?; - Ok(SyncOutcome { - records_ingested: outcome.records_ingested, - more_pending: outcome.more_pending, - actions_called: outcome.actions_called, - provider_cost_usd: outcome.provider_cost_usd, - note: outcome.note, - }) -} /// Delegates to the engine-free pipelines (#18 §B1); kept because OpenHuman's /// backfill binary reaches it through the engine shim path. From 1e43273cefe69b3359a4588342c8021305994dd6 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:42:45 +0300 Subject: [PATCH 07/16] chore(engine): remove deprecated sync adapter and its tests Remove the `SyncStateStore` implementation for `HostSyncAdapter` that was kept during the transition from engine-owned to core-owned sync state, along with the associated test module. The core now owns sync state directly, making this adapter and its integration tests obsolete. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinymemory-core/src/engine/sync.rs | 483 ---------------------- 1 file changed, 483 deletions(-) diff --git a/crates/tinymemory-core/src/engine/sync.rs b/crates/tinymemory-core/src/engine/sync.rs index 2cc23986..2820636f 100644 --- a/crates/tinymemory-core/src/engine/sync.rs +++ b/crates/tinymemory-core/src/engine/sync.rs @@ -557,34 +557,6 @@ impl SyncStateStore for HostSyncAdapter { } } -/// Core's state seam, on the engine adapter. -/// -/// `SyncState` is core-owned now (#18 §B1a) and its `load`/`save` take -/// core's `SyncStateStore`; OpenHuman pairs that type with this adapter in -/// its integration tests. Same KV calls as the engine-trait impl below — -/// one storage, two trait names during the transition. -#[async_trait] -impl crate::sync::composio::providers::sync_state::SyncStateStore for HostSyncAdapter { - async fn get(&self, namespace: &str, key: &str) -> anyhow::Result> { - self.memory - .kv_get(Some(namespace), key) - .await - .map_err(anyhow::Error::msg) - } - - async fn set( - &self, - namespace: &str, - key: &str, - value: &serde_json::Value, - ) -> anyhow::Result<()> { - self.memory - .kv_set(Some(namespace), key, value) - .await - .map_err(anyhow::Error::msg) - } -} - #[async_trait] impl SyncEventSink for HostSyncAdapter { async fn emit(&self, event: SyncEvent) -> anyhow::Result<()> { @@ -610,458 +582,3 @@ fn stage_name(stage: SyncStage) -> &'static str { SyncStage::Failed => "failed", } } - -#[cfg(test)] -mod tests { - use super::build_pipeline; - use crate::sources::MemorySourceEntry; - use crate::sync::composio::{get_composio_sync_provider, init_default_composio_sync_providers}; - use crate::sync::pipelines::host::{is_composio_toolkit_syncable, syncable_composio_toolkits}; - - /// The advertised set (`memory_sources.supported_toolkits`, sourced from the - /// provider registry) and the syncable set (`build_pipeline`) must not - /// diverge: a toolkit that is advertised but has no pipeline reports ACTIVE - /// and then silently never ingests — the exact defect of #4957. - /// - /// Both directions are asserted against an explicit built-in slug set. The - /// provider registry is process-global and sibling tests register throwaway - /// providers into it without unregistering, so walking it directly would be - /// order-flaky; pinning the built-in set keeps this deterministic. - #[test] - fn advertised_and_syncable_toolkit_sets_cannot_diverge() { - init_default_composio_sync_providers(); - - // Every syncable toolkit must have a registered provider — otherwise it - // could never be advertised or auto-registered in the first place. - for &slug in syncable_composio_toolkits() { - assert!( - get_composio_sync_provider(slug).is_some(), - "syncable toolkit `{slug}` has no registered memory-sync provider" - ); - } - - // Every built-in provider shipped by `init_default_composio_sync_providers` - // must be syncable. This is the #4957 direction: advertising a provider - // that `build_pipeline` rejects is the silent failure we guard against. - // - // We pin the built-in slug set explicitly rather than walking - // `all_composio_sync_providers()`: that registry is process-global and - // sibling tests register throwaway providers into it that they never - // unregister (e.g. `provideronly` in composio/tools_tests.rs, `stub-no-active` - // in composio/identity.rs), so a raw registry walk fails nondeterministically - // depending on test execution order. A new built-in toolkit must be added to - // this list, to `syncable_composio_toolkits`, and to `build_pipeline` together - // — the assert_eq below fails loudly if the first two ever drift apart. - const BUILTIN_SYNC_PROVIDERS: &[&str] = - &["clickup", "github", "gmail", "linear", "notion", "slack"]; - - let mut builtin = BUILTIN_SYNC_PROVIDERS.to_vec(); - builtin.sort_unstable(); - let mut syncable = syncable_composio_toolkits().to_vec(); - syncable.sort_unstable(); - assert_eq!( - builtin, syncable, - "the built-in provider set and syncable set diverged — a provider is \ - advertised without a matching `build_pipeline` arm, or vice versa (#4957)" - ); - - for &slug in BUILTIN_SYNC_PROVIDERS { - assert!( - get_composio_sync_provider(slug).is_some(), - "built-in provider `{slug}` is not registered by \ - init_default_composio_sync_providers" - ); - assert!( - is_composio_toolkit_syncable(slug), - "built-in provider `{slug}` is advertised but has no build_pipeline arm — \ - it would report ACTIVE and silently fail to sync (#4957)" - ); - } - } - - /// Behavioural regression for #4957: an unsupported Composio toolkit is - /// rejected by `build_pipeline` *before* any credential/client resolution. - /// - /// We hand it a default `Config` (no Composio auth configured). If the gate - /// ran AFTER config resolution we would get a config error ("backend bearer - /// token is not configured" / "direct API key is not configured"); instead - /// we must get the unsupported-toolkit error, proving the fail-closed - /// ordering that stops an unsyncable toolkit from ever reaching a pipeline. - #[test] - fn build_pipeline_refuses_composio_sources() { - // `googlecalendar` is a real Composio toolkit with no native pipeline — - // exactly the prod case from #4957. - let source: MemorySourceEntry = serde_json::from_value(serde_json::json!({ - "id": "composio:googlecalendar:conn-1", - "kind": "composio", - "label": "googlecalendar connection", - "toolkit": "googlecalendar", - "connection_id": "conn-1", - })) - .expect("construct composio source"); - - let config = tinymemory_api::host::test_support::TestHostConfig::default(); - let mut memory_config = - tinycortex::memory::config::MemoryConfig::new("/tmp/openhuman-test-ws"); - - // Composio never reaches this seam any more: `run_source_pipeline` - // routes it to the engine-free pipelines (#18 §B1). The seam's job is - // to say so, not to half-build one. - let err = match build_pipeline(&source, &config, &mut memory_config) { - Ok(_) => panic!("the engine seam must refuse composio sources"), - Err(e) => e, - }; - assert!( - err.contains("does not build composio pipelines"), - "expected the composio refusal, got: {err}" - ); - } - - /// Locks the reported prod failures (googlecalendar / googlesheets) as - /// non-syncable, and pins case-insensitive/trimming behaviour. - #[test] - fn is_composio_toolkit_syncable_classifies_known_slugs() { - assert!(!is_composio_toolkit_syncable("googlecalendar")); - assert!(!is_composio_toolkit_syncable("googlesheets")); - assert!(!is_composio_toolkit_syncable("discord")); - assert!(!is_composio_toolkit_syncable("")); - assert!(is_composio_toolkit_syncable("gmail")); - assert!(is_composio_toolkit_syncable("Gmail")); - assert!(is_composio_toolkit_syncable(" slack ")); - } - - /// Regression for #5473: a Composio connector sync must feed the memory tree, - /// not just the `skill-` document store. The TinyCortex migration - /// (#4794) dropped the tree-ingest half, so synced items stopped producing - /// `mem_tree_chunks` rows and fell out of tree-backed recall. This fails if - /// the `SkillDocSink` store path ever stops writing tree chunks again. - #[tokio::test] - async fn composio_sync_document_reaches_memory_tree() { - use crate::store::{MemoryClient, MemoryClientRef}; - use std::sync::Arc; - use tinycortex::memory::sync::{SkillDocSink, SkillDocument}; - use tinymemory_api::host::test_support::TestHostConfig; - use tinymemory_api::host::MemoryHostConfig; - - crate::test_seams::init(); - let workspace = tempfile::tempdir().expect("workspace"); - let workspace_dir = workspace.path().join("workspace"); - - let mut host = TestHostConfig::default(); - host.workspace_dir = workspace_dir.clone(); - let config = host.to_arc(); - - let client: MemoryClientRef = Arc::new( - MemoryClient::from_workspace_dir(workspace_dir) - .expect("memory client initialises against a fresh workspace"), - ); - let adapter = super::HostSyncAdapter::with_config(client, config.clone()); - - // Precondition: a fresh tree is empty, so a post-store non-zero count is - // attributable to the sync path rather than to pre-existing state. - assert_eq!( - crate::store::chunks::store::count_chunks(&*config).expect("count chunks"), - 0, - "fresh workspace must start with an empty memory tree" - ); - - adapter - .store(SkillDocument { - namespace_skill_id: "gmail".into(), - connection_id: "conn-1".into(), - document_id: "gmail:msg-1".into(), - title: "Quarterly planning".into(), - content: "Let's finalise the Q3 roadmap and align on the launch date.".into(), - toolkit: "gmail".into(), - metadata: serde_json::json!({ "source": "composio-provider-incremental" }), - }) - .await - .expect("storing a synced document must also ingest it into the memory tree"); - - let chunks = crate::store::chunks::store::count_chunks(&*config).expect("count chunks"); - assert!( - chunks > 0, - "a Composio sync must add mem_tree_chunks rows for the ingested item (#5473)" - ); - - // The chunk must carry the deterministic per-item source id - // `{toolkit}:{connection_id}:{document_id}`; its `path_scope` - // (`gmail:conn-1`) is what tree retrieval resolves by platform prefix. - // A drift here is the silent "ingests but is never retrievable" trap. - let scoped = crate::store::chunks::store::list_chunks( - &*config, - &tinycortex::memory::chunks::ListChunksQuery { - source_id: Some("gmail:conn-1:gmail:msg-1".into()), - limit: Some(8), - ..Default::default() - }, - ) - .expect("list chunks by source id"); - assert!( - !scoped.is_empty(), - "ingested chunks must be keyed by the deterministic connector source id" - ); - assert!( - scoped - .iter() - .all(|chunk| chunk.metadata.path_scope.as_deref() == Some("gmail:conn-1")), - "connector chunks must carry the `{{toolkit}}:{{connection_id}}` tree scope so \ - query_source resolves them (gmail → email)" - ); - - // Retrievability is the real goal, and L0 chunks alone do NOT imply it: - // `query_source` reads sealed summaries and skips unsealed trees, so - // before a seal the freshly-ingested item is not yet retrievable. - let before = crate::tree::retrieval::query_source( - &*config, - Some("gmail:conn-1"), - None, - None, - None, - 10, - ) - .await - .expect("query_source before seal"); - assert!( - before.hits.is_empty(), - "an unsealed connector tree must not yet be retrievable" - ); - - // Drive the async extract worker to append the leaf, then force-seal the - // buffer (the time-based flush path) so a level-1 summary exists. - crate::queue::drain_until_idle(&*config) - .await - .expect("drain tree jobs"); - crate::tree::tree::flush::flush_stale_buffers( - &*config, - chrono::Duration::zero(), - &crate::tree::tree::bucket_seal::LabelStrategy::Empty, - ) - .await - .expect("force-seal stale buffers"); - - // Now the connector item is retrievable through the same path the - // product uses for tree-backed recall — the property #5473 restores. - let after = crate::tree::retrieval::query_source( - &*config, - Some("gmail:conn-1"), - None, - None, - None, - 10, - ) - .await - .expect("query_source after seal"); - assert!( - !after.hits.is_empty(), - "a sealed connector tree must be retrievable via query_source (#5473)" - ); - } - - /// The tree-ingest half of `store` is best-effort: when - /// `ingest_document_with_scope` fails, `store` must log and still return - /// `Ok(())`, so one deterministically-poisonous item cannot abort the whole - /// connector run and re-fetch the page (Composio spend) on every retry — the - /// #4947 stall that propagating the error re-created (sanil-23's review - /// blocker #2). The skill store runs first and is the source of truth, so it - /// must remain committed. This forces a real ingest failure by pointing the - /// adapter's tree-ingest `config.workspace_dir` under a regular file (so the - /// tree store cannot be created) while the skill-store client keeps a healthy - /// workspace — isolating the failure to the tree half. If `store` ever - /// propagates the ingest error again, the `.expect` on the store call fails. - #[tokio::test] - async fn tree_ingest_failure_is_tolerated_and_skill_store_is_retained() { - use crate::store::{MemoryClient, MemoryClientRef}; - use std::sync::Arc; - use tinycortex::memory::sync::{SkillDocSink, SkillDocument}; - use tinymemory_api::host::test_support::TestHostConfig; - use tinymemory_api::host::MemoryHostConfig; - - crate::test_seams::init(); - let workspace = tempfile::tempdir().expect("workspace"); - - // The skill store (source of truth) gets a healthy workspace … - let client: MemoryClientRef = Arc::new( - MemoryClient::from_workspace_dir(workspace.path().join("skill-store")) - .expect("memory client initialises against a fresh workspace"), - ); - - // … but the tree-ingest config points at a workspace *under* a regular - // file, so `ingest_document_with_scope` cannot create its store and - // returns `Err` (same failure shape as the `fallible_audit_read` guard). - let blocker = workspace.path().join("blocker"); - std::fs::write(&blocker, b"not a directory").expect("write blocker file"); - let mut host = TestHostConfig::default(); - host.workspace_dir = blocker.join("workspace"); - let config = host.to_arc(); - - let adapter = super::HostSyncAdapter::with_config(client.clone(), config.clone()); - let document = SkillDocument { - namespace_skill_id: "gmail".into(), - connection_id: "conn-1".into(), - document_id: "gmail:msg-1".into(), - title: "Quarterly planning".into(), - content: "Let's finalise the Q3 roadmap.".into(), - toolkit: "gmail".into(), - metadata: serde_json::json!({ "source": "composio-provider-incremental" }), - }; - - // Guard against a vacuous test: the tree-ingest half must *genuinely* - // fail under the broken config. If the lever ever stops failing (e.g. - // ingest resolves its store path elsewhere), this fires rather than the - // test silently passing without exercising the tolerance path. - assert!( - adapter - .ingest_document_into_memory_tree(&*config, &document) - .await - .is_err(), - "the broken tree-ingest workspace must make ingest fail" - ); - - // `store` must swallow that tree-ingest failure and still succeed. - adapter - .store(document) - .await - .expect("store must tolerate a memory-tree ingest failure (best-effort tree)"); - - // The skill store, committed before the tree half, still holds the item — - // best-effort tree ingest must never cost the durable skill write. - let skill_docs = client - .list_documents(Some("skill-gmail")) - .await - .expect("list skill-gmail documents"); - let documents = skill_docs - .get("documents") - .and_then(|value| value.as_array()) - .cloned() - .unwrap_or_default(); - assert_eq!( - documents.len(), - 1, - "the skill store must retain the synced document even when tree ingest fails" - ); - let persisted = serde_json::to_string(&documents).expect("serialise skill documents"); - assert!( - persisted.contains("gmail:msg-1"), - "the retained skill document must carry the synced id" - ); - } - - /// The config-less adapter (`sync_context`) has no ingest pipeline and is not - /// on the connector sync path, so it stores the skill document without - /// touching the memory tree. Guards the `None` branch of `store` from - /// regressing into a panic or an accidental (workspace-less) ingest. - #[tokio::test] - async fn config_less_adapter_skips_memory_tree_ingest() { - use crate::store::{MemoryClient, MemoryClientRef}; - use std::sync::Arc; - use tinycortex::memory::sync::{SkillDocSink, SkillDocument}; - use tinymemory_api::host::test_support::TestHostConfig; - use tinymemory_api::host::MemoryHostConfig; - - crate::test_seams::init(); - let workspace = tempfile::tempdir().expect("workspace"); - let workspace_dir = workspace.path().join("workspace"); - - let mut host = TestHostConfig::default(); - host.workspace_dir = workspace_dir.clone(); - let config = host.to_arc(); - - let client: MemoryClientRef = Arc::new( - MemoryClient::from_workspace_dir(workspace_dir) - .expect("memory client initialises against a fresh workspace"), - ); - // `new` leaves `config: None` — the config-less variant. Keep a handle - // to the shared client so we can read the skill store back afterwards. - let store_client = client.clone(); - let adapter = super::HostSyncAdapter::new(client); - - adapter - .store(SkillDocument { - namespace_skill_id: "gmail".into(), - connection_id: "conn-1".into(), - document_id: "gmail:msg-1".into(), - title: "Quarterly planning".into(), - content: "Let's finalise the Q3 roadmap and align on the launch date.".into(), - toolkit: "gmail".into(), - metadata: serde_json::json!({ "source": "composio-provider-incremental" }), - }) - .await - .expect("config-less store must still persist the skill document"); - - // The skill store still receives the document (the always-on half of - // `store`), keyed by its stable document id under `skill-gmail`. - let skill_docs = store_client - .list_documents(Some("skill-gmail")) - .await - .expect("list skill-gmail documents"); - let documents = skill_docs - .get("documents") - .and_then(|value| value.as_array()) - .cloned() - .unwrap_or_default(); - assert_eq!( - documents.len(), - 1, - "config-less store must persist exactly the one synced skill document" - ); - let persisted = serde_json::to_string(&documents).expect("serialise skill documents"); - assert!( - persisted.contains("gmail:msg-1") && persisted.contains("Quarterly planning"), - "the persisted skill document must carry the synced id and title" - ); - - // …but the tree is untouched, because the config-less adapter has no - // ingest pipeline. - assert_eq!( - crate::store::chunks::store::count_chunks(&*config).expect("count chunks"), - 0, - "a config-less adapter must not ingest into the memory tree" - ); - } - - /// The blank-scope guard: an item whose toolkit is empty would form an - /// unreachable `":conn"` tree scope, so `ingest_document_into_memory_tree` - /// skips it — the skill store still receives it, the tree does not. Covers - /// the early-return branch (a valid toolkit yields chunks, as the retrieval - /// test proves; a blank one must not). - #[tokio::test] - async fn blank_scope_item_is_skipped_for_memory_tree_ingest() { - use crate::store::{MemoryClient, MemoryClientRef}; - use std::sync::Arc; - use tinycortex::memory::sync::{SkillDocSink, SkillDocument}; - use tinymemory_api::host::test_support::TestHostConfig; - use tinymemory_api::host::MemoryHostConfig; - - crate::test_seams::init(); - let workspace = tempfile::tempdir().expect("workspace"); - let workspace_dir = workspace.path().join("workspace"); - let mut host = TestHostConfig::default(); - host.workspace_dir = workspace_dir.clone(); - let config = host.to_arc(); - let client: MemoryClientRef = Arc::new( - MemoryClient::from_workspace_dir(workspace_dir).expect("memory client initialises"), - ); - let adapter = super::HostSyncAdapter::with_config(client, config.clone()); - - adapter - .store(SkillDocument { - namespace_skill_id: "gmail".into(), - connection_id: "conn-1".into(), - document_id: "gmail:msg-1".into(), - title: "Quarterly planning".into(), - content: "Let's finalise the Q3 roadmap.".into(), - // Blank after trim — no platform scope can be formed. - toolkit: " ".into(), - metadata: serde_json::json!({}), - }) - .await - .expect("store must still succeed for an item without a tree scope"); - - assert_eq!( - crate::store::chunks::store::count_chunks(&*config).expect("count chunks"), - 0, - "an item without a toolkit/connection scope must be skipped for tree ingest" - ); - } -} From 039c98f09928688c2a8ba8b3022102012bef4032 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:43:05 +0300 Subject: [PATCH 08/16] chore(engine,store): remove duplicate `canonicalize` function and unused sync exports Remove a duplicate definition of the `canonicalize` function in the identity store that was accidentally left after a refactor, and clean up the engine module's public re-exports by dropping `load_composio_sync_state`, `run_composio_connection`, `run_composio_connection_with_budgets`, and `run_slack_search_backfill` which are no longer used. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinymemory-core/src/engine/mod.rs | 5 ++--- crates/tinymemory-core/src/store/identity.rs | 18 ------------------ 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/crates/tinymemory-core/src/engine/mod.rs b/crates/tinymemory-core/src/engine/mod.rs index aec048c9..25c2a733 100644 --- a/crates/tinymemory-core/src/engine/mod.rs +++ b/crates/tinymemory-core/src/engine/mod.rs @@ -65,9 +65,8 @@ pub use seal::{ }; pub use summariser::HostSummariser; pub use sync::{ - estimate_cost_usd, load_composio_sync_state, needs_rebuild, raw_coverage, read_audit_log, - rebuild_tree_from_raw, run_composio_connection, run_composio_connection_with_budgets, - run_github_sync, run_gmail_backfill, run_slack_search_backfill, run_source_pipeline, + estimate_cost_usd, needs_rebuild, raw_coverage, read_audit_log, + rebuild_tree_from_raw, run_github_sync, run_gmail_backfill, run_source_pipeline, sync_context, HostSyncAdapter, RawCoverage, RawFileRef, RealCostAccumulator, RebuildOutcome, SourcePipelineFailure, HOST_SYNC_STATE_NAMESPACE, }; diff --git a/crates/tinymemory-core/src/store/identity.rs b/crates/tinymemory-core/src/store/identity.rs index 5142b236..dd2cc778 100644 --- a/crates/tinymemory-core/src/store/identity.rs +++ b/crates/tinymemory-core/src/store/identity.rs @@ -102,24 +102,6 @@ pub fn canonicalize(kind: IdentityKind, raw: &str) -> Option { }) } -pub fn canonicalize(kind: IdentityKind, raw: &str) -> Option { - let trimmed = raw.trim(); - if trimmed.is_empty() { - return None; - } - Some(match kind { - IdentityKind::Email => trimmed.to_lowercase(), - IdentityKind::Handle => trimmed.trim_start_matches('@').to_lowercase(), - IdentityKind::Phone => trimmed - .chars() - .filter(|c| c.is_ascii_digit() || *c == '+') - .collect(), - IdentityKind::DisplayName => trimmed.split_whitespace().collect::>().join(" "), - IdentityKind::UserId | IdentityKind::AvatarUrl | IdentityKind::ProfileUrl => { - trimmed.to_string() - } - }) -} /// Cross-toolkit variant — matches against every connected provider's /// rows of this kind. Used for marking memory-tree entity rows: an email From f7c4729bcd1508dd83ec53884a1efeee92f1115c Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:43:17 +0300 Subject: [PATCH 09/16] chore(engine): remove unused gmail backfill function The `run_gmail_backfill` function in the sync module was a thin wrapper that delegated to the engine-free pipeline, kept only because OpenHuman's backfill binary reached it through the engine shim path. That binary no longer uses this path, so the function and its re-export are removed to eliminate dead code. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinymemory-core/src/engine/mod.rs | 2 +- crates/tinymemory-core/src/engine/sync.rs | 30 ----------------------- 2 files changed, 1 insertion(+), 31 deletions(-) diff --git a/crates/tinymemory-core/src/engine/mod.rs b/crates/tinymemory-core/src/engine/mod.rs index 25c2a733..a6bf2963 100644 --- a/crates/tinymemory-core/src/engine/mod.rs +++ b/crates/tinymemory-core/src/engine/mod.rs @@ -66,7 +66,7 @@ pub use seal::{ pub use summariser::HostSummariser; pub use sync::{ estimate_cost_usd, needs_rebuild, raw_coverage, read_audit_log, - rebuild_tree_from_raw, run_github_sync, run_gmail_backfill, run_source_pipeline, + rebuild_tree_from_raw, run_github_sync, run_source_pipeline, sync_context, HostSyncAdapter, RawCoverage, RawFileRef, RealCostAccumulator, RebuildOutcome, SourcePipelineFailure, HOST_SYNC_STATE_NAMESPACE, }; diff --git a/crates/tinymemory-core/src/engine/sync.rs b/crates/tinymemory-core/src/engine/sync.rs index 2820636f..f0099144 100644 --- a/crates/tinymemory-core/src/engine/sync.rs +++ b/crates/tinymemory-core/src/engine/sync.rs @@ -361,36 +361,6 @@ pub async fn run_source_pipeline( -/// Delegates to the engine-free pipelines (#18 §B1); kept because OpenHuman's -/// backfill binary reaches it through the engine shim path. -pub async fn run_gmail_backfill( - connection_id: &str, - query: &str, - max_pages: usize, - page_size: usize, - config: &Config, -) -> Result { - let outcome = crate::sync::pipelines::host::run_gmail_backfill( - connection_id, - query, - max_pages, - page_size, - config, - ) - .await - .map_err(|failure| SourcePipelineFailure { - message: failure.message, - actions_called: failure.actions_called, - provider_cost_usd: failure.provider_cost_usd, - })?; - Ok(SyncOutcome { - records_ingested: outcome.records_ingested, - more_pending: outcome.more_pending, - actions_called: outcome.actions_called, - provider_cost_usd: outcome.provider_cost_usd, - note: outcome.note, - }) -} fn build_pipeline( source: &MemorySourceEntry, From e1d433b4d7b7d3323433b998bda1baec02eb139e Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:44:02 +0300 Subject: [PATCH 10/16] feat(workspace): expose cadence module for periodic sync Extract the cadence-related functions into a dedicated `cadence` module and re-export them from the workspace module, so that periodic sync logic can reference them through the workspace path instead of the composio module. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../src/sync/workspace/cadence.rs | 81 +++++++++++++++++++ .../tinymemory-core/src/sync/workspace/mod.rs | 1 + .../src/sync/workspace/periodic.rs | 2 +- 3 files changed, 83 insertions(+), 1 deletion(-) create mode 100644 crates/tinymemory-core/src/sync/workspace/cadence.rs diff --git a/crates/tinymemory-core/src/sync/workspace/cadence.rs b/crates/tinymemory-core/src/sync/workspace/cadence.rs new file mode 100644 index 00000000..af9b558f --- /dev/null +++ b/crates/tinymemory-core/src/sync/workspace/cadence.rs @@ -0,0 +1,81 @@ +//! When the periodic scheduler should fire, and when it should hold off. +//! +//! # Why this is not with the connector +//! +//! None of it is specific to any source. "How often may this sync run" and +//! "should the scheduler run at all right now" are questions about the user's +//! cadence setting and the machine's pause policy, and the answers are the +//! same whether the source is a mailbox, a folder, or an RSS feed. +//! +//! It lived under the Composio tree because Composio was the first source +//! with a periodic loop. The loop is still here; only the fetching left. + +use std::time::Duration; + +use crate::scheduler_gate::{current_policy, PauseReason}; +use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; + +/// Resolve the effective periodic sync interval (seconds) for one connection, +/// combining the provider's own default with the user's global +/// memory-sync cadence ([`Config::memory_sync_interval_secs`], #3302). +/// +/// - `global == Some(0)` → `None`: "Manual only" — the scheduler skips this +/// source entirely (manual sync still works). +/// - `global == Some(n)` → `Some(max(n, provider_default))`: the user's +/// cadence overrides the provider default but is floored at it, so we never +/// sync *more* often than the provider intended. +/// - `global == None` → `Some(max(DEFAULT, provider_default))`: no explicit +/// user choice, so fall back to the 24h default cadence (also floored at the +/// provider default). +pub(crate) fn effective_interval_secs(provider_default: u64, global: Option) -> Option { + match global { + Some(0) => None, + Some(n) => Some(n.max(provider_default)), + None => Some(DEFAULT_MEMORY_SYNC_INTERVAL_SECS.max(provider_default)), + } +} + +/// Decide whether a connection is due for a periodic sync right now, given the +/// effective interval and how long ago it last synced this run. +/// +/// `since_last_sync == None` means we have no record of a sync this process +/// lifetime, so we fire immediately (the restart-recovery path). Kept pure so +/// the due-check can be simulated without driving the real `Instant` clock. +pub(crate) fn connection_is_due(interval_secs: u64, since_last_sync: Option) -> bool { + match since_last_sync { + Some(elapsed) => elapsed >= Duration::from_secs(interval_secs), + None => true, + } +} + +/// Inspect the scheduler-gate policy and decide whether this tick should +/// fire at all. Returns `Some(reason)` for paused states so the caller can +/// log a single, attributable line instead of doing the work and discovering +/// per-LLM-call later that everything's gated. +/// +/// Covers two reasons the memory subsystem treats as "do no background +/// work": +/// - [`PauseReason::UserDisabled`] — user flipped the Memory Tree toggle off +/// in Settings (#1856 Part 1). The 20-min Composio fetch loop honouring +/// this flag is the explicit follow-up listed in the #2719 PR body. +/// - [`PauseReason::SignedOut`] — no live session; periodic work would just +/// 401-loop against the backend. +/// +/// Other [`PauseReason`] variants: +/// - `OnBattery` / `CpuPressure` (future, per #1073) — intentionally **not** +/// gated here; periodic Composio fetch is network-light, so battery / CPU +/// pressure shouldn't stop the user's data flowing in. Those signals +/// already throttle LLM-bound work through the regular gate. +/// - `Unknown` — documented in `scheduler_gate::policy` as a safe fallback; +/// `Policy::pause_reason()` returns it only when the gate state is in a +/// transitional / not-yet-resolved condition. Letting the tick proceed +/// here keeps periodic sync running through brief transitions instead of +/// pausing on stale unresolved state. +pub(crate) fn periodic_pause_reason() -> Option { + // Delegate the `Policy::Paused { .. }` → `PauseReason` extraction to + // the existing `Policy::pause_reason()` helper (avoids re-implementing + // the same destructure twice). The allow-list below is the only thing + // this site has to own — future `PauseReason` variants stay opt-in. + let reason = current_policy().pause_reason()?; + matches!(reason, PauseReason::UserDisabled | PauseReason::SignedOut).then_some(reason) +} diff --git a/crates/tinymemory-core/src/sync/workspace/mod.rs b/crates/tinymemory-core/src/sync/workspace/mod.rs index a21d477d..d7e868d1 100644 --- a/crates/tinymemory-core/src/sync/workspace/mod.rs +++ b/crates/tinymemory-core/src/sync/workspace/mod.rs @@ -20,6 +20,7 @@ //! workspace-kind memory sources (GitHub repos, folders, RSS, web pages) //! syncing without manual "Sync now" clicks. +pub mod cadence; pub mod periodic; pub use periodic::start_workspace_periodic_sync; diff --git a/crates/tinymemory-core/src/sync/workspace/periodic.rs b/crates/tinymemory-core/src/sync/workspace/periodic.rs index a02470a8..2c18fc7a 100644 --- a/crates/tinymemory-core/src/sync/workspace/periodic.rs +++ b/crates/tinymemory-core/src/sync/workspace/periodic.rs @@ -37,7 +37,7 @@ use crate::scheduler_gate::resume_notify; use crate::sources::sync::sync_source; use crate::sources::types::{MemorySourceEntry, SourceKind}; use crate::sync::audit::{read_audit_log, SyncAuditEntry}; -use crate::sync::composio::periodic::{ +use crate::sync::workspace::cadence::{ connection_is_due, effective_interval_secs, periodic_pause_reason, }; use tinymemory_api::host::DEFAULT_MEMORY_SYNC_INTERVAL_SECS; From 293eed0d403f4a2a6c288f3e1b47cc55fe81d33d Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:44:43 +0300 Subject: [PATCH 11/16] chore: files changed crates/tinymemory-core/src/sources/reconcile.rs,crates/tinymemory-core/src/sour Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinymemory-core/src/sources/reconcile.rs | 97 ------------------- crates/tinymemory-core/src/sources/sync.rs | 4 +- crates/tinymemory-core/src/sync/mod.rs | 1 + crates/tinymemory-core/src/sync/usage.rs | 23 +++++ 4 files changed, 26 insertions(+), 99 deletions(-) create mode 100644 crates/tinymemory-core/src/sync/usage.rs diff --git a/crates/tinymemory-core/src/sources/reconcile.rs b/crates/tinymemory-core/src/sources/reconcile.rs index 6f521a98..637aa316 100644 --- a/crates/tinymemory-core/src/sources/reconcile.rs +++ b/crates/tinymemory-core/src/sources/reconcile.rs @@ -11,110 +11,13 @@ use crate::config_loader as config_rpc; use crate::sources::registry; use crate::sources::types::{MemorySourceEntry, SourceKind}; -use crate::sync::composio; use std::collections::HashSet; /// Current version of the caps migration. Bump when the migration logic changes /// so installs that ran an earlier revision re-run it exactly once. const CURRENT_CAPS_MIGRATION_VERSION: u32 = 1; -/// Reconcile active Composio connections into the memory sources registry and -/// return the live active-connection set scanned this call. -/// -/// Returns `Some(connection_ids)` — the `connection_id`s of every active sync -/// target — when the live Composio scan **succeeded**, so callers (notably -/// `rpc::list_rpc`) can filter the listing down to connections that are still -/// active and dedupe identical rows. Returns `None` when the scan could not run -/// (config load / network / auth failure); callers must treat `None` as "active -/// set unavailable" and **not** hide any sources — an empty scan from a transient -/// blip must never be read as "everything is inactive". -pub async fn ensure_composio_sources() -> Option> { - tracing::debug!("[memory_sources:reconcile] starting composio reconciliation"); - - let config = match config_rpc::load_config_with_timeout().await { - Ok(c) => c, - Err(e) => { - tracing::warn!( - error = %e, - "[memory_sources:reconcile] failed to load config; skipping" - ); - return None; - } - }; - // Always hit Composio directly here — using list_sync_targets would - // short-circuit through the registry and miss new connections. - let targets = match composio::scan_active_sync_targets(&*config).await { - Ok(t) => t, - Err(e) => { - tracing::debug!( - error = %e, - "[memory_sources:reconcile] no composio sync targets available; skipping" - ); - return None; - } - }; - - // Build the upsert targets up front, then apply them with a single config - // load + save via the batch path. The per-call upsert does its own - // load-modify-save, so the old loop cost 2N config round-trips for N - // connections; batching collapses that to 2. - let upsert_targets = build_upsert_targets(&targets); - let upserted = match registry::upsert_composio_sources_batch(&upsert_targets).await { - Ok(n) => n, - Err(e) => { - tracing::warn!( - targets = targets.len(), - error = %e, - "[memory_sources:reconcile] batch upsert failed" - ); - 0 - } - }; - - if !targets.is_empty() { - tracing::info!( - targets = targets.len(), - upserted = upserted, - "[memory_sources:reconcile] composio reconciliation complete" - ); - } - - // Run the one-time caps migration after the reconcile loop so any - // sources upserted just above are also considered. - if let Err(e) = apply_composio_source_caps_migration().await { - tracing::warn!( - error = %e, - "[memory_sources:reconcile] caps migration failed (non-fatal, will retry next time)" - ); - } - - // The scan succeeded — surface the live active-connection set so the list - // path can hide rows for connections that are no longer active (re-auth / - // token expiry mints a fresh connection_id, stranding the old row) and - // collapse identical same-id duplicates. - Some(targets.iter().map(|t| t.connection_id.clone()).collect()) -} - -/// Build the `(toolkit, connection_id, label)` upsert targets for a batch -/// reconcile from the scanned Composio sync targets. -/// -/// The label is a title-cased toolkit name plus the truncated connection id so -/// distinct accounts of the same toolkit (e.g. two Gmail logins) don't all show -/// as "Gmail connection". Pure (no I/O) so it can be unit-tested directly. -fn build_upsert_targets(targets: &[composio::SyncTarget]) -> Vec { - targets - .iter() - .map(|target| { - let label = format!( - "{} · {}", - title_case(&target.toolkit), - short_id(&target.connection_id) - ); - (target.toolkit.clone(), target.connection_id.clone(), label) - }) - .collect() -} /// Apply conservative default caps in-place to every cap-less source. /// diff --git a/crates/tinymemory-core/src/sources/sync.rs b/crates/tinymemory-core/src/sources/sync.rs index 5f901676..d57eddfd 100644 --- a/crates/tinymemory-core/src/sources/sync.rs +++ b/crates/tinymemory-core/src/sources/sync.rs @@ -18,7 +18,7 @@ use std::sync::Mutex; use tinymemory_api::host::test_support::TestHostConfig; use crate::sources::types::{MemorySourceEntry, SourceKind}; -use crate::sync::composio::ComposioUsage; +use crate::sync::usage::ProviderUsage; use crate::sync_events::{emit_sync_stage, MemorySyncStage, MemorySyncTrigger}; use crate::Config; @@ -86,7 +86,7 @@ pub async fn sync_source(source: MemorySourceEntry, config: Arc) -> Resu let sync_start = std::time::Instant::now(); // Composio billable-action usage for this run, populated by // `sync_composio` (#3111). Stays zero for non-Composio kinds. - let mut composio_usage = ComposioUsage::default(); + let mut composio_usage = ProviderUsage::default(); let outcome = match source.kind { SourceKind::Composio => { match crate::engine::run_source_pipeline(&source, &*config).await { diff --git a/crates/tinymemory-core/src/sync/mod.rs b/crates/tinymemory-core/src/sync/mod.rs index 59ea17ac..574cfb69 100644 --- a/crates/tinymemory-core/src/sync/mod.rs +++ b/crates/tinymemory-core/src/sync/mod.rs @@ -27,6 +27,7 @@ //! single shape to call; everything else stays local. pub mod audit; +pub mod usage; pub mod mcp; pub mod sync_status; pub mod workspace; diff --git a/crates/tinymemory-core/src/sync/usage.rs b/crates/tinymemory-core/src/sync/usage.rs new file mode 100644 index 00000000..21b31adf --- /dev/null +++ b/crates/tinymemory-core/src/sync/usage.rs @@ -0,0 +1,23 @@ +//! What one sync run cost. + +use serde::{Deserialize, Serialize}; + +/// Per-run accumulator for a source's billable provider calls. +/// +/// # Why it outlived the Composio tree +/// +/// It is what the sync audit log records, and the audit log is this crate's. +/// A run against a connected account has a price attached — the provider +/// charges per action — and an operator asking "why did this month cost that" +/// is asking a question about stored rows, not about whoever fetched them. +/// +/// Zero for the sources that cost nothing to read, which is most of them. +/// That is not a gap: a folder scan really did call nothing and spend nothing, +/// and the field says so rather than being absent. +#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] +pub struct ProviderUsage { + /// Calls that returned a response this run. + pub actions_called: u32, + /// Sum of each response's provider-reported cost. + pub cost_usd: f64, +} From 567ad5b5338b7a0961619de3ee654f7a51da77ad Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:45:02 +0300 Subject: [PATCH 12/16] chore: files changed crates/tinymemory-core/src/engine/sync.rs Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinymemory-core/src/engine/sync.rs | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/crates/tinymemory-core/src/engine/sync.rs b/crates/tinymemory-core/src/engine/sync.rs index f0099144..74f47ffd 100644 --- a/crates/tinymemory-core/src/engine/sync.rs +++ b/crates/tinymemory-core/src/engine/sync.rs @@ -251,7 +251,15 @@ impl ExternalSourceReader for HostSyncAdapter { .as_ref() .ok_or_else(|| anyhow::anyhow!("external source reader requires host config"))?; let host_source: MemorySourceEntry = serde_json::from_value(serde_json::to_value(source)?)?; - let reader = crate::sources::readers::reader_for(&host_source.kind); + // A kind with no reader is an error, not an empty listing: answering + // "0 items" for a source nobody read would let the caller record the + // sync as complete and move its cursor past everything it skipped. + let reader = crate::sources::readers::reader_for(&host_source.kind).ok_or_else(|| { + anyhow::anyhow!( + "no reader for source kind {:?}: it is fetched outside this crate", + host_source.kind + ) + })?; let items = reader .list_items(&host_source, &**config) .await @@ -269,7 +277,12 @@ impl ExternalSourceReader for HostSyncAdapter { .as_ref() .ok_or_else(|| anyhow::anyhow!("external source reader requires host config"))?; let host_source: MemorySourceEntry = serde_json::from_value(serde_json::to_value(source)?)?; - let reader = crate::sources::readers::reader_for(&host_source.kind); + let reader = crate::sources::readers::reader_for(&host_source.kind).ok_or_else(|| { + anyhow::anyhow!( + "no reader for source kind {:?}: it is fetched outside this crate", + host_source.kind + ) + })?; let content = reader .read_item(&host_source, item_id, &**config) .await From 94a7d0922683e9661fa575d25acbd5ea5347e497 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:45:15 +0300 Subject: [PATCH 13/16] fix(sources): remove unused HashSet import Removed the import of `std::collections::HashSet` from the reconcile module as it was no longer used anywhere in the source file, eliminating a compiler warning about unused imports. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinymemory-core/src/sources/reconcile.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/tinymemory-core/src/sources/reconcile.rs b/crates/tinymemory-core/src/sources/reconcile.rs index 637aa316..d6e87f31 100644 --- a/crates/tinymemory-core/src/sources/reconcile.rs +++ b/crates/tinymemory-core/src/sources/reconcile.rs @@ -11,14 +11,11 @@ use crate::config_loader as config_rpc; use crate::sources::registry; use crate::sources::types::{MemorySourceEntry, SourceKind}; -use std::collections::HashSet; /// Current version of the caps migration. Bump when the migration logic changes /// so installs that ran an earlier revision re-run it exactly once. const CURRENT_CAPS_MIGRATION_VERSION: u32 = 1; - - /// Apply conservative default caps in-place to every cap-less source. /// /// For a Composio source with no `max_items`/`sync_depth_days`, writes the From 3f2c8aa22d6ebf32940c012685d1353144fb8f49 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:45:23 +0300 Subject: [PATCH 14/16] chore: reorder re-exports and remove stray blank lines Clean up formatting inconsistencies across several files by removing extraneous blank lines in engine/sync.rs and store/identity.rs, and by reordering the module declaration in sync/mod.rs so that usage appears after mcp and sync_status. These changes have no effect on behaviour and are purely cosmetic. Auto-committed-on: dragonfly Co-authored-by: Medulla --- crates/tinymemory-core/src/engine/mod.rs | 7 +++---- crates/tinymemory-core/src/engine/sync.rs | 5 ----- crates/tinymemory-core/src/store/identity.rs | 1 - crates/tinymemory-core/src/sync/mod.rs | 2 +- 4 files changed, 4 insertions(+), 11 deletions(-) diff --git a/crates/tinymemory-core/src/engine/mod.rs b/crates/tinymemory-core/src/engine/mod.rs index a6bf2963..437dd9f5 100644 --- a/crates/tinymemory-core/src/engine/mod.rs +++ b/crates/tinymemory-core/src/engine/mod.rs @@ -65,10 +65,9 @@ pub use seal::{ }; pub use summariser::HostSummariser; pub use sync::{ - estimate_cost_usd, needs_rebuild, raw_coverage, read_audit_log, - rebuild_tree_from_raw, run_github_sync, run_source_pipeline, - sync_context, HostSyncAdapter, RawCoverage, RawFileRef, RealCostAccumulator, RebuildOutcome, - SourcePipelineFailure, HOST_SYNC_STATE_NAMESPACE, + estimate_cost_usd, needs_rebuild, raw_coverage, read_audit_log, rebuild_tree_from_raw, + run_github_sync, run_source_pipeline, sync_context, HostSyncAdapter, RawCoverage, RawFileRef, + RealCostAccumulator, RebuildOutcome, SourcePipelineFailure, HOST_SYNC_STATE_NAMESPACE, }; // The audit type, under the seam path OpenHuman already names // (`memory::tinycortex::SyncAuditEntry` embeds it in an RPC response type). diff --git a/crates/tinymemory-core/src/engine/sync.rs b/crates/tinymemory-core/src/engine/sync.rs index 74f47ffd..9f0d3f9b 100644 --- a/crates/tinymemory-core/src/engine/sync.rs +++ b/crates/tinymemory-core/src/engine/sync.rs @@ -370,11 +370,6 @@ pub async fn run_source_pipeline( }) } - - - - - fn build_pipeline( source: &MemorySourceEntry, _config: &Config, diff --git a/crates/tinymemory-core/src/store/identity.rs b/crates/tinymemory-core/src/store/identity.rs index dd2cc778..1a5e4c7a 100644 --- a/crates/tinymemory-core/src/store/identity.rs +++ b/crates/tinymemory-core/src/store/identity.rs @@ -102,7 +102,6 @@ pub fn canonicalize(kind: IdentityKind, raw: &str) -> Option { }) } - /// Cross-toolkit variant — matches against every connected provider's /// rows of this kind. Used for marking memory-tree entity rows: an email /// in a Slack message that matches the user's Gmail address is still diff --git a/crates/tinymemory-core/src/sync/mod.rs b/crates/tinymemory-core/src/sync/mod.rs index 574cfb69..8b043f7b 100644 --- a/crates/tinymemory-core/src/sync/mod.rs +++ b/crates/tinymemory-core/src/sync/mod.rs @@ -27,7 +27,7 @@ //! single shape to call; everything else stays local. pub mod audit; -pub mod usage; pub mod mcp; pub mod sync_status; +pub mod usage; pub mod workspace; From a738bbc0406c353b05e2642c3b8a7d83ba4d0540 Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:46:56 +0300 Subject: [PATCH 15/16] chore(tests): remove stale Composio test seams and unused helpers Remove the `TestComposioHost` stub and its registration from the test seam initialiser, along with the `sync_target` helper and two `build_upsert_targets` tests that are no longer relevant. These were left over from an earlier implementation and are not exercised by any remaining test. Auto-committed-on: dragonfly Co-authored-by: Medulla --- .../tinymemory-core/src/sources/reconcile.rs | 31 ++--------------- crates/tinymemory-core/src/test_seams.rs | 33 ------------------- 2 files changed, 3 insertions(+), 61 deletions(-) diff --git a/crates/tinymemory-core/src/sources/reconcile.rs b/crates/tinymemory-core/src/sources/reconcile.rs index d6e87f31..e9bd07de 100644 --- a/crates/tinymemory-core/src/sources/reconcile.rs +++ b/crates/tinymemory-core/src/sources/reconcile.rs @@ -246,36 +246,11 @@ mod tests { } } - fn sync_target(toolkit: &str, connection_id: &str) -> composio::SyncTarget { - composio::SyncTarget { - toolkit: toolkit.to_string(), - connection_id: connection_id.to_string(), - } - } - + #[test] - fn build_upsert_targets_formats_label_and_preserves_order() { - let targets = vec![ - sync_target("gmail", "ca_WaktIDFlZwXO"), - sync_target("slack", "short"), - ]; - let out = build_upsert_targets(&targets); - assert_eq!(out.len(), 2); - // (toolkit, connection_id, label) — toolkit/connection_id carried through verbatim. - assert_eq!(out[0].0, "gmail"); - assert_eq!(out[0].1, "ca_WaktIDFlZwXO"); - assert_eq!(out[0].2, "Gmail · IDFlZwXO"); - assert_eq!(out[1].0, "slack"); - assert_eq!(out[1].1, "short"); - assert_eq!(out[1].2, "Slack · short"); - } - + #[test] - fn build_upsert_targets_empty_is_empty() { - let out = build_upsert_targets(&[]); - assert!(out.is_empty()); - } - + #[test] fn short_id_truncates_ascii() { assert_eq!(short_id("ca_WaktIDFlZwXO"), "IDFlZwXO"); diff --git a/crates/tinymemory-core/src/test_seams.rs b/crates/tinymemory-core/src/test_seams.rs index 17ca16b9..24a27007 100644 --- a/crates/tinymemory-core/src/test_seams.rs +++ b/crates/tinymemory-core/src/test_seams.rs @@ -51,7 +51,6 @@ pub(crate) fn init() { crate::embedding_host::TestEmbeddingHost::install(); crate::config_loader::set_config_loader(Arc::new(TestConfigLoader)); crate::chat_host::set_chat_host(Arc::new(TestChatHost)); - crate::composio_host::set_composio_host(Arc::new(TestComposioHost)); }); } @@ -93,38 +92,6 @@ impl crate::chat_host::ChatHost for TestChatHost { /// /// Every method reports the no-backend-session state, which is the branch the /// core's own tests exercise; a stub that succeeded would need to fake Composio -/// itself. -#[derive(Debug)] -struct TestComposioHost; - -#[async_trait] -impl crate::composio_host::ComposioHost for TestComposioHost { - async fn list_connections( - &self, - _config: &Config, - ) -> Result, String> { - Err(NO_SESSION.to_string()) - } - - async fn execute( - &self, - _config: &Config, - _tool: &str, - _arguments: Option, - _entity_id: &str, - _connection_id: Option<&str>, - ) -> Result { - Err(NO_SESSION.to_string()) - } - - fn api_key(&self, _config: &Config) -> Option { - None - } - - fn is_available(&self, _config: &Config) -> bool { - false - } -} /// The message [`TestComposioHost`] reports. Matches the shape the real backend /// client produces when no session token is stored, which is what the tests From 6007de49137d16b351aa6b9b59e9280899ac945b Mon Sep 17 00:00:00 2001 From: Steven Enamakel Date: Sun, 30 Aug 2026 18:47:09 +0300 Subject: [PATCH 16/16] Remove the Composio connector tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reaching a connected account needs a credential this crate does not hold and must not. The host fetches through the tinyconnectors module and hands the records back through MemorySourceSink::accept_source_items. Four things were memory's all along and stay, moved out from under the tree: the self-identity matcher (reads this crate's profile store), the periodic cadence helpers (about the user's setting, not any source), the per-run usage tally (what the audit log records), and SourceKind::Composio itself — rows already written are still stored, queried and forgotten under it. reader_for returns Option now, so a caller has to decide what to do about a kind it cannot read rather than discovering it per item at runtime. Co-authored-by: Medulla --- crates/tinymemory-core/src/sources/reconcile.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/crates/tinymemory-core/src/sources/reconcile.rs b/crates/tinymemory-core/src/sources/reconcile.rs index e9bd07de..3e4e7fd2 100644 --- a/crates/tinymemory-core/src/sources/reconcile.rs +++ b/crates/tinymemory-core/src/sources/reconcile.rs @@ -246,11 +246,8 @@ mod tests { } } - #[test] - #[test] - #[test] fn short_id_truncates_ascii() { assert_eq!(short_id("ca_WaktIDFlZwXO"), "IDFlZwXO");