From 01cf5d2c17c7ddc8b4e882a291e5ad69d26ab56f Mon Sep 17 00:00:00 2001 From: mocha06 <52426811+mocha06@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:47:46 -0300 Subject: [PATCH 1/2] feat: LLM provider discovery read tools and access probe Add get_llm_providers, get_available_ai_models, get_default_llm_provider, get_llm_provider_dependencies, and validate_llm_provider_access with full SDK + MCP + CLI parity, plus the shared SDK-level GraphQL error classifier (pipefy_sdk.graphql_problem) the probe and later write gating reuse. Closes #411 --- README.md | 5 +- docs/mcp/README.md | 1 + docs/mcp/tools/automations-and-ai.md | 2 +- docs/mcp/tools/llm-providers.md | 41 ++ docs/parity.md | 9 +- .../cli/src/pipefy_cli/commands/_common.py | 10 + .../src/pipefy_cli/commands/ai_provider.py | 153 +++++++ packages/cli/src/pipefy_cli/main.py | 2 + .../cli/tests/fixtures/cli_help_golden.txt | 124 ++++++ .../cli/tests/test_ai_provider_commands.py | 218 ++++++++++ .../src/pipefy_mcp/tools/ai_agent_tools.py | 5 +- .../pipefy_mcp/tools/llm_provider_tools.py | 275 ++++++++++++ packages/mcp/src/pipefy_mcp/tools/registry.py | 7 + .../tests/tools/test_llm_provider_tools.py | 312 ++++++++++++++ packages/sdk/src/pipefy_sdk/__init__.py | 18 + packages/sdk/src/pipefy_sdk/client.py | 55 +++ .../sdk/src/pipefy_sdk/graphql_problem.py | 154 +++++++ .../queries/llm_provider_queries.py | 126 ++++++ .../services/llm_provider_service.py | 247 +++++++++++ packages/sdk/src/pipefy_sdk/services/types.py | 66 +++ .../services/test_llm_provider_service.py | 391 ++++++++++++++++++ skills/ai-agents/pipefy-ai-agents/SKILL.md | 2 +- 22 files changed, 2215 insertions(+), 8 deletions(-) create mode 100644 docs/mcp/tools/llm-providers.md create mode 100644 packages/cli/src/pipefy_cli/commands/ai_provider.py create mode 100644 packages/cli/tests/test_ai_provider_commands.py create mode 100644 packages/mcp/src/pipefy_mcp/tools/llm_provider_tools.py create mode 100644 packages/mcp/tests/tools/test_llm_provider_tools.py create mode 100644 packages/sdk/src/pipefy_sdk/graphql_problem.py create mode 100644 packages/sdk/src/pipefy_sdk/queries/llm_provider_queries.py create mode 100644 packages/sdk/src/pipefy_sdk/services/llm_provider_service.py create mode 100644 packages/sdk/tests/services/test_llm_provider_service.py diff --git a/README.md b/README.md index e86c5c87..581ce1eb 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Open-source toolkit for **Pipefy** developers: a Model Context Protocol (MCP) se | Component | Package / path | Purpose | |-----------|----------------|---------| -| **MCP server** | `pipefy-mcp-server` | Exposes **157** tools to MCP clients (Cursor, Claude Desktop, Claude Code, and others). | +| **MCP server** | `pipefy-mcp-server` | Exposes **162** tools to MCP clients (Cursor, Claude Desktop, Claude Code, and others). | | **CLI** | `pipefy-cli` | Terminal commands aligned with MCP capabilities; see [`docs/parity.md`](docs/parity.md). | | **SDK** | `pipefy` | Vendor GraphQL client, services, and models shared by MCP and CLI. | | **Skills** | [`skills/`](skills/) | Markdown playbooks (Anthropic Skills format) for common Pipefy workflows. | @@ -141,7 +141,7 @@ Deprecation and semver (post-1.0): [`docs/DEPRECATION.md`](docs/DEPRECATION.md). ## MCP server -The server registers **157 tools** across eleven domains. Canonical names: `PIPEFY_TOOL_NAMES` in [`packages/mcp/src/pipefy_mcp/tools/registry.py`](packages/mcp/src/pipefy_mcp/tools/registry.py). +The server registers **162 tools** across twelve domains. Canonical names: `PIPEFY_TOOL_NAMES` in [`packages/mcp/src/pipefy_mcp/tools/registry.py`](packages/mcp/src/pipefy_mcp/tools/registry.py). Tool descriptions and `Args:` blocks come from Python docstrings (what MCP clients show to models). Per-area reference docs cover parameters, edge cases, and cross-cutting behavior. @@ -154,6 +154,7 @@ Tool descriptions and `Args:` blocks come from Python docstrings (what MCP clien | **Relations** | 8 | Pipe and card relations. | [docs](docs/mcp/tools/relations.md) | | **Reports** | 17 | Pipe and organization reports, async exports. | [docs](docs/mcp/tools/reports.md) | | **Automations & AI** | 23 | Automations, AI automations, AI agents, validators. | [docs](docs/mcp/tools/automations-and-ai.md) | +| **LLM providers** | 5 | Discovery reads for the organization's LLM providers (custom + Pipefy-managed), vendor model lists, owner defaults, dependencies, and a read-access probe. | [docs](docs/mcp/tools/llm-providers.md) | | **iPaaS** | 4 | Lazy discovery, invocation, and app-connection setup for a pipe's iPaaS (Advanced Automations) workspace (`get_ipaas_tools`, `call_ipaas_tool`, plus the connection meta-tools). | [docs](docs/mcp/tools/ipaas.md) | | **Observability** | 11 | Logs, usage, credits, execution metrics, job exports. | [docs](docs/mcp/tools/observability.md) | | **Members, email & webhooks** | 11 | Membership, inbox email, webhooks. | [docs](docs/mcp/tools/members-email-webhooks.md) | diff --git a/docs/mcp/README.md b/docs/mcp/README.md index 0d6b0c16..706ecd80 100644 --- a/docs/mcp/README.md +++ b/docs/mcp/README.md @@ -12,6 +12,7 @@ Material in this tree describes **`pipefy-mcp-server`**: the MCP process, tool b | [`tools/relations.md`](tools/relations.md) | Pipe and card relations | | [`tools/reports.md`](tools/reports.md) | Pipe and organization reports, async exports | | [`tools/automations-and-ai.md`](tools/automations-and-ai.md) | Traditional automations, AI automations, AI agents, validators | +| [`tools/llm-providers.md`](tools/llm-providers.md) | LLM provider discovery: custom + system providers, vendor models, defaults, dependencies, access probe | | [`tools/observability.md`](tools/observability.md) | Logs, usage, credits, execution metrics, job exports | | [`tools/members-email-webhooks.md`](tools/members-email-webhooks.md) | Membership, inbox email, webhooks | | [`tools/organization.md`](tools/organization.md) | Organization metadata | diff --git a/docs/mcp/tools/automations-and-ai.md b/docs/mcp/tools/automations-and-ai.md index 77bd40da..338aac32 100644 --- a/docs/mcp/tools/automations-and-ai.md +++ b/docs/mcp/tools/automations-and-ai.md @@ -182,7 +182,7 @@ On **create**, if the caller omits `condition`, the MCP layer supplies `DEFAULT_ Optional inside `actionParams.aiBehaviorParams`: - **`capabilitiesAttributes`** — list of capability entries, each exactly `{ "capabilityType": "", "enabled": true|false }`. Both keys are required and no other keys are accepted; legacy shapes (bare string lists, `{ "type": ... }`) are rejected. Common `capabilityType` values: `advanced_ocr` (product name IDP / Intelligent Document Processing), `math_operations` (Calculations & Analysis), `web_search`, `web_scraping`, `max_effort`. `capabilityType` is not checked against a fixed set — any value passes through and the API validates the enum on write. Validation checks shape only, not plan/feature entitlement — a capability may still require organization-level enablement to take effect. -- **`providerId`** / **`systemProviderId`** — select the behavior's LLM provider; set at most one (reads resolve a single active provider). Discover IDs via the organization's AI settings in the Pipefy UI. +- **`providerId`** / **`systemProviderId`** — select the behavior's LLM provider; set at most one (reads resolve a single active provider). Discover IDs with `get_llm_providers` (see [LLM providers](llm-providers.md)): use `providerId` for a custom (`byom`) provider and `systemProviderId` for a Pipefy-managed (`system`) one. IDs are also visible in the organization's AI settings in the Pipefy UI. ### Optional `eventParams` (trigger filters) diff --git a/docs/mcp/tools/llm-providers.md b/docs/mcp/tools/llm-providers.md new file mode 100644 index 00000000..fb9916f7 --- /dev/null +++ b/docs/mcp/tools/llm-providers.md @@ -0,0 +1,41 @@ +# LLM providers + +Read-only discovery of the LLM providers an organization can use: custom (BYOM) providers, Pipefy-managed system providers, vendor model lists, owner defaults, provider dependencies, and a read-access probe. **5 tools.** + +These are the discovery counterpart to the `providerId` / `systemProviderId` fields on AI agent behaviors (see [Automations & AI](automations-and-ai.md)): use `get_llm_providers` to find the IDs a behavior accepts. + +--- + +| Tool | Read-only | Role | +|------|-----------|------| +| `get_llm_providers` | Yes | Lists custom and system providers in one union, paginated (`first` default 50, `after`) with `only_active` (filters custom providers only). Each node carries `type` (`byom` = custom, `system` = Pipefy-managed) plus `configuration`. | +| `get_available_ai_models` | Yes | Lists the model names a provider vendor exposes. `provider_name` is one of `openai`, `azure_openai`, `amazon_bedrock`, `custom`, `google_vertex_ai`, `oracle_oci`, `anthropic` (the API validates membership). | +| `get_default_llm_provider` | Yes | Resolves the default provider for an owner: `owner_type` is `organization` (default), `assistant`, or `behavior`. The returned `type` says whether the default is custom or system. | +| `get_llm_provider_dependencies` | Yes | Lists the owners (`ownerId` / `ownerType`) that depend on a provider — the blockers to check before deactivating or removing one. Paginated, with `total_count`. | +| `validate_llm_provider_access` | Yes | Probes whether the current credential can read the organization's providers, classifying failures into structured problems (permission denied / not found / invalid arguments) instead of opaque errors. | + +## Identifiers: UUID vs numeric ID + +The organization identifier is **not uniform** across these queries — this follows the Pipefy GraphQL API, not a toolkit choice: + +- `get_llm_providers` and `get_llm_provider_dependencies` take the organization **UUID** (`organization_uuid`). +- `get_default_llm_provider` with `owner_type="organization"` takes the **numeric organization ID** as `owner_id`. +- `get_available_ai_models` takes no organization argument at all (vendor-level lookup). + +`get_organization` returns both `id` and `uuid`, so one call resolves whichever form a tool needs. + +## Configuration redaction + +Provider `configuration` is included in read output, and the API **redacts secret values server-side**: placeholder markers come back instead of secrets (e.g. an access token reads as a redaction placeholder). Reading configuration never exposes credentials. + +## Probe semantics + +- A green `validate_llm_provider_access` proves **read access only** — never write entitlement. Provider mutations may still be denied. +- An **empty system-provider list** can mean Pipefy-managed system models are not enabled for the organization, rather than a permission problem. The probe reports `system_providers_visible` / `custom_providers_visible` separately and says so in its `note`. +- Permissions are asymmetric across the read surface: the provider **list** needs a weaker permission than the models / default / dependencies reads, so a green probe does not guarantee those three succeed for the same credential. +- The scoping of the stronger permission also differs: `get_available_ai_models` and `get_default_llm_provider` authorize against the **credential's own organization context**, while `get_llm_provider_dependencies` authorizes against the **target organization** passed as `organization_uuid` — the same token can be denied on the former and green on the latter. +- Probing an **unknown organization UUID** surfaces as permission denied rather than not-found (the API does not reveal whether the organization exists). + +## Error classification + +Failures on this surface are classified by a shared SDK-level module (`pipefy_sdk.graphql_problem`) into structured problems — `permission_denied`, `not_found`, `invalid_arguments`, `feature_not_enabled`, or `runtime` — carried on `error.details.kind` alongside the GraphQL `extensions.code` and `correlation_id`. The same classifier backs the CLI (`pipefy ai-provider ...`), so both surfaces report the same problem kinds. diff --git a/docs/parity.md b/docs/parity.md index 4a1e59b2..53e0d6ee 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -2,7 +2,7 @@ This matrix is the source of truth for **MCP tool ↔ `pipefy` CLI** coverage. Update it whenever MCP tools or CLI commands are added, renamed, or removed. -**Registry source:** `PIPEFY_TOOL_NAMES` in `packages/mcp/src/pipefy_mcp/tools/registry.py` (must stay in sync with this table: **157** tools). +**Registry source:** `PIPEFY_TOOL_NAMES` in `packages/mcp/src/pipefy_mcp/tools/registry.py` (must stay in sync with this table: **162** tools). **Later CLI coverage:** areas such as attachments, field conditions, email, audit export, traditional automations, exports/usage, introspection, and raw GraphQL appear as **shipped** below when the matching Typer commands exist in `packages/cli`. @@ -102,16 +102,20 @@ For **database records**, `find_records` result nodes may use **`fields`** while | `get_automation_logs_by_repo` | `pipefy automation logs --repo` | shipped | — | | `get_automations` | `pipefy automation list` | shipped | (optional `--organization` / `--pipe`). | | `get_automations_usage` | `pipefy usage automations` (also `pipefy automation usage`) | shipped | (`--organization`, `--from`, `--to` ISO range). | +| `get_available_ai_models` | `pipefy ai-provider models` | shipped | LLM provider discovery; vendor model list (`--provider-name`). | | `get_card` | `pipefy card get` | shipped | Supports `--include-fields`. | | `get_card_inbox_emails` | `pipefy email inbox list` | shipped | (`--card`). | | `get_card_relations` | `pipefy relation card list` | shipped | (raw ``get_card_relations`` payload). | | `get_cards` | `pipefy card list` | shipped | ``list`` maps to ``get_cards`` (``--pipe``, ``--title``, ``--search`` JSON / ``CardSearch``, ``--include-fields``, ``--first`` / ``--after``). Use ``pipefy card find`` for ``find_cards`` (single field equality). | +| `get_default_llm_provider` | `pipefy ai-provider default get` | shipped | LLM provider discovery; owner default (`--owner-id`, `--owner-type`, default `organization`). | | `get_email_templates` | `pipefy email template list` | shipped | (`--repo`). | | `get_ipaas_connection_auth_url` | — | deferred | iPaaS (Advanced Automations) OAuth consent URL for connecting an app; MCP-first surface. | | `get_field_condition` | `pipefy field-condition get` | shipped | — | | `get_field_conditions` | `pipefy field-condition list` | shipped | (`--phase`). | | `get_ipaas_tools` | — | deferred | iPaaS (Advanced Automations) tool discovery; MCP-first surface, CLI twin considered alongside invocation's. | | `get_labels` | `pipefy label list` | shipped | — | +| `get_llm_provider_dependencies` | `pipefy ai-provider dependencies` | shipped | LLM provider discovery; owners depending on a provider (`--provider-id`, `--org-uuid`). | +| `get_llm_providers` | `pipefy ai-provider list` | shipped | LLM provider discovery; custom + system providers in one union (`--org-uuid`, `--only-active`, `--first` / `--after`). | | `get_organization` | `pipefy org get` | shipped | Organization-level read; out of v0.1 parity per spec. | | `get_organization_report` | `pipefy report-org get` | shipped | Organization reports. | | `get_organization_report_export` | (poll via `pipefy report-org export --format json`) | shipped | Organization reports + export-shaped. | @@ -182,6 +186,7 @@ For **database records**, `find_records` result nodes may use **`fields`** while | `upload_attachment_to_table_record` | `pipefy attachment upload --record` | shipped | (same supporting flags as card). | | `validate_ai_agent_behaviors` | `pipefy agent validate-behaviors` | shipped | AI Agents validation tooling. | | `validate_ai_automation_prompt` | `pipefy ai-automation validate-prompt` | shipped | AI Automations validation tooling. | +| `validate_llm_provider_access` | `pipefy ai-provider validate-access` | shipped | LLM provider read-access probe; green proves read access only, never write entitlement. | ## Row count check @@ -194,6 +199,6 @@ for n in m.body: print(len(v.args[0].elts))" ``` -Expect **157** tool names in `PIPEFY_TOOL_NAMES` and **157** data rows in the parity table (excluding the header rows). +Expect **162** tool names in `PIPEFY_TOOL_NAMES` and **162** data rows in the parity table (excluding the header rows). When adding or removing an MCP tool, update **this file** and `PIPEFY_TOOL_NAMES` in the same change set. diff --git a/packages/cli/src/pipefy_cli/commands/_common.py b/packages/cli/src/pipefy_cli/commands/_common.py index 11e883fa..0171fb61 100644 --- a/packages/cli/src/pipefy_cli/commands/_common.py +++ b/packages/cli/src/pipefy_cli/commands/_common.py @@ -335,6 +335,7 @@ def run_cli_command( *, exit_code_2_on_value_error: bool = True, format_transport_query_error: Callable[[TransportQueryError], str] | None = None, + exit_1_on_unsuccessful: bool = False, ) -> None: """Run an async coroutine factory with a configured client and render the result. @@ -345,6 +346,9 @@ def run_cli_command( exit_code_2_on_value_error: Map ``ValueError`` to process exit code 2 (stderr). format_transport_query_error: Optional override for GraphQL transport errors (defaults to a single-line formatter). + exit_1_on_unsuccessful: After rendering, exit 1 when the result dict has + ``success`` falsy. For probe-style commands whose failure is data, + not an exception, but should still be scriptable via exit code. """ pipefy_settings, auth = settings_and_auth_from_ctx(ctx) transport_fmt = format_transport_query_error or _format_transport_query_error @@ -374,3 +378,9 @@ async def _run() -> _R: render_json(data) else: render_rich(data) + if ( + exit_1_on_unsuccessful + and isinstance(data, dict) + and not data.get("success", True) + ): + raise typer.Exit(1) diff --git a/packages/cli/src/pipefy_cli/commands/ai_provider.py b/packages/cli/src/pipefy_cli/commands/ai_provider.py new file mode 100644 index 00000000..5f585068 --- /dev/null +++ b/packages/cli/src/pipefy_cli/commands/ai_provider.py @@ -0,0 +1,153 @@ +"""LLM provider discovery (organization-scoped, read-only).""" + +from __future__ import annotations + +import typer +from pipefy_sdk import PipefyClient + +from pipefy_cli.commands._common import run_cli_command + +ai_provider_app = typer.Typer( + help="LLM providers (discovery reads; organization-scoped).", + no_args_is_help=True, +) +default_app = typer.Typer(help="Default LLM provider.", no_args_is_help=True) +ai_provider_app.add_typer(default_app, name="default") + +_ORG_UUID_HELP = "Organization UUID (not the numeric ID; `pipefy org get` shows both)." + + +@ai_provider_app.command("list") +def ai_provider_list( + ctx: typer.Context, + org_uuid: str = typer.Option(..., "--org-uuid", help=_ORG_UUID_HELP), + only_active: bool = typer.Option( + False, + "--only-active", + help="Only return active custom providers (system providers unaffected).", + ), + first: int = typer.Option(50, "--first", help="Page size."), + after: str | None = typer.Option( + None, "--after", help="Cursor from the previous page's page_info.endCursor." + ), + json_out: bool = typer.Option( + False, "--json", "-j", help="Print machine-readable JSON to stdout." + ), +) -> None: + """List custom (BYOM) and Pipefy-managed system LLM providers (``get_llm_providers``). + + Each provider carries ``type`` (``byom`` or ``system``); ``configuration`` + comes back with secret values redacted server-side. An empty system list + can mean system models are not enabled for the organization. + """ + + async def factory(client: PipefyClient): + result = await client.get_llm_providers( + org_uuid, only_active=only_active, first=first, after=after + ) + return {"success": True, **result} + + run_cli_command(ctx, json_out, factory) + + +@ai_provider_app.command("models") +def ai_provider_models( + ctx: typer.Context, + provider_name: str = typer.Option( + ..., + "--provider-name", + help=( + "Provider vendor: openai, azure_openai, amazon_bedrock, custom, " + "google_vertex_ai, oracle_oci, or anthropic." + ), + ), + json_out: bool = typer.Option( + False, "--json", "-j", help="Print machine-readable JSON to stdout." + ), +) -> None: + """List the model names a provider vendor exposes (``get_available_ai_models``).""" + + async def factory(client: PipefyClient): + models = await client.get_available_ai_models(provider_name) + return {"success": True, "models": models} + + run_cli_command(ctx, json_out, factory) + + +@default_app.command("get") +def ai_provider_default_get( + ctx: typer.Context, + owner_id: str = typer.Option( + ..., + "--owner-id", + help=( + "Owner identifier. For --owner-type organization pass the numeric " + "organization ID (not the UUID)." + ), + ), + owner_type: str = typer.Option( + "organization", + "--owner-type", + help="One of: organization (default), assistant, behavior.", + ), + json_out: bool = typer.Option( + False, "--json", "-j", help="Print machine-readable JSON to stdout." + ), +) -> None: + """Resolve the default LLM provider for an owner (``get_default_llm_provider``).""" + + async def factory(client: PipefyClient): + provider = await client.get_default_llm_provider( + owner_id, owner_type=owner_type + ) + return {"success": True, "provider": provider} + + run_cli_command(ctx, json_out, factory) + + +@ai_provider_app.command("dependencies") +def ai_provider_dependencies( + ctx: typer.Context, + provider_id: str = typer.Option( + ..., "--provider-id", help="Provider ID (from `pipefy ai-provider list`)." + ), + org_uuid: str = typer.Option(..., "--org-uuid", help=_ORG_UUID_HELP), + first: int = typer.Option(50, "--first", help="Page size."), + after: str | None = typer.Option( + None, "--after", help="Cursor from the previous page's page_info.endCursor." + ), + json_out: bool = typer.Option( + False, "--json", "-j", help="Print machine-readable JSON to stdout." + ), +) -> None: + """List owners that depend on a provider (``get_llm_provider_dependencies``).""" + + async def factory(client: PipefyClient): + result = await client.get_llm_provider_dependencies( + provider_id, org_uuid, first=first, after=after + ) + return {"success": True, **result} + + run_cli_command(ctx, json_out, factory) + + +@ai_provider_app.command("validate-access") +def ai_provider_validate_access( + ctx: typer.Context, + org_uuid: str = typer.Option(..., "--org-uuid", help=_ORG_UUID_HELP), + json_out: bool = typer.Option( + False, "--json", "-j", help="Print machine-readable JSON to stdout." + ), +) -> None: + """Probe LLM provider read access (``validate_llm_provider_access``). + + A green probe proves read access only, never write entitlement. Exits 1 + when the probe classifies a failure (permission denied / not found / other), + after rendering the structured problem. + """ + + async def factory(client: PipefyClient): + probe = await client.validate_llm_provider_access(org_uuid) + return {"success": bool(probe.get("ok")), **probe} + + run_cli_command(ctx, json_out, factory, exit_1_on_unsuccessful=True) diff --git a/packages/cli/src/pipefy_cli/main.py b/packages/cli/src/pipefy_cli/main.py index 299f471a..a0e9d79a 100644 --- a/packages/cli/src/pipefy_cli/main.py +++ b/packages/cli/src/pipefy_cli/main.py @@ -9,6 +9,7 @@ from pipefy_cli.auth import BearerToken from pipefy_cli.commands.agent import agent_app from pipefy_cli.commands.ai_automation import ai_automation_app +from pipefy_cli.commands.ai_provider import ai_provider_app from pipefy_cli.commands.attachment import attachment_app from pipefy_cli.commands.audit import audit_app from pipefy_cli.commands.auth import auth_app @@ -106,6 +107,7 @@ def main( app.add_typer(agent_app, name="agent") app.add_typer(ai_automation_app, name="ai-automation") +app.add_typer(ai_provider_app, name="ai-provider") app.add_typer(attachment_app, name="attachment") app.add_typer(audit_app, name="audit") app.add_typer(auth_app, name="auth") diff --git a/packages/cli/tests/fixtures/cli_help_golden.txt b/packages/cli/tests/fixtures/cli_help_golden.txt index 6b970fe0..06855c57 100644 --- a/packages/cli/tests/fixtures/cli_help_golden.txt +++ b/packages/cli/tests/fixtures/cli_help_golden.txt @@ -36,6 +36,7 @@ Usage: pipefy [OPTIONS] COMMAND [ARGS]... ╭─ Commands ───────────────────────────────────────────────────────────────────╮ │ agent AI Agents (repo-scoped). │ │ ai-automation AI Automations (generate_with_ai). │ +│ ai-provider LLM providers (discovery reads; organization-scoped). │ │ attachment Attachment uploads (presigned URL + field update). │ │ audit Pipe audit log exports. │ │ auth Authenticate the CLI as your Pipefy user (browser-based │ @@ -392,6 +393,129 @@ Usage: pipefy ai-automation validate-prompt [OPTIONS] │ --help Show this message and exit. │ ╰──────────────────────────────────────────────────────────────────────────────╯ +### HELP ai-provider +Usage: pipefy ai-provider [OPTIONS] COMMAND [ARGS]... + + LLM providers (discovery reads; organization-scoped). + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────╮ +│ list List custom (BYOM) and Pipefy-managed system LLM providers │ +│ (``get_llm_providers``). │ +│ models List the model names a provider vendor exposes │ +│ (``get_available_ai_models``). │ +│ dependencies List owners that depend on a provider │ +│ (``get_llm_provider_dependencies``). │ +│ validate-access Probe LLM provider read access │ +│ (``validate_llm_provider_access``). │ +│ default Default LLM provider. │ +╰──────────────────────────────────────────────────────────────────────────────╯ + +### HELP ai-provider/default +Usage: pipefy ai-provider default [OPTIONS] COMMAND [ARGS]... + + Default LLM provider. + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────╮ +│ get Resolve the default LLM provider for an owner │ +│ (``get_default_llm_provider``). │ +╰──────────────────────────────────────────────────────────────────────────────╯ + +### HELP ai-provider/default/get +Usage: pipefy ai-provider default get [OPTIONS] + + Resolve the default LLM provider for an owner (``get_default_llm_provider``). + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ * --owner-id TEXT Owner identifier. For --owner-type │ +│ organization pass the numeric organization ID │ +│ (not the UUID). │ +│ [required] │ +│ --owner-type TEXT One of: organization (default), assistant, │ +│ behavior. │ +│ [default: organization] │ +│ --json -j Print machine-readable JSON to stdout. │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ + +### HELP ai-provider/dependencies +Usage: pipefy ai-provider dependencies [OPTIONS] + + List owners that depend on a provider (``get_llm_provider_dependencies``). + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ * --provider-id TEXT Provider ID (from `pipefy ai-provider │ +│ list`). │ +│ [required] │ +│ * --org-uuid TEXT Organization UUID (not the numeric ID; │ +│ `pipefy org get` shows both). │ +│ [required] │ +│ --first INTEGER Page size. [default: 50] │ +│ --after TEXT Cursor from the previous page's │ +│ page_info.endCursor. │ +│ --json -j Print machine-readable JSON to stdout. │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ + +### HELP ai-provider/list +Usage: pipefy ai-provider list [OPTIONS] + + List custom (BYOM) and Pipefy-managed system LLM providers + (``get_llm_providers``). + + Each provider carries ``type`` (``byom`` or ``system``); ``configuration`` + comes back with secret values redacted server-side. An empty system list can + mean system models are not enabled for the organization. + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ * --org-uuid TEXT Organization UUID (not the numeric ID; │ +│ `pipefy org get` shows both). │ +│ [required] │ +│ --only-active Only return active custom providers │ +│ (system providers unaffected). │ +│ --first INTEGER Page size. [default: 50] │ +│ --after TEXT Cursor from the previous page's │ +│ page_info.endCursor. │ +│ --json -j Print machine-readable JSON to stdout. │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ + +### HELP ai-provider/models +Usage: pipefy ai-provider models [OPTIONS] + + List the model names a provider vendor exposes (``get_available_ai_models``). + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ * --provider-name TEXT Provider vendor: openai, azure_openai, │ +│ amazon_bedrock, custom, google_vertex_ai, │ +│ oracle_oci, or anthropic. │ +│ [required] │ +│ --json -j Print machine-readable JSON to stdout. │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ + +### HELP ai-provider/validate-access +Usage: pipefy ai-provider validate-access [OPTIONS] + + Probe LLM provider read access (``validate_llm_provider_access``). + + A green probe proves read access only, never write entitlement. Exits 1 when + the probe classifies a failure (permission denied / not found / other), after + rendering the structured problem. + +╭─ Options ────────────────────────────────────────────────────────────────────╮ +│ * --org-uuid TEXT Organization UUID (not the numeric ID; `pipefy │ +│ org get` shows both). │ +│ [required] │ +│ --json -j Print machine-readable JSON to stdout. │ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────╯ + ### HELP attachment Usage: pipefy attachment [OPTIONS] COMMAND [ARGS]... diff --git a/packages/cli/tests/test_ai_provider_commands.py b/packages/cli/tests/test_ai_provider_commands.py new file mode 100644 index 00000000..cded7ca9 --- /dev/null +++ b/packages/cli/tests/test_ai_provider_commands.py @@ -0,0 +1,218 @@ +"""Tests for ``pipefy ai-provider`` commands.""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, MagicMock, patch + +from pipefy_cli.main import app + +BYOM_NODE = { + "__typename": "LlmProvider", + "id": "42", + "name": "Azure custom", + "type": "byom", + "active": True, + "organizationDefault": False, + "configuration": {"auth": {"accessToken": "__REDACTED__"}}, +} + +SYSTEM_NODE = { + "__typename": "SystemLlmProvider", + "id": "7", + "name": "Pipefy GPT", + "type": "system", + "organizationDefault": True, + "systemDefault": True, + "state": "active", + "configuration": {"model": "gpt-4o"}, +} + + +def _env(monkeypatch) -> None: + monkeypatch.setenv("PIPEFY_SERVICE_ACCOUNT_CLIENT_ID", "cid") + monkeypatch.setenv("PIPEFY_SERVICE_ACCOUNT_CLIENT_SECRET", "sec") + + +def _client_patch(mock_client): + return patch( + "pipefy_cli.commands._common.get_authenticated_client", + return_value=mock_client, + ) + + +def test_ai_provider_list_json(runner, clean_pipefy_env, saved_cwd, monkeypatch): + _env(monkeypatch) + mock_client = MagicMock() + mock_client.get_llm_providers = AsyncMock( + return_value={ + "providers": [SYSTEM_NODE, BYOM_NODE], + "page_info": {"hasNextPage": False, "endCursor": None}, + } + ) + + with _client_patch(mock_client): + result = runner.invoke( + app, + [ + "ai-provider", + "list", + "--org-uuid", + "org-uuid-1", + "--only-active", + "--first", + "10", + "--json", + ], + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + data = json.loads(result.stdout) + assert data["success"] is True + assert [p["type"] for p in data["providers"]] == ["system", "byom"] + mock_client.get_llm_providers.assert_awaited_once_with( + "org-uuid-1", only_active=True, first=10, after=None + ) + + +def test_ai_provider_models_json(runner, clean_pipefy_env, saved_cwd, monkeypatch): + _env(monkeypatch) + mock_client = MagicMock() + mock_client.get_available_ai_models = AsyncMock(return_value=["gpt-4o"]) + + with _client_patch(mock_client): + result = runner.invoke( + app, + ["ai-provider", "models", "--provider-name", "openai", "--json"], + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + assert json.loads(result.stdout) == {"success": True, "models": ["gpt-4o"]} + mock_client.get_available_ai_models.assert_awaited_once_with("openai") + + +def test_ai_provider_default_get_defaults_owner_type( + runner, clean_pipefy_env, saved_cwd, monkeypatch +): + _env(monkeypatch) + mock_client = MagicMock() + mock_client.get_default_llm_provider = AsyncMock(return_value=BYOM_NODE) + + with _client_patch(mock_client): + result = runner.invoke( + app, + ["ai-provider", "default", "get", "--owner-id", "123456", "--json"], + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + data = json.loads(result.stdout) + assert data["provider"]["id"] == "42" + mock_client.get_default_llm_provider.assert_awaited_once_with( + "123456", owner_type="organization" + ) + + +def test_ai_provider_dependencies_json( + runner, clean_pipefy_env, saved_cwd, monkeypatch +): + _env(monkeypatch) + mock_client = MagicMock() + mock_client.get_llm_provider_dependencies = AsyncMock( + return_value={ + "dependencies": [{"ownerId": "1", "ownerType": "organization"}], + "page_info": {"hasNextPage": False, "endCursor": None}, + "total_count": 1, + } + ) + + with _client_patch(mock_client): + result = runner.invoke( + app, + [ + "ai-provider", + "dependencies", + "--provider-id", + "42", + "--org-uuid", + "org-uuid-1", + "--after", + "cur-1", + "--json", + ], + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + data = json.loads(result.stdout) + assert data["total_count"] == 1 + mock_client.get_llm_provider_dependencies.assert_awaited_once_with( + "42", "org-uuid-1", first=50, after="cur-1" + ) + + +def test_ai_provider_validate_access_green_exits_0( + runner, clean_pipefy_env, saved_cwd, monkeypatch +): + _env(monkeypatch) + mock_client = MagicMock() + mock_client.validate_llm_provider_access = AsyncMock( + return_value={ + "ok": True, + "system_providers_visible": True, + "custom_providers_visible": True, + "provider_count": 2, + "note": "Read access confirmed.", + } + ) + + with _client_patch(mock_client): + result = runner.invoke( + app, + ["ai-provider", "validate-access", "--org-uuid", "org-uuid-1", "--json"], + ) + assert result.exit_code == 0, result.stdout + (result.stderr or "") + data = json.loads(result.stdout) + assert data["success"] is True + assert data["ok"] is True + + +def test_ai_provider_validate_access_failure_exits_1_with_problem( + runner, clean_pipefy_env, saved_cwd, monkeypatch +): + _env(monkeypatch) + mock_client = MagicMock() + mock_client.validate_llm_provider_access = AsyncMock( + return_value={ + "ok": False, + "problem": { + "kind": "permission_denied", + "message": "Permission denied", + "code": "PERMISSION_DENIED", + "correlation_id": None, + }, + } + ) + + with _client_patch(mock_client): + result = runner.invoke( + app, + ["ai-provider", "validate-access", "--org-uuid", "org-uuid-1", "--json"], + ) + assert result.exit_code == 1 + data = json.loads(result.stdout) + assert data["success"] is False + assert data["problem"]["kind"] == "permission_denied" + + +def test_ai_provider_list_blank_org_uuid_exits_2( + runner, clean_pipefy_env, saved_cwd, monkeypatch +): + _env(monkeypatch) + mock_client = MagicMock() + + async def raise_value_error(*args, **kwargs): + raise ValueError("organization_uuid must be a non-empty string") + + mock_client.get_llm_providers = AsyncMock(side_effect=raise_value_error) + + with _client_patch(mock_client): + result = runner.invoke( + app, ["ai-provider", "list", "--org-uuid", " ", "--json"] + ) + assert result.exit_code == 2 + assert "organization_uuid" in (result.stderr or "") diff --git a/packages/mcp/src/pipefy_mcp/tools/ai_agent_tools.py b/packages/mcp/src/pipefy_mcp/tools/ai_agent_tools.py index 094d68cd..3a5b9409 100644 --- a/packages/mcp/src/pipefy_mcp/tools/ai_agent_tools.py +++ b/packages/mcp/src/pipefy_mcp/tools/ai_agent_tools.py @@ -218,8 +218,9 @@ async def create_ai_agent( entitlement on write (a capability may require organization-level enablement). Optional ``actionParams.aiBehaviorParams.providerId`` / ``systemProviderId`` select the - behavior's LLM provider; set at most one. Discover IDs via the organization's AI - settings in the Pipefy UI. + behavior's LLM provider; set at most one. Discover IDs with ``get_llm_providers`` + (``providerId`` for a custom/byom provider, ``systemProviderId`` for a + Pipefy-managed/system one). Optional ``eventParams`` per behavior (filters when the trigger fires): - ``field_updated`` event → ``{"triggerFieldIds": [""]}`` to fire only on specific fields. diff --git a/packages/mcp/src/pipefy_mcp/tools/llm_provider_tools.py b/packages/mcp/src/pipefy_mcp/tools/llm_provider_tools.py new file mode 100644 index 00000000..e8e9859a --- /dev/null +++ b/packages/mcp/src/pipefy_mcp/tools/llm_provider_tools.py @@ -0,0 +1,275 @@ +"""MCP tools for LLM provider discovery (read-only) and the access probe.""" + +from __future__ import annotations + +from typing import Any + +from mcp.server.fastmcp import Context, FastMCP +from mcp.types import ToolAnnotations +from pipefy_sdk import classify_exception + +from pipefy_mcp.core.tool_error_envelope import ( + is_unified_envelope_enabled, + tool_error, + tool_success, +) +from pipefy_mcp.tools.pagination_helpers import ( + build_pagination_info, + validate_page_size, +) +from pipefy_mcp.tools.tool_context import get_pipefy_client +from pipefy_mcp.tools.validation_helpers import validate_tool_id + +_PROVIDER_ID_DISCOVERY_HINT = ( + "Use 'get_llm_providers' to list provider IDs for the organization." +) + + +def _provider_tool_error_from_exception(exc: BaseException) -> dict[str, Any]: + """Map an SDK/GraphQL exception onto the canonical tool failure envelope. + + Uses the shared SDK classifier (``pipefy_sdk.classify_exception``) so the + kind/code the CLI and probes see is the same one reported here. A + transport-level failure with no GraphQL errors falls back to ``str(exc)``. + """ + problem = classify_exception(exc) + if problem is None: + return tool_error(str(exc)) + message = problem.message + if problem.kind.value == "not_found": + message = f"{message} {_PROVIDER_ID_DISCOVERY_HINT}" + details: dict[str, Any] = {"kind": problem.kind.value} + if problem.correlation_id: + details["correlation_id"] = problem.correlation_id + return tool_error(message, code=problem.code, details=details) + + +def _blank_error(value: str, field: str) -> dict[str, Any] | None: + if not value.strip(): + return tool_error(f"'{field}' must be non-empty.") + return None + + +def _read_success( + data: dict[str, Any], + *, + message: str, + page_info: dict[str, Any] | None = None, + page_size: int | None = None, +) -> dict[str, Any]: + """Unified-envelope success (legacy flat payload when the flag is off). + + The legacy path keeps ``page_info`` in the payload so flag-off clients can + still page (the unified path carries it as the ``pagination`` block). + """ + if is_unified_envelope_enabled(): + pagination = ( + build_pagination_info(page_info=page_info, page_size=page_size) + if page_size is not None + else None + ) + return tool_success(data=data, message=message, pagination=pagination) + legacy: dict[str, Any] = {"success": True, **data} + if page_size is not None: + legacy["page_info"] = page_info + return legacy + + +class LlmProviderTools: + """Declares MCP tools for LLM provider discovery reads and the access probe.""" + + @staticmethod + def register(mcp: FastMCP) -> None: + """Register LLM provider tools on the MCP server.""" + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True)) + async def get_llm_providers( + ctx: Context, + organization_uuid: str, + only_active: bool = False, + first: int = 50, + after: str | None = None, + ) -> dict[str, Any]: + """List all LLM providers available to an organization: custom (BYOM) and Pipefy-managed system providers in one surface. Use this to discover the `providerId` / `systemProviderId` values that AI agent behaviors accept. + + Each provider carries `type` (`byom` = custom, `system` = Pipefy-managed) + and `configuration`; the API redacts secret values server-side, so + placeholders come back instead of secrets. An empty system-provider list + can mean Pipefy-managed system models are not enabled for the + organization, not that access was denied. + + Args: + organization_uuid: Organization UUID (not the numeric ID; `get_organization` returns both). + only_active: Only return active custom providers (system providers are unaffected). + first: Page size (default 50). + after: Cursor from a previous page's pagination `end_cursor`. + """ + client = get_pipefy_client(ctx) + err = _blank_error(organization_uuid, "organization_uuid") + if err is not None: + return err + nfirst, page_err = validate_page_size(first) + if page_err is not None: + return page_err + try: + result = await client.get_llm_providers( + organization_uuid.strip(), + only_active=only_active, + first=nfirst, + after=after, + ) + except Exception as exc: # noqa: BLE001 + return _provider_tool_error_from_exception(exc) + return _read_success( + {"providers": result["providers"]}, + message="LLM providers retrieved.", + page_info=result["page_info"], + page_size=nfirst, + ) + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True)) + async def get_available_ai_models( + ctx: Context, + provider_name: str, + ) -> dict[str, Any]: + """List the model names an LLM provider vendor exposes (for configuring custom providers). + + Args: + provider_name: Provider vendor enum value: `openai`, `azure_openai`, + `amazon_bedrock`, `custom`, `google_vertex_ai`, `oracle_oci`, + or `anthropic`. The API validates membership. + """ + client = get_pipefy_client(ctx) + err = _blank_error(provider_name, "provider_name") + if err is not None: + return err + try: + models = await client.get_available_ai_models(provider_name.strip()) + except Exception as exc: # noqa: BLE001 + return _provider_tool_error_from_exception(exc) + return _read_success( + {"models": models}, message="Available AI models retrieved." + ) + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True)) + async def get_default_llm_provider( + ctx: Context, + owner_id: str, + owner_type: str = "organization", + ) -> dict[str, Any]: + """Resolve the default LLM provider for an owner (organization by default). + + The returned provider's `type` field says whether the default is a + custom (`byom`) or Pipefy-managed (`system`) provider. `configuration` + comes back with secret values redacted server-side. + + Args: + owner_id: Owner identifier. For `owner_type="organization"` pass the + **numeric organization ID** (not the UUID; `get_organization` + returns both). + owner_type: One of `organization` (default), `assistant`, `behavior`. + """ + client = get_pipefy_client(ctx) + err = _blank_error(owner_id, "owner_id") or _blank_error( + owner_type, "owner_type" + ) + if err is not None: + return err + try: + provider = await client.get_default_llm_provider( + owner_id.strip(), owner_type=owner_type.strip() + ) + except Exception as exc: # noqa: BLE001 + return _provider_tool_error_from_exception(exc) + return _read_success( + {"provider": provider}, message="Default LLM provider retrieved." + ) + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True)) + async def get_llm_provider_dependencies( + ctx: Context, + provider_id: str, + organization_uuid: str, + first: int = 50, + after: str | None = None, + ) -> dict[str, Any]: + """List the owners (organization or assistants) that depend on an LLM provider. Check this before deactivating or removing a provider. + + Args: + provider_id: Provider ID (from `get_llm_providers`). + organization_uuid: Organization UUID (not the numeric ID). + first: Page size (default 50). + after: Cursor from a previous page's pagination `end_cursor`. + """ + client = get_pipefy_client(ctx) + provider_id, id_err = validate_tool_id(provider_id, "provider_id") + if id_err is not None: + return id_err + err = _blank_error(organization_uuid, "organization_uuid") + if err is not None: + return err + nfirst, page_err = validate_page_size(first) + if page_err is not None: + return page_err + try: + result = await client.get_llm_provider_dependencies( + provider_id, + organization_uuid.strip(), + first=nfirst, + after=after, + ) + except Exception as exc: # noqa: BLE001 + return _provider_tool_error_from_exception(exc) + return _read_success( + { + "dependencies": result["dependencies"], + "total_count": result["total_count"], + }, + message="LLM provider dependencies retrieved.", + page_info=result["page_info"], + page_size=nfirst, + ) + + @mcp.tool(annotations=ToolAnnotations(readOnlyHint=True)) + async def validate_llm_provider_access( + ctx: Context, + organization_uuid: str, + ) -> dict[str, Any]: + """Probe whether the current credential can read the organization's LLM providers. A green result proves READ access only, never write entitlement; provider mutations may still be denied. + + On success, reports whether system and custom providers are visible. + No system providers can mean Pipefy-managed system models are not + enabled for the organization rather than a permission problem. On a + classified failure, returns the structured problem (permission denied / + not found / invalid arguments) instead of an opaque error. + + Args: + organization_uuid: Organization UUID (not the numeric ID). + """ + client = get_pipefy_client(ctx) + err = _blank_error(organization_uuid, "organization_uuid") + if err is not None: + return err + try: + probe = await client.validate_llm_provider_access( + organization_uuid.strip() + ) + except Exception as exc: # noqa: BLE001 + return _provider_tool_error_from_exception(exc) + if probe.get("ok"): + return _read_success( + dict(probe), message="LLM provider read access confirmed." + ) + problem = probe.get("problem") or {} + return tool_error( + str(problem.get("message") or "LLM provider access probe failed."), + code=problem.get("code"), + details={ + k: v + for k, v in ( + ("kind", problem.get("kind")), + ("correlation_id", problem.get("correlation_id")), + ) + if v + }, + ) diff --git a/packages/mcp/src/pipefy_mcp/tools/registry.py b/packages/mcp/src/pipefy_mcp/tools/registry.py index 2107f219..9cd7d740 100644 --- a/packages/mcp/src/pipefy_mcp/tools/registry.py +++ b/packages/mcp/src/pipefy_mcp/tools/registry.py @@ -14,6 +14,7 @@ from pipefy_mcp.tools.field_condition_tools import FieldConditionTools from pipefy_mcp.tools.introspection_tools import IntrospectionTools from pipefy_mcp.tools.ipaas_tools import IpaasTools +from pipefy_mcp.tools.llm_provider_tools import LlmProviderTools from pipefy_mcp.tools.member_tools import MemberTools from pipefy_mcp.tools.observability_tools import ObservabilityTools from pipefy_mcp.tools.organization_tools import OrganizationTools @@ -111,6 +112,8 @@ "get_automation_logs_by_repo", "get_automations", "get_automations_usage", + "get_available_ai_models", + "get_default_llm_provider", "get_field_condition", "get_field_conditions", "get_card", @@ -121,6 +124,8 @@ "get_ipaas_connection_auth_url", "get_ipaas_tools", "get_labels", + "get_llm_provider_dependencies", + "get_llm_providers", "get_organization", "get_organization_report", "get_organization_report_export", @@ -191,6 +196,7 @@ "upload_attachment_to_table_record", "validate_ai_agent_behaviors", "validate_ai_automation_prompt", + "validate_llm_provider_access", } ) @@ -215,6 +221,7 @@ ObservabilityTools, AiAutomationTools, AiAgentTools, + LlmProviderTools, ) diff --git a/packages/mcp/tests/tools/test_llm_provider_tools.py b/packages/mcp/tests/tools/test_llm_provider_tools.py new file mode 100644 index 00000000..67f4d20e --- /dev/null +++ b/packages/mcp/tests/tools/test_llm_provider_tools.py @@ -0,0 +1,312 @@ +"""Tests for LLM provider discovery MCP tools (mocked PipefyClient).""" + +from datetime import timedelta +from unittest.mock import AsyncMock, MagicMock + +import pytest +from gql.transport.exceptions import TransportQueryError +from mcp.shared.memory import ( + create_connected_server_and_client_session as create_client_session, +) +from pipefy_sdk import PipefyClient + +from pipefy_mcp.core.tool_error_envelope import tool_error_message +from pipefy_mcp.tools.llm_provider_tools import LlmProviderTools +from tools.conftest import build_tool_test_server + +BYOM_NODE = { + "__typename": "LlmProvider", + "id": "42", + "name": "Azure custom", + "type": "byom", + "active": True, + "organizationDefault": False, + "configuration": {"auth": {"accessToken": "__REDACTED__"}}, +} + +SYSTEM_NODE = { + "__typename": "SystemLlmProvider", + "id": "7", + "name": "Pipefy GPT", + "type": "system", + "organizationDefault": True, + "systemDefault": True, + "state": "active", + "description": "Managed model", + "aiCredits": 2, + "deprecationDate": None, + "configuration": {"model": "gpt-4o"}, +} + + +def permission_denied_error() -> TransportQueryError: + return TransportQueryError( + "denied", + errors=[ + { + "message": "Permission denied", + "extensions": {"code": "PERMISSION_DENIED", "correlation_id": "corr-9"}, + } + ], + ) + + +@pytest.fixture +def mock_provider_client(): + client = MagicMock(PipefyClient) + client.get_llm_providers = AsyncMock() + client.get_available_ai_models = AsyncMock() + client.get_default_llm_provider = AsyncMock() + client.get_llm_provider_dependencies = AsyncMock() + client.validate_llm_provider_access = AsyncMock() + return client + + +@pytest.fixture +def provider_mcp_server(mock_provider_client): + return build_tool_test_server( + "Pipefy LLM Provider Tools Test", + LlmProviderTools.register, + mock_provider_client, + ) + + +@pytest.fixture +def provider_session(provider_mcp_server, request): + elicitation = getattr(request, "param", None) + return create_client_session( + provider_mcp_server, + read_timeout_seconds=timedelta(seconds=10), + raise_exceptions=True, + elicitation_callback=elicitation, + ) + + +@pytest.mark.anyio +@pytest.mark.parametrize("provider_session", [None], indirect=True) +async def test_get_llm_providers_success_with_pagination( + provider_session, mock_provider_client, extract_payload +): + mock_provider_client.get_llm_providers = AsyncMock( + return_value={ + "providers": [SYSTEM_NODE, BYOM_NODE], + "page_info": {"hasNextPage": True, "endCursor": "c1"}, + } + ) + async with provider_session as session: + result = await session.call_tool( + "get_llm_providers", {"organization_uuid": "org-uuid-1", "first": 2} + ) + assert result.isError is False + mock_provider_client.get_llm_providers.assert_awaited_once_with( + "org-uuid-1", only_active=False, first=2, after=None + ) + payload = extract_payload(result) + assert payload["success"] is True + assert [p["type"] for p in payload["data"]["providers"]] == ["system", "byom"] + assert payload["pagination"] == { + "has_more": True, + "end_cursor": "c1", + "page_size": 2, + } + + +@pytest.mark.anyio +@pytest.mark.parametrize("provider_session", [None], indirect=True) +async def test_get_llm_providers_blank_org_uuid_rejected( + provider_session, mock_provider_client, extract_payload +): + async with provider_session as session: + result = await session.call_tool( + "get_llm_providers", {"organization_uuid": " "} + ) + payload = extract_payload(result) + assert payload["success"] is False + assert "organization_uuid" in tool_error_message(payload) + mock_provider_client.get_llm_providers.assert_not_awaited() + + +@pytest.mark.anyio +@pytest.mark.parametrize("provider_session", [None], indirect=True) +async def test_get_llm_providers_invalid_page_size_rejected( + provider_session, mock_provider_client, extract_payload +): + async with provider_session as session: + result = await session.call_tool( + "get_llm_providers", {"organization_uuid": "org-uuid-1", "first": 0} + ) + payload = extract_payload(result) + assert payload["success"] is False + assert payload["error"]["code"] == "INVALID_ARGUMENTS" + + +@pytest.mark.anyio +@pytest.mark.parametrize("provider_session", [None], indirect=True) +async def test_get_llm_providers_permission_denied_classified( + provider_session, mock_provider_client, extract_payload +): + mock_provider_client.get_llm_providers = AsyncMock( + side_effect=permission_denied_error() + ) + async with provider_session as session: + result = await session.call_tool( + "get_llm_providers", {"organization_uuid": "org-uuid-1"} + ) + payload = extract_payload(result) + assert payload["success"] is False + assert payload["error"]["code"] == "PERMISSION_DENIED" + assert payload["error"]["details"]["kind"] == "permission_denied" + assert payload["error"]["details"]["correlation_id"] == "corr-9" + + +@pytest.mark.anyio +@pytest.mark.parametrize("provider_session", [None], indirect=True) +async def test_get_available_ai_models_success( + provider_session, mock_provider_client, extract_payload +): + mock_provider_client.get_available_ai_models = AsyncMock( + return_value=["gpt-4o", "gpt-4o-mini"] + ) + async with provider_session as session: + result = await session.call_tool( + "get_available_ai_models", {"provider_name": "openai"} + ) + payload = extract_payload(result) + assert payload["success"] is True + assert payload["data"]["models"] == ["gpt-4o", "gpt-4o-mini"] + mock_provider_client.get_available_ai_models.assert_awaited_once_with("openai") + + +@pytest.mark.anyio +@pytest.mark.parametrize("provider_session", [None], indirect=True) +async def test_get_available_ai_models_invalid_enum_surfaces_api_error( + provider_session, mock_provider_client, extract_payload +): + mock_provider_client.get_available_ai_models = AsyncMock( + side_effect=TransportQueryError( + "bad enum", + errors=[ + { + "message": "Invalid provider name", + "extensions": {"code": "INVALID_ARGUMENTS"}, + } + ], + ) + ) + async with provider_session as session: + result = await session.call_tool( + "get_available_ai_models", {"provider_name": "bogus"} + ) + payload = extract_payload(result) + assert payload["success"] is False + assert payload["error"]["details"]["kind"] == "invalid_arguments" + + +@pytest.mark.anyio +@pytest.mark.parametrize("provider_session", [None], indirect=True) +async def test_get_default_llm_provider_defaults_to_organization( + provider_session, mock_provider_client, extract_payload +): + mock_provider_client.get_default_llm_provider = AsyncMock(return_value=BYOM_NODE) + async with provider_session as session: + result = await session.call_tool( + "get_default_llm_provider", {"owner_id": "123456"} + ) + payload = extract_payload(result) + assert payload["success"] is True + assert payload["data"]["provider"]["id"] == "42" + mock_provider_client.get_default_llm_provider.assert_awaited_once_with( + "123456", owner_type="organization" + ) + + +@pytest.mark.anyio +@pytest.mark.parametrize("provider_session", [None], indirect=True) +async def test_get_llm_provider_dependencies_success( + provider_session, mock_provider_client, extract_payload +): + mock_provider_client.get_llm_provider_dependencies = AsyncMock( + return_value={ + "dependencies": [{"ownerId": "1", "ownerType": "organization"}], + "page_info": {"hasNextPage": False, "endCursor": None}, + "total_count": 1, + } + ) + async with provider_session as session: + result = await session.call_tool( + "get_llm_provider_dependencies", + {"provider_id": "42", "organization_uuid": "org-uuid-1"}, + ) + payload = extract_payload(result) + assert payload["success"] is True + assert payload["data"]["total_count"] == 1 + assert payload["data"]["dependencies"][0]["ownerType"] == "organization" + assert payload["pagination"]["has_more"] is False + + +@pytest.mark.anyio +@pytest.mark.parametrize("provider_session", [None], indirect=True) +async def test_validate_llm_provider_access_green( + provider_session, mock_provider_client, extract_payload +): + mock_provider_client.validate_llm_provider_access = AsyncMock( + return_value={ + "ok": True, + "system_providers_visible": True, + "custom_providers_visible": False, + "provider_count": 3, + "note": "Read access confirmed. This proves list/read access only...", + } + ) + async with provider_session as session: + result = await session.call_tool( + "validate_llm_provider_access", {"organization_uuid": "org-uuid-1"} + ) + payload = extract_payload(result) + assert payload["success"] is True + assert payload["data"]["ok"] is True + assert payload["data"]["system_providers_visible"] is True + + +@pytest.mark.anyio +@pytest.mark.parametrize("provider_session", [None], indirect=True) +async def test_validate_llm_provider_access_failure_maps_problem( + provider_session, mock_provider_client, extract_payload +): + mock_provider_client.validate_llm_provider_access = AsyncMock( + return_value={ + "ok": False, + "problem": { + "kind": "permission_denied", + "message": "Permission denied", + "code": "PERMISSION_DENIED", + "correlation_id": "corr-2", + }, + } + ) + async with provider_session as session: + result = await session.call_tool( + "validate_llm_provider_access", {"organization_uuid": "org-uuid-1"} + ) + payload = extract_payload(result) + assert payload["success"] is False + assert payload["error"]["code"] == "PERMISSION_DENIED" + assert payload["error"]["details"]["kind"] == "permission_denied" + assert payload["error"]["details"]["correlation_id"] == "corr-2" + + +@pytest.mark.anyio +@pytest.mark.parametrize("provider_session", [None], indirect=True) +async def test_transport_failure_without_graphql_errors_falls_back_to_str( + provider_session, mock_provider_client, extract_payload +): + mock_provider_client.get_llm_providers = AsyncMock( + side_effect=RuntimeError("socket closed") + ) + async with provider_session as session: + result = await session.call_tool( + "get_llm_providers", {"organization_uuid": "org-uuid-1"} + ) + payload = extract_payload(result) + assert payload["success"] is False + assert "socket closed" in tool_error_message(payload) diff --git a/packages/sdk/src/pipefy_sdk/__init__.py b/packages/sdk/src/pipefy_sdk/__init__.py index 4a513703..5d3ff4db 100644 --- a/packages/sdk/src/pipefy_sdk/__init__.py +++ b/packages/sdk/src/pipefy_sdk/__init__.py @@ -11,6 +11,12 @@ filter_fields_by_definitions, skipped_field_ids, ) +from pipefy_sdk.graphql_problem import ( + GraphQLProblem, + GraphQLProblemKind, + classify_exception, + classify_graphql_error_dicts, +) from pipefy_sdk.models import ( Attachment, AttachmentTarget, @@ -64,7 +70,11 @@ from pipefy_sdk.services.types import ( AiAgentGraphPayload, CardSearch, + LlmProviderPayload, + LlmProvidersResult, MePayload, + ProviderAccessProbeResult, + ProviderDependenciesResult, copy_card_search, ) from pipefy_sdk.settings import PipefySettings @@ -100,6 +110,14 @@ "CreatePortalElementInput", "CreateSendTaskAutomationInput", "DeleteCommentInput", + "GraphQLProblem", + "GraphQLProblemKind", + "LlmProviderPayload", + "LlmProvidersResult", + "ProviderAccessProbeResult", + "ProviderDependenciesResult", + "classify_exception", + "classify_graphql_error_dicts", "download_bytes", "filter_editable_field_definitions", "filter_fields_by_definitions", diff --git a/packages/sdk/src/pipefy_sdk/client.py b/packages/sdk/src/pipefy_sdk/client.py index 1dcd22fd..97d7bd44 100644 --- a/packages/sdk/src/pipefy_sdk/client.py +++ b/packages/sdk/src/pipefy_sdk/client.py @@ -50,6 +50,10 @@ ) from pipefy_sdk.services.automation_service import AutomationService from pipefy_sdk.services.card_service import CardService +from pipefy_sdk.services.llm_provider_service import ( + DEFAULT_PROVIDER_PAGE_SIZE, + LlmProviderService, +) from pipefy_sdk.services.member_service import MemberService from pipefy_sdk.services.observability_service import ( AUTOMATION_EXECUTION_METRICS_MAX_PAGE_SIZE, @@ -76,7 +80,11 @@ AiAgentGraphPayload, AutomationServiceResult, CardSearch, + LlmProviderPayload, + LlmProvidersResult, MePayload, + ProviderAccessProbeResult, + ProviderDependenciesResult, ToggleAgentStatusResult, ) from pipefy_sdk.services.user_service import UserService @@ -268,6 +276,7 @@ def _wire(self, ex: Executors, settings: PipefySettings) -> None: ) self._automation_service = AutomationService(executor=ex.public) self._ai_agent_service = AiAgentService(executor=ex.public) + self._llm_provider_service = LlmProviderService(executor=ex.public) self._observability_service = ObservabilityService(executor=ex.public) self._report_service = ReportService(executor=ex.public) self._organization_service = OrganizationService(executor=ex.public) @@ -910,6 +919,52 @@ async def delete_ai_agent(self, agent_uuid: str) -> dict: """Delete an AI Agent by UUID (permanent).""" return await self._ai_agent_service.delete_agent(agent_uuid) + async def get_llm_providers( + self, + organization_uuid: str, + *, + only_active: bool = False, + first: int = DEFAULT_PROVIDER_PAGE_SIZE, + after: str | None = None, + ) -> LlmProvidersResult: + """List the organization's LLM providers (custom + Pipefy-managed system).""" + return await self._llm_provider_service.get_llm_providers( + organization_uuid, only_active=only_active, first=first, after=after + ) + + async def get_available_ai_models(self, provider_name: str) -> list[str]: + """List the model names a provider vendor exposes (ProviderName enum).""" + return await self._llm_provider_service.get_available_ai_models(provider_name) + + async def get_default_llm_provider( + self, owner_id: str, *, owner_type: str = "organization" + ) -> LlmProviderPayload: + """Resolve the default LLM provider for an owner (org default by default).""" + return await self._llm_provider_service.get_default_llm_provider( + owner_id, owner_type=owner_type + ) + + async def get_llm_provider_dependencies( + self, + provider_id: str, + organization_uuid: str, + *, + first: int = DEFAULT_PROVIDER_PAGE_SIZE, + after: str | None = None, + ) -> ProviderDependenciesResult: + """List the owners that depend on an LLM provider.""" + return await self._llm_provider_service.get_llm_provider_dependencies( + provider_id, organization_uuid, first=first, after=after + ) + + async def validate_llm_provider_access( + self, organization_uuid: str + ) -> ProviderAccessProbeResult: + """Probe LLM provider read access; classifies errors instead of raising.""" + return await self._llm_provider_service.validate_llm_provider_access( + organization_uuid + ) + async def create_ai_agent( self, agent_input: CreateAiAgentInput ) -> AgentServiceResult: diff --git a/packages/sdk/src/pipefy_sdk/graphql_problem.py b/packages/sdk/src/pipefy_sdk/graphql_problem.py new file mode 100644 index 00000000..b8135d20 --- /dev/null +++ b/packages/sdk/src/pipefy_sdk/graphql_problem.py @@ -0,0 +1,154 @@ +"""Standalone GraphQL error classifier shared across the SDK, CLI, and MCP. + +Maps a Pipefy GraphQL error payload (or the ``TransportQueryError`` that +carries it) onto a small closed set of structured problems: permission denied, +not found, invalid arguments, feature not enabled, or an unclassified runtime +error. Provider probes and CLI write-gating consume this instead of each +re-deriving ``except``-mapping locally. + +It lives in the SDK on purpose. The richer MCP enrichment +(``pipefy_mcp.tools.graphql_error_helpers``) imports MCP settings and the MCP +response envelope, so neither the SDK nor the CLI can import it. This module +depends on nothing beyond the standard library and the ``gql`` error shape the +SDK already surfaces, so all three layers can share one classifier. + +Classification reads ``extensions.code`` first (the authoritative signal the +Pipefy GraphQL layer attaches) and falls back to substring markers on the +message only for not-found, where legacy paths omit a code. An error the +classifier does not recognize maps to :attr:`GraphQLProblemKind.RUNTIME` rather +than being forced into a more specific bucket. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Any + +__all__ = [ + "GraphQLProblem", + "GraphQLProblemKind", + "classify_graphql_error_dicts", + "classify_exception", +] + + +class GraphQLProblemKind(str, Enum): + """The closed set of structured problems the classifier resolves. + + ``FEATURE_NOT_ENABLED`` is reserved for the (rare) explicit feature-flag + error codes; the provider read surface signals a disabled feature by an + empty system-provider list rather than an error, so probes infer that case + separately from this classification. + """ + + PERMISSION_DENIED = "permission_denied" + NOT_FOUND = "not_found" + INVALID_ARGUMENTS = "invalid_arguments" + FEATURE_NOT_ENABLED = "feature_not_enabled" + RUNTIME = "runtime" + + +# extensions.code values the Pipefy GraphQL API attaches, grouped by the +# problem they map to (PERMISSION_DENIED, RESOURCE_NOT_FOUND / RECORD_NOT_FOUND, +# RECORD_INVALID, ...). The extra synonyms guard against endpoints that use the +# more generic GraphQL spellings. +_PERMISSION_CODES = frozenset({"PERMISSION_DENIED", "FORBIDDEN", "UNAUTHORIZED"}) +_NOT_FOUND_CODES = frozenset({"NOT_FOUND", "RESOURCE_NOT_FOUND", "RECORD_NOT_FOUND"}) +_INVALID_ARGUMENT_CODES = frozenset( + {"INVALID_ARGUMENTS", "BAD_USER_INPUT", "RECORD_INVALID", "ARGUMENT_ERROR"} +) +_FEATURE_CODES = frozenset( + {"FEATURE_NOT_ENABLED", "FEATURE_DISABLED", "FEATURE_UNAVAILABLE"} +) + +_NOT_FOUND_MESSAGE_MARKERS = ( + "not found", + "does not exist", + "doesn't exist", + "couldn't find", +) + +_CODE_TO_KIND: dict[str, GraphQLProblemKind] = { + **{c: GraphQLProblemKind.PERMISSION_DENIED for c in _PERMISSION_CODES}, + **{c: GraphQLProblemKind.NOT_FOUND for c in _NOT_FOUND_CODES}, + **{c: GraphQLProblemKind.INVALID_ARGUMENTS for c in _INVALID_ARGUMENT_CODES}, + **{c: GraphQLProblemKind.FEATURE_NOT_ENABLED for c in _FEATURE_CODES}, +} + + +@dataclass(frozen=True) +class GraphQLProblem: + """A classified GraphQL error: what kind it is, plus the raw diagnostics. + + ``code`` and ``correlation_id`` are carried verbatim from ``extensions`` so + callers can surface them for support without re-parsing the error. + """ + + kind: GraphQLProblemKind + message: str + code: str | None = None + correlation_id: str | None = None + + +def _kind_from_code(code: str | None) -> GraphQLProblemKind | None: + if not code: + return None + return _CODE_TO_KIND.get(code.strip().upper()) + + +def _kind_from_message(message: str) -> GraphQLProblemKind | None: + lowered = message.lower() + if any(marker in lowered for marker in _NOT_FOUND_MESSAGE_MARKERS): + return GraphQLProblemKind.NOT_FOUND + return None + + +def classify_graphql_error_dicts( + errors: list[dict[str, Any]], +) -> GraphQLProblem | None: + """Classify the first error in a GraphQL ``errors`` list. + + Returns ``None`` for an empty list. A recognized ``extensions.code`` wins; + otherwise a not-found substring marker on the message is honored; anything + else classifies as :attr:`GraphQLProblemKind.RUNTIME` (still a problem, just + unclassified) so callers always get a structured result for a real error. + """ + if not errors: + return None + first = errors[0] if isinstance(errors[0], dict) else {} + extensions = ( + first.get("extensions") if isinstance(first.get("extensions"), dict) else {} + ) + code_raw = extensions.get("code") + code = str(code_raw) if code_raw not in (None, "") else None + correlation_raw = extensions.get("correlation_id") + correlation_id = str(correlation_raw) if correlation_raw not in (None, "") else None + message = str(first.get("message") or "Unknown error") + + kind = ( + _kind_from_code(code) + or _kind_from_message(message) + or GraphQLProblemKind.RUNTIME + ) + return GraphQLProblem( + kind=kind, + message=message, + code=code, + correlation_id=correlation_id, + ) + + +def classify_exception(exc: BaseException) -> GraphQLProblem | None: + """Classify an exception raised by the GraphQL executor. + + Handles the public endpoint's ``TransportQueryError`` (and any exception + that duck-types it) by reading its ``errors`` list. Returns ``None`` when + the exception carries no GraphQL error dicts, so callers can distinguish a + classified GraphQL problem from a transport/network failure they should + re-raise or report as-is. + """ + errors = getattr(exc, "errors", None) + if isinstance(errors, list) and errors and all(isinstance(e, dict) for e in errors): + return classify_graphql_error_dicts(errors) + return None diff --git a/packages/sdk/src/pipefy_sdk/queries/llm_provider_queries.py b/packages/sdk/src/pipefy_sdk/queries/llm_provider_queries.py new file mode 100644 index 00000000..c8963678 --- /dev/null +++ b/packages/sdk/src/pipefy_sdk/queries/llm_provider_queries.py @@ -0,0 +1,126 @@ +"""GraphQL queries for LLM provider discovery (read-only). + +Covers the organization's provider inventory (custom + Pipefy-managed system +providers in one union), the models a vendor exposes, the organization/owner +default provider, and a provider's dependents. All are reads; provider +mutations ship separately. +""" + +from __future__ import annotations + +from gql import gql + +# allLlmProvidersByOrganization returns a union connection whose nodes are +# either LlmProvider (custom / byom) or SystemLlmProvider (Pipefy-managed). +# __typename plus the shared `type` field (ProviderType: byom | system) both +# discriminate; configuration is JSON with secrets redacted server-side. +GET_LLM_PROVIDERS_QUERY = gql( + """ + query allLlmProvidersByOrganization( + $organizationUuid: String! + $onlyActiveProviders: Boolean + $first: Int + $after: String + ) { + allLlmProvidersByOrganization( + organizationUuid: $organizationUuid + onlyActiveProviders: $onlyActiveProviders + first: $first + after: $after + ) { + edges { + node { + __typename + ... on LlmProvider { + id + name + type + active + organizationDefault + configuration + } + ... on SystemLlmProvider { + id + name + type + organizationDefault + systemDefault + state + description + aiCredits + deprecationDate + configuration + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } + """ +) + +GET_AVAILABLE_AI_MODELS_QUERY = gql( + """ + query availableAiModels($providerName: ProviderName!) { + availableAiModels(providerName: $providerName) + } + """ +) + +# defaultLlmProvider resolves a single provider for one owner (organization, +# assistant, or behavior); it returns the LlmProvider shape even for a system +# default, with `type` carrying the discriminator. +GET_DEFAULT_LLM_PROVIDER_QUERY = gql( + """ + query defaultLlmProvider($ownerType: OwnerProvider!, $ownerId: String!) { + defaultLlmProvider(ownerType: $ownerType, ownerId: $ownerId) { + id + name + type + active + organizationDefault + configuration + } + } + """ +) + +GET_PROVIDER_DEPENDENCIES_QUERY = gql( + """ + query providerDependencies( + $providerId: ID! + $organizationUuid: String! + $first: Int + $after: String + ) { + providerDependencies( + providerId: $providerId + organizationUuid: $organizationUuid + first: $first + after: $after + ) { + edges { + node { + ownerId + ownerType + } + } + pageInfo { + hasNextPage + endCursor + } + totalCount + } + } + """ +) + +__all__ = [ + "GET_AVAILABLE_AI_MODELS_QUERY", + "GET_DEFAULT_LLM_PROVIDER_QUERY", + "GET_LLM_PROVIDERS_QUERY", + "GET_PROVIDER_DEPENDENCIES_QUERY", +] diff --git a/packages/sdk/src/pipefy_sdk/services/llm_provider_service.py b/packages/sdk/src/pipefy_sdk/services/llm_provider_service.py new file mode 100644 index 00000000..27d76c24 --- /dev/null +++ b/packages/sdk/src/pipefy_sdk/services/llm_provider_service.py @@ -0,0 +1,247 @@ +"""Service for LLM provider discovery reads and the read-access probe.""" + +from __future__ import annotations + +from typing import Any + +from pipefy_sdk.graphql_executor import GraphQLExecutor +from pipefy_sdk.graphql_problem import ( + GraphQLProblem, + GraphQLProblemKind, + classify_graphql_error_dicts, +) +from pipefy_sdk.queries.llm_provider_queries import ( + GET_AVAILABLE_AI_MODELS_QUERY, + GET_DEFAULT_LLM_PROVIDER_QUERY, + GET_LLM_PROVIDERS_QUERY, + GET_PROVIDER_DEPENDENCIES_QUERY, +) +from pipefy_sdk.services.types import ( + LlmProviderPayload, + LlmProvidersResult, + ProviderAccessProbeResult, + ProviderDependenciesResult, +) +from pipefy_sdk.utils.relay import unwrap_relay_connection_nodes + +DEFAULT_PROVIDER_PAGE_SIZE = 50 + +_PROBE_FEATURE_NOTE = ( + "No system providers returned: the organization may not have Pipefy-managed " + "system models enabled; this is not by itself a permission problem." +) +_PROBE_READ_ONLY_NOTE = ( + "Read access confirmed. This proves list/read access only, never write " + "entitlement; provider mutations may still be denied." +) + + +def _require_non_blank(value: str, name: str) -> str: + stripped = value.strip() + if not stripped: + raise ValueError(f"{name} must be a non-empty string") + return stripped + + +def _problem_dict(problem: GraphQLProblem) -> dict[str, Any]: + """Project a classified problem onto the probe's plain-dict shape.""" + return { + "kind": problem.kind.value, + "message": problem.message, + "code": problem.code, + "correlation_id": problem.correlation_id, + } + + +class LlmProviderService: + """Read-only LLM provider discovery via GraphQL.""" + + def __init__(self, *, executor: GraphQLExecutor) -> None: + self._executor = executor + + async def get_llm_providers( + self, + organization_uuid: str, + *, + only_active: bool = False, + first: int = DEFAULT_PROVIDER_PAGE_SIZE, + after: str | None = None, + ) -> LlmProvidersResult: + """List all LLM providers available to the organization (custom + system). + + Args: + organization_uuid: Organization UUID (not the numeric id). + only_active: Only return active *custom* providers; system + providers are unaffected by this filter. + first: Page size (default 50). + after: Cursor from the previous page's ``page_info.endCursor``. + + Returns: + ``providers`` (union nodes; ``type`` is ``byom`` or ``system``) and + ``page_info``. ``configuration`` comes back with secret values + redacted server-side. + """ + variables: dict[str, Any] = { + "organizationUuid": _require_non_blank( + organization_uuid, "organization_uuid" + ), + "onlyActiveProviders": only_active, + "first": first, + } + if after is not None: + variables["after"] = after + response = await self._executor.execute_query( + GET_LLM_PROVIDERS_QUERY, variables + ) + connection = response.get("allLlmProvidersByOrganization") + providers = unwrap_relay_connection_nodes(connection) + page_info = connection.get("pageInfo") if isinstance(connection, dict) else None + return {"providers": providers, "page_info": page_info} + + async def get_available_ai_models(self, provider_name: str) -> list[str]: + """List the model names a provider vendor exposes. + + Args: + provider_name: ProviderName enum value: ``openai``, ``azure_openai``, + ``amazon_bedrock``, ``custom``, ``google_vertex_ai``, + ``oracle_oci``, or ``anthropic``. The API validates membership. + + Returns: + Model name strings (empty when the API returns null). + """ + name = _require_non_blank(provider_name, "provider_name") + response = await self._executor.execute_query( + GET_AVAILABLE_AI_MODELS_QUERY, {"providerName": name} + ) + models = response.get("availableAiModels") + return [str(m) for m in models] if isinstance(models, list) else [] + + async def get_default_llm_provider( + self, owner_id: str, *, owner_type: str = "organization" + ) -> LlmProviderPayload: + """Resolve the default LLM provider for one owner. + + Args: + owner_id: Owner identifier. For ``owner_type="organization"`` this + is the **numeric organization id** (not the UUID). + owner_type: OwnerProvider enum value (``organization`` (default), + ``assistant``, or ``behavior``). + + Returns: + The provider dict (``type`` discriminates byom/system); empty dict + when the API resolves no default. + """ + variables = { + "ownerType": _require_non_blank(owner_type, "owner_type"), + "ownerId": _require_non_blank(owner_id, "owner_id"), + } + response = await self._executor.execute_query( + GET_DEFAULT_LLM_PROVIDER_QUERY, variables + ) + provider = response.get("defaultLlmProvider") + return provider if isinstance(provider, dict) else {} + + async def get_llm_provider_dependencies( + self, + provider_id: str, + organization_uuid: str, + *, + first: int = DEFAULT_PROVIDER_PAGE_SIZE, + after: str | None = None, + ) -> ProviderDependenciesResult: + """List the owners that depend on a provider (blockers for removal). + + Args: + provider_id: Provider ID (as returned by ``get_llm_providers``). + organization_uuid: Organization UUID (not the numeric id). + first: Page size (default 50). + after: Cursor from the previous page's ``page_info.endCursor``. + + Returns: + ``dependencies`` (``ownerId``/``ownerType`` pairs), ``page_info``, + and ``total_count``. + """ + variables: dict[str, Any] = { + "providerId": _require_non_blank(provider_id, "provider_id"), + "organizationUuid": _require_non_blank( + organization_uuid, "organization_uuid" + ), + "first": first, + } + if after is not None: + variables["after"] = after + response = await self._executor.execute_query( + GET_PROVIDER_DEPENDENCIES_QUERY, variables + ) + connection = response.get("providerDependencies") + dependencies = unwrap_relay_connection_nodes(connection) + page_info = connection.get("pageInfo") if isinstance(connection, dict) else None + total_count = ( + connection.get("totalCount") if isinstance(connection, dict) else None + ) + return { + "dependencies": dependencies, + "page_info": page_info, + "total_count": total_count, + } + + async def validate_llm_provider_access( + self, organization_uuid: str + ) -> ProviderAccessProbeResult: + """Probe LLM provider *read* access for an organization. + + Runs the provider list query (one default-size page; the API orders + system providers before custom ones, so the first page is authoritative + for system-provider visibility) through the partial-success executor + and classifies any GraphQL errors instead of raising. + A green result proves read access only, never write entitlement, and an + empty system-provider list can mean the feature is not enabled for the + organization rather than a permission problem — both facts are spelled + out in ``note``. + + Note: the list query is gated by a weaker permission than the + models/default/dependencies reads, which require provider-management + rights; a green probe does not guarantee those three succeed. + """ + org_uuid = _require_non_blank(organization_uuid, "organization_uuid") + variables = { + "organizationUuid": org_uuid, + "onlyActiveProviders": False, + "first": DEFAULT_PROVIDER_PAGE_SIZE, + } + result = await self._executor.execute(GET_LLM_PROVIDERS_QUERY, variables) + connection = result.data.get("allLlmProvidersByOrganization") + if connection is None: + problem = classify_graphql_error_dicts(result.errors) + if problem is None: + problem_dict: dict[str, Any] = { + "kind": GraphQLProblemKind.RUNTIME.value, + "message": "Query returned no data and no errors.", + } + else: + problem_dict = _problem_dict(problem) + return {"ok": False, "problem": problem_dict} + + providers = unwrap_relay_connection_nodes(connection) + system_visible = any(p.get("type") == "system" for p in providers) + custom_visible = any(p.get("type") == "byom" for p in providers) + note = _PROBE_READ_ONLY_NOTE + if not system_visible: + note = f"{note} {_PROBE_FEATURE_NOTE}" + probe: ProviderAccessProbeResult = { + "ok": True, + "system_providers_visible": system_visible, + "custom_providers_visible": custom_visible, + "provider_count": len(providers), + "note": note, + } + # A response can carry readable data alongside per-node errors; a green + # probe still surfaces them so partial denial is never read as full access. + partial = classify_graphql_error_dicts(result.errors) + if partial is not None: + probe["note"] = ( + f"{probe['note']} The response also carried GraphQL errors; " + "see problem." + ) + probe["problem"] = _problem_dict(partial) + return probe diff --git a/packages/sdk/src/pipefy_sdk/services/types.py b/packages/sdk/src/pipefy_sdk/services/types.py index 8bb55bb5..786fc00e 100644 --- a/packages/sdk/src/pipefy_sdk/services/types.py +++ b/packages/sdk/src/pipefy_sdk/services/types.py @@ -60,3 +60,69 @@ class MePayload(TypedDict): email: str name: str | None + + +class LlmProviderPayload(TypedDict, total=False): + """One provider node from the LLM provider union (custom or system). + + Covers both union members: ``__typename``/``type`` discriminate (custom = + ``LlmProvider``/``byom``, system = ``SystemLlmProvider``/``system``). + System-only keys (``systemDefault``, ``state``, ``aiCredits``, + ``deprecationDate``, ``description``) and the custom-only ``active`` are + present only for their member. ``configuration`` is a JSON object with + secret values redacted server-side (placeholders, not real secrets). + """ + + __typename: str + id: str + name: str | None + type: str + active: bool + organizationDefault: bool + systemDefault: bool + state: str + description: str | None + aiCredits: int + deprecationDate: str | None + configuration: dict[str, Any] + + +class LlmProvidersResult(TypedDict): + """Unwrapped page of the organization's LLM providers with paging cursor.""" + + providers: list[LlmProviderPayload] + page_info: dict[str, Any] | None + + +class ProviderDependencyPayload(TypedDict, total=False): + """One dependent of a provider: an owner that references it.""" + + ownerId: str + ownerType: str + + +class ProviderDependenciesResult(TypedDict): + """Unwrapped page of a provider's dependents with paging cursor and total.""" + + dependencies: list[ProviderDependencyPayload] + page_info: dict[str, Any] | None + total_count: int | None + + +class ProviderAccessProbeResult(TypedDict, total=False): + """Outcome of the LLM-provider read-access probe. + + ``ok`` is True when the list query succeeded (proves *read* access only, + never write entitlement). On success, ``system_providers_visible`` reports + whether any Pipefy-managed system provider was returned; when False, + Pipefy-managed system models may simply not be enabled for the organization + rather than access being denied. On failure, ``problem`` carries the + classified GraphQL problem (kind/message/code/correlation_id). + """ + + ok: bool + system_providers_visible: bool + custom_providers_visible: bool + provider_count: int + note: str + problem: dict[str, Any] diff --git a/packages/sdk/tests/services/test_llm_provider_service.py b/packages/sdk/tests/services/test_llm_provider_service.py new file mode 100644 index 00000000..66a8d917 --- /dev/null +++ b/packages/sdk/tests/services/test_llm_provider_service.py @@ -0,0 +1,391 @@ +"""Unit tests for LlmProviderService and the shared GraphQL problem classifier.""" + +from __future__ import annotations + +import pytest +from _shared.mock_clients import mock_executor + +from pipefy_sdk.graphql_executor import GraphQLResult +from pipefy_sdk.graphql_problem import ( + GraphQLProblemKind, + classify_exception, + classify_graphql_error_dicts, +) +from pipefy_sdk.services.llm_provider_service import LlmProviderService + +BYOM_NODE = { + "__typename": "LlmProvider", + "id": "42", + "name": "Azure custom", + "type": "byom", + "active": True, + "organizationDefault": False, + "configuration": {"provider": "azure_openai", "auth": {"accessToken": "__REDACTED__"}}, +} + +SYSTEM_NODE = { + "__typename": "SystemLlmProvider", + "id": "7", + "name": "Pipefy GPT", + "type": "system", + "organizationDefault": True, + "systemDefault": True, + "state": "active", + "description": "Managed model", + "aiCredits": 2, + "deprecationDate": None, + "configuration": {"model": "gpt-4o"}, +} + + +def providers_connection(nodes: list[dict], *, has_next: bool = False, cursor: str | None = None) -> dict: + return { + "allLlmProvidersByOrganization": { + "edges": [{"node": n} for n in nodes], + "pageInfo": {"hasNextPage": has_next, "endCursor": cursor}, + } + } + + +class TestGetLlmProviders: + @pytest.mark.anyio + async def test_returns_both_union_members_and_page_info(self): + executor = mock_executor( + providers_connection([SYSTEM_NODE, BYOM_NODE], has_next=True, cursor="c1") + ) + service = LlmProviderService(executor=executor) + + result = await service.get_llm_providers("org-uuid-1") + + assert result["providers"] == [SYSTEM_NODE, BYOM_NODE] + assert result["page_info"] == {"hasNextPage": True, "endCursor": "c1"} + types = {p["type"] for p in result["providers"]} + assert types == {"system", "byom"} + + @pytest.mark.anyio + async def test_pagination_variables_default_and_cursor(self): + executor = mock_executor(providers_connection([])) + service = LlmProviderService(executor=executor) + + await service.get_llm_providers("org-uuid-1") + _, variables = executor.execute_query.await_args.args + assert variables == { + "organizationUuid": "org-uuid-1", + "onlyActiveProviders": False, + "first": 50, + } + + await service.get_llm_providers( + "org-uuid-1", only_active=True, first=10, after="cursor-a" + ) + _, variables = executor.execute_query.await_args.args + assert variables == { + "organizationUuid": "org-uuid-1", + "onlyActiveProviders": True, + "first": 10, + "after": "cursor-a", + } + + @pytest.mark.anyio + async def test_null_connection_yields_empty_page(self): + executor = mock_executor({"allLlmProvidersByOrganization": None}) + service = LlmProviderService(executor=executor) + + result = await service.get_llm_providers("org-uuid-1") + + assert result == {"providers": [], "page_info": None} + + @pytest.mark.anyio + async def test_blank_org_uuid_rejected_before_wire(self): + executor = mock_executor({}) + service = LlmProviderService(executor=executor) + + with pytest.raises(ValueError, match="organization_uuid"): + await service.get_llm_providers(" ") + executor.execute_query.assert_not_awaited() + + +class TestGetAvailableAiModels: + @pytest.mark.anyio + async def test_returns_model_list(self): + executor = mock_executor({"availableAiModels": ["gpt-4o", "gpt-4o-mini"]}) + service = LlmProviderService(executor=executor) + + models = await service.get_available_ai_models("openai") + + assert models == ["gpt-4o", "gpt-4o-mini"] + _, variables = executor.execute_query.await_args.args + assert variables == {"providerName": "openai"} + + @pytest.mark.anyio + async def test_null_payload_yields_empty_list(self): + executor = mock_executor({"availableAiModels": None}) + service = LlmProviderService(executor=executor) + + assert await service.get_available_ai_models("anthropic") == [] + + @pytest.mark.anyio + async def test_blank_provider_name_rejected(self): + service = LlmProviderService(executor=mock_executor({})) + with pytest.raises(ValueError, match="provider_name"): + await service.get_available_ai_models("") + + +class TestGetDefaultLlmProvider: + @pytest.mark.anyio + async def test_organization_default_by_numeric_id(self): + executor = mock_executor({"defaultLlmProvider": BYOM_NODE}) + service = LlmProviderService(executor=executor) + + provider = await service.get_default_llm_provider("123456") + + assert provider == BYOM_NODE + _, variables = executor.execute_query.await_args.args + assert variables == {"ownerType": "organization", "ownerId": "123456"} + + @pytest.mark.anyio + async def test_custom_owner_type_passthrough(self): + executor = mock_executor({"defaultLlmProvider": None}) + service = LlmProviderService(executor=executor) + + provider = await service.get_default_llm_provider( + "beh-1", owner_type="behavior" + ) + + assert provider == {} + _, variables = executor.execute_query.await_args.args + assert variables == {"ownerType": "behavior", "ownerId": "beh-1"} + + @pytest.mark.anyio + async def test_blank_owner_id_rejected(self): + service = LlmProviderService(executor=mock_executor({})) + with pytest.raises(ValueError, match="owner_id"): + await service.get_default_llm_provider(" ") + + +class TestGetLlmProviderDependencies: + @pytest.mark.anyio + async def test_returns_dependencies_page_info_and_total(self): + executor = mock_executor( + { + "providerDependencies": { + "edges": [ + {"node": {"ownerId": "1", "ownerType": "organization"}}, + {"node": {"ownerId": "77", "ownerType": "assistant"}}, + ], + "pageInfo": {"hasNextPage": False, "endCursor": "z"}, + "totalCount": 2, + } + } + ) + service = LlmProviderService(executor=executor) + + result = await service.get_llm_provider_dependencies("42", "org-uuid-1") + + assert result["dependencies"] == [ + {"ownerId": "1", "ownerType": "organization"}, + {"ownerId": "77", "ownerType": "assistant"}, + ] + assert result["page_info"] == {"hasNextPage": False, "endCursor": "z"} + assert result["total_count"] == 2 + _, variables = executor.execute_query.await_args.args + assert variables == { + "providerId": "42", + "organizationUuid": "org-uuid-1", + "first": 50, + } + + @pytest.mark.anyio + async def test_after_cursor_forwarded(self): + executor = mock_executor({"providerDependencies": None}) + service = LlmProviderService(executor=executor) + + result = await service.get_llm_provider_dependencies( + "42", "org-uuid-1", first=5, after="dep-cursor" + ) + + assert result == {"dependencies": [], "page_info": None, "total_count": None} + _, variables = executor.execute_query.await_args.args + assert variables["after"] == "dep-cursor" + assert variables["first"] == 5 + + @pytest.mark.anyio + async def test_blank_provider_id_rejected(self): + service = LlmProviderService(executor=mock_executor({})) + with pytest.raises(ValueError, match="provider_id"): + await service.get_llm_provider_dependencies("", "org-uuid-1") + + +class TestValidateLlmProviderAccess: + @pytest.mark.anyio + async def test_green_probe_reports_visibility_and_read_only_note(self): + executor = mock_executor( + execute_result=GraphQLResult( + data=providers_connection([SYSTEM_NODE, BYOM_NODE]), errors=[] + ) + ) + service = LlmProviderService(executor=executor) + + probe = await service.validate_llm_provider_access("org-uuid-1") + + assert probe["ok"] is True + assert probe["system_providers_visible"] is True + assert probe["custom_providers_visible"] is True + assert probe["provider_count"] == 2 + assert "read access only" in probe["note"].lower() + + @pytest.mark.anyio + async def test_empty_system_list_notes_possible_feature_gap(self): + executor = mock_executor( + execute_result=GraphQLResult( + data=providers_connection([BYOM_NODE]), errors=[] + ) + ) + service = LlmProviderService(executor=executor) + + probe = await service.validate_llm_provider_access("org-uuid-1") + + assert probe["ok"] is True + assert probe["system_providers_visible"] is False + assert "not by itself a permission problem" in probe["note"] + + @pytest.mark.anyio + async def test_permission_denied_maps_to_structured_problem(self): + executor = mock_executor( + execute_result=GraphQLResult( + data={"allLlmProvidersByOrganization": None}, + errors=[ + { + "message": "Permission denied", + "extensions": { + "code": "PERMISSION_DENIED", + "correlation_id": "corr-1", + }, + } + ], + ) + ) + service = LlmProviderService(executor=executor) + + probe = await service.validate_llm_provider_access("org-uuid-1") + + assert probe["ok"] is False + assert probe["problem"]["kind"] == "permission_denied" + assert probe["problem"]["code"] == "PERMISSION_DENIED" + assert probe["problem"]["correlation_id"] == "corr-1" + + @pytest.mark.anyio + async def test_not_found_maps_to_structured_problem(self): + executor = mock_executor( + execute_result=GraphQLResult( + data={"allLlmProvidersByOrganization": None}, + errors=[ + { + "message": "Couldn't find Organization with uuid bogus", + "extensions": {"code": "RESOURCE_NOT_FOUND"}, + } + ], + ) + ) + service = LlmProviderService(executor=executor) + + probe = await service.validate_llm_provider_access("bogus") + + assert probe["ok"] is False + assert probe["problem"]["kind"] == "not_found" + + @pytest.mark.anyio + async def test_partial_errors_alongside_data_stay_visible(self): + executor = mock_executor( + execute_result=GraphQLResult( + data=providers_connection([SYSTEM_NODE]), + errors=[ + { + "message": "Permission denied", + "extensions": {"code": "PERMISSION_DENIED"}, + } + ], + ) + ) + service = LlmProviderService(executor=executor) + + probe = await service.validate_llm_provider_access("org-uuid-1") + + assert probe["ok"] is True + assert probe["problem"]["kind"] == "permission_denied" + assert "also carried GraphQL errors" in probe["note"] + + @pytest.mark.anyio + async def test_null_data_without_errors_reports_runtime_problem(self): + executor = mock_executor( + execute_result=GraphQLResult( + data={"allLlmProvidersByOrganization": None}, errors=[] + ) + ) + service = LlmProviderService(executor=executor) + + probe = await service.validate_llm_provider_access("org-uuid-1") + + assert probe["ok"] is False + assert probe["problem"]["kind"] == "runtime" + + +class TestGraphQLProblemClassifier: + def test_code_wins_over_message(self): + problem = classify_graphql_error_dicts( + [ + { + "message": "Record not found", + "extensions": {"code": "PERMISSION_DENIED"}, + } + ] + ) + assert problem is not None + assert problem.kind is GraphQLProblemKind.PERMISSION_DENIED + + def test_not_found_falls_back_to_message_marker(self): + problem = classify_graphql_error_dicts( + [{"message": "Couldn't find Repo with uuid x", "extensions": {}}] + ) + assert problem is not None + assert problem.kind is GraphQLProblemKind.NOT_FOUND + assert problem.code is None + + def test_invalid_arguments_and_feature_codes(self): + invalid = classify_graphql_error_dicts( + [{"message": "bad", "extensions": {"code": "RECORD_INVALID"}}] + ) + feature = classify_graphql_error_dicts( + [{"message": "off", "extensions": {"code": "FEATURE_NOT_ENABLED"}}] + ) + assert invalid is not None and invalid.kind is GraphQLProblemKind.INVALID_ARGUMENTS + assert feature is not None and feature.kind is GraphQLProblemKind.FEATURE_NOT_ENABLED + + def test_unrecognized_error_is_runtime(self): + problem = classify_graphql_error_dicts( + [{"message": "boom", "extensions": {"code": "SOMETHING_ELSE"}}] + ) + assert problem is not None + assert problem.kind is GraphQLProblemKind.RUNTIME + assert problem.code == "SOMETHING_ELSE" + + def test_empty_list_returns_none(self): + assert classify_graphql_error_dicts([]) is None + + def test_classify_exception_reads_errors_attribute(self): + class FakeTransportError(Exception): + def __init__(self): + super().__init__("gql error") + self.errors = [ + { + "message": "Permission denied", + "extensions": {"code": "PERMISSION_DENIED"}, + } + ] + + problem = classify_exception(FakeTransportError()) + assert problem is not None + assert problem.kind is GraphQLProblemKind.PERMISSION_DENIED + + def test_classify_exception_without_graphql_errors_returns_none(self): + assert classify_exception(RuntimeError("socket closed")) is None diff --git a/skills/ai-agents/pipefy-ai-agents/SKILL.md b/skills/ai-agents/pipefy-ai-agents/SKILL.md index 7967b671..c565d53e 100644 --- a/skills/ai-agents/pipefy-ai-agents/SKILL.md +++ b/skills/ai-agents/pipefy-ai-agents/SKILL.md @@ -135,7 +135,7 @@ Inside `actionParams.aiBehaviorParams` a behavior may also carry: | Max effort | `max_effort` | `capabilityType` is not checked against a fixed set — any value passes through and the API validates the enum on write, so new capabilities work without a toolkit update. Validation checks **shape only, not entitlement** — a capability may still require organization-level enablement to have any effect, so a green pre-flight does not guarantee the capability is active for the org. -- **`providerId`** / **`systemProviderId`** — pick the behavior's LLM provider. Set **at most one** (a behavior resolves to a single active provider). Discover valid IDs via the organization's AI settings in the Pipefy UI. +- **`providerId`** / **`systemProviderId`** — pick the behavior's LLM provider. Set **at most one** (a behavior resolves to a single active provider). Discover valid IDs with `get_llm_providers` (CLI: `pipefy ai-provider list`): each provider carries `type` — use `providerId` for a custom (`byom`) provider and `systemProviderId` for a Pipefy-managed (`system`) one. `get_default_llm_provider` shows what a behavior falls back to when neither is set. IDs are also visible in the organization's AI settings in the Pipefy UI. ```json { From 7cf1aa7d0576a8642f2338f4c086024481501a28 Mon Sep 17 00:00:00 2001 From: mocha06 <52426811+mocha06@users.noreply.github.com> Date: Fri, 17 Jul 2026 14:58:00 -0300 Subject: [PATCH 2/2] fix: format provider service tests; register ai-provider in skill-ref linter --- .github/workflows/scripts/lint_skill_refs.py | 1 + .../services/test_llm_provider_service.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.github/workflows/scripts/lint_skill_refs.py b/.github/workflows/scripts/lint_skill_refs.py index a26c79d1..f6ae66ca 100644 --- a/.github/workflows/scripts/lint_skill_refs.py +++ b/.github/workflows/scripts/lint_skill_refs.py @@ -15,6 +15,7 @@ { "agent", "ai-automation", + "ai-provider", "attachment", "audit", "automation", diff --git a/packages/sdk/tests/services/test_llm_provider_service.py b/packages/sdk/tests/services/test_llm_provider_service.py index 66a8d917..505d1b82 100644 --- a/packages/sdk/tests/services/test_llm_provider_service.py +++ b/packages/sdk/tests/services/test_llm_provider_service.py @@ -20,7 +20,10 @@ "type": "byom", "active": True, "organizationDefault": False, - "configuration": {"provider": "azure_openai", "auth": {"accessToken": "__REDACTED__"}}, + "configuration": { + "provider": "azure_openai", + "auth": {"accessToken": "__REDACTED__"}, + }, } SYSTEM_NODE = { @@ -38,7 +41,9 @@ } -def providers_connection(nodes: list[dict], *, has_next: bool = False, cursor: str | None = None) -> dict: +def providers_connection( + nodes: list[dict], *, has_next: bool = False, cursor: str | None = None +) -> dict: return { "allLlmProvidersByOrganization": { "edges": [{"node": n} for n in nodes], @@ -358,8 +363,13 @@ def test_invalid_arguments_and_feature_codes(self): feature = classify_graphql_error_dicts( [{"message": "off", "extensions": {"code": "FEATURE_NOT_ENABLED"}}] ) - assert invalid is not None and invalid.kind is GraphQLProblemKind.INVALID_ARGUMENTS - assert feature is not None and feature.kind is GraphQLProblemKind.FEATURE_NOT_ENABLED + assert ( + invalid is not None and invalid.kind is GraphQLProblemKind.INVALID_ARGUMENTS + ) + assert ( + feature is not None + and feature.kind is GraphQLProblemKind.FEATURE_NOT_ENABLED + ) def test_unrecognized_error_is_runtime(self): problem = classify_graphql_error_dicts(