diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..9afd1fb --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# ─── AI Provider (choose one) ─────────────────────────────────────────────── +# Set ONE of these keys. If both are set, Anthropic is used by default. + +ANTHROPIC_API_KEY=sk-ant-... +# OPENAI_API_KEY=sk-... + +# ─── Provider selection (optional) ────────────────────────────────────────── +# Force a specific provider when both keys are present: "anthropic" or "openai" +# WORKFLOWS_AI_PROVIDER=anthropic + +# ─── Model override (optional) ────────────────────────────────────────────── +# Default: claude-sonnet-4-6 (Anthropic) or gpt-5-nano (OpenAI) +# WORKFLOWS_AI_MODEL=claude-sonnet-4-6 + +# ─── Runtime ──────────────────────────────────────────────────────────────── +ENVIRONMENT=dev +# SENTRY_DSN= diff --git a/.gitignore b/.gitignore index ad509d6..c0c89fb 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,7 @@ dmypy.json .pyre/ # .vscode -.vscode/ \ No newline at end of file +.vscode/ + +# Generated connector outputs (test artifacts) +*-connector/ \ No newline at end of file diff --git a/README.md b/README.md index 5567f02..3a59e0a 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,34 @@ # Workflows CDK -A powerful CDK (Connector Development Kit) for building Stacksync Workflows Connectors with Python and Flask. +A CDK (Connector Development Kit) for building **Stacksync Workflows** connectors with **Python** and **Flask**, plus a **`workflows`** CLI for AI-assisted scaffolding and day-to-day tasks. ## Features -- 🚀 Automatic route discovery and registration file based (like in Next.js!) -- 🔒 Built-in error handling and Sentry integration -- 📦 Standardized request/response handling -- 🛠️ Error management with standardized error handling -- 🔄 Environment-aware configuration +**Runtime (library)** + +- 🚀 File-based route discovery under `src/modules/` (like the [app connector template](https://github.com/stacksyncdata/workflows-app-connector-template)) +- 🔒 Built-in error handling and optional Sentry integration +- 📦 Standardized `Request` / `Response` handling +- 🔄 Environment-aware configuration via `app_config.yaml` + +**CLI (`workflows` command)** + +- 🤖 Generate connectors or modules from a short natural-language description (`workflows create`) +- 📋 Interactive **main menu** when you run `workflows` with no arguments +- ➕ **Update** an existing project with new modules (`create --module-only`, also from the menu) +- ✅ **Validate** a connector folder (`validate`, with path prompt in the menu) +- 🖥️ **Run locally** from the menu (`run_dev.sh` or `python main.py`) +- 🌐 **Expose with ngrok** from the menu (starts the app on the configured port when needed, then tunnel) +- 📍 **Region** from the project `.env` shown in the UI (keep Studio and workflows in the same region) +- 📖 **`workflows guide`** — short help for run, ngrok, register, test +- 🔎 **`workflows list`** / **`workflows inspect`** — browse built-in capability manifests + +## Prerequisites + +- **Python 3.10+** +- **Docker** — recommended for `./run_dev.sh` (same flow as the official template) +- **ngrok** — optional; install from [ngrok.com](https://ngrok.com/download) if you use menu option **Expose with ngrok** +- **Anthropic or OpenAI API key** — optional; required for AI generation (otherwise use `--no-ai` or template matching) ## Installation @@ -16,195 +36,182 @@ A powerful CDK (Connector Development Kit) for building Stacksync Workflows Conn pip install workflows-cdk ``` -## Quick Start +## Quick start -1. Create a new project directory: +### 1. Create a connector ```bash -mkdir my-workflow-connector -cd my-workflow-connector +workflows create "Klaviyo connector with API key" ``` -2. Install the required dependencies: +Confirm at the preview, choose overwrite/version if prompted, then open the generated folder (name is derived from your description, e.g. `klaviyo-connector`). + +If you have no API key, the CLI can run **`workflows setup`** or fall back to template matching. + +### 2. Run it locally ```bash -pip install workflows-cdk flask pyyaml +cd klaviyo-connector # use your generated folder name +pip install -r requirements.txt +./run_dev.sh ``` -3. Create the basic project structure: +Default URL: `http://localhost:2003` (change port in `app_config.yaml` if needed). -``` -my-workflow-connector/ -├── main.py -├── app_config.yaml -├── requirements.txt -└── routes/ - └── hello/ - └── v1/ - └── route.py -``` +### 3. Expose and register in Stacksync + +- Use **Expose with ngrok** from the post-create menu, or run: `ngrok http 2003` (use your real port). +- Copy the **HTTPS** URL into **Stacksync Developer Studio**. +- Use the **same `REGION`** in Studio and in workflows as in the project `.env` (e.g. `REGION=besg`). + +If Studio says the URL already exists, start a **new** ngrok session for a new URL, or remove/edit the existing private app. + +### 4. (Optional) Open the full menu anytime -4. Set up your `app_config.yaml`: - -```yaml -app_settings: - app_type: "example" - app_name: "My Workflow Connector" - app_description: "A simple workflow connector" - sentry_dsn: "your-sentry-dsn" # Optional - cors_origins: ["*"] - routes_directory: "routes" - debug: true - host: "0.0.0.0" - port: 2005 +```bash +workflows ``` -5. Create your `main.py`: +Pick **1–8** at the prompt (enter a **path** only when the menu asks for it). -```python -from flask import Flask -from workflows_cdk import Router +--- -# Create Flask app -app = Flask("my-workflow-connector") +## Interactive main menu -# Initialize router with configuration -router = Router(app) +Run: -# Run the app -if __name__ == "__main__": - router.run_app(app) +```bash +workflows ``` -6. Create your first route in `routes/send_message/v1/route.py`: +| # | Option | What to do | +|---|--------|------------| +| **1** | Create a connector | Enter a description; same flow as `workflows create "…"` | +| **2** | Update a connector | Enter project path, then what to add → adds modules only | +| **3** | Validate a project | Enter connector root path | +| **4** | Run connector locally | Enter path → `run_dev.sh` or `python main.py` | +| **5** | Expose with ngrok | Enter path → app started if needed, then ngrok; copy the HTTPS URL | +| **6** | View documentation | Opens the custom connector guide in the browser | +| **7** | Setup AI provider | Configure Anthropic / OpenAI key (saved to `.env` in the current directory) | +| **8** | Exit | Leave the menu | -```python -from workflows_cdk import Request, Response, ManagedError -from main import router +After each action, press **Enter** when asked to return to the menu. -@router.route("/execute", methods=["POST"]) -def execute(): - """Execute the send message action.""" - request = Request(flask_request) - data = request.data - - name = data.get("name", "World") - return Response.success(data={ - "message": f"Hello, {name}!" - }) -``` +--- -## Core Components +## After `workflows create` — next steps menu -### Router +| # | Option | What to do | +|---|--------|------------| +| **1** | Run the connector | `run_dev.sh` or `python main.py` in the new project | +| **2** | Expose with ngrok | Same as main menu **5**, using the new project automatically | +| **3** | Open documentation | Opens Stacksync developer docs | +| **4** | Exit | Close this menu | -The `Router` class is the heart of the CDK, providing: +The panel shows your **Stacksync region** from the generated `.env`. -- Automatic route discovery based on file system structure -- Built-in error handling and Sentry integration -- CORS configuration -- Health check endpoints -- API documentation +--- -### Request +## Command reference -The `Request` class wraps Flask's request object, providing: +| Command | Use it to… | +|---------|------------| +| `workflows` | Open the interactive main menu | +| `workflows create ""` | Generate a new connector (or use `-o` for parent directory) | +| `workflows create --dry-run` | Preview without writing files | +| `workflows create --no-ai` | Template matching only (no LLM) | +| `workflows create --module-only` | Add modules into an existing connector directory (`-o` = that directory) | +| `workflows validate` | Validate the current directory | +| `workflows validate -p ` | Validate a specific connector root | +| `workflows setup` | Configure AI provider and API key | +| `workflows list` | List built-in app slugs in the registry | +| `workflows inspect ` | Show actions/triggers for one app | +| `workflows guide run` | Print how to run locally | +| `workflows guide ngrok` | Print how to expose with ngrok | +| `workflows guide register` | Print how to register in Developer Studio | +| `workflows guide test` | Print how to test in a workflow | -- Easy access to request data and credentials -- Automatic JSON parsing -- Type-safe access to common properties +--- -### Response +## AI configuration -The `Response` class provides standardized response formatting: +Set keys in the environment or in a `.env` file in the directory where you run the CLI: -- Success responses with optional metadata -- Error responses with appropriate status codes -- Environment-aware error details -- Sentry integration +| Variable | Purpose | +|----------|---------| +| `ANTHROPIC_API_KEY` | Claude (default when set) | +| `OPENAI_API_KEY` | OpenAI | +| `WORKFLOWS_AI_PROVIDER` | `anthropic` or `openai` if both keys are set | +| `WORKFLOWS_AI_MODEL` | Override the default model | -### ManagedError +Or run: -The `ManagedError` class provides structured error handling: +```bash +workflows setup +``` -- Type-safe error creation -- Automatic Sentry reporting -- Environment-aware error details -- Common error types (validation, not found, unauthorized, etc.) +--- -## Project Structure +## Generated project layout -Recommended project structure for a workflow connector: +Generated projects follow the [app connector template](https://github.com/stacksyncdata/workflows-app-connector-template) layout: ``` -my-workflow-connector/ -├── main.py # Application entry point -├── app_config.yaml # Application configuration -├── requirements.txt # Python dependencies -├── README.md # Project documentation -├── Dockerfile # Container configuration -├── .env # Environment variables -└── routes/ # Route modules - └── action_name/ # Group routes by action - ├── v1/ # Version 1 of the action - │ ├── route.py # Route implementation - │ └── schema.json # JSON Schema for validation - └── v2/ # Version 2 of the action +my-connector/ +├── main.py +├── app_config.yaml +├── requirements.txt +├── README.md +├── .env +├── .gitignore +├── Dockerfile +├── entrypoint.sh +├── run_dev.sh +├── run_dev.bat +├── config/ +│ ├── Dockerfile.dev +│ ├── entrypoint.sh +│ └── gunicorn_config.py +└── src/modules/ + └── / + └── v1/ ├── route.py - └── schema.json + ├── schema.json + └── module_config.yaml ``` -## Error Handling +--- -The CDK provides comprehensive error handling: +## Writing routes + +Each module’s `route.py` uses the CDK helpers. Minimal pattern: ```python -from workflows_cdk import ManagedError - -# Validation error -raise ManagedError.validation_error( - error="Invalid input", - data={"field": "email"} -) - -# Not found error -raise ManagedError.not_found( - resource="User", - identifier="123" -) - -# Authorization error -raise ManagedError.unauthorized( - message="Invalid API key" -) - -# Server error -raise ManagedError.server_error( - error="Database connection failed" -) +from workflows_cdk import Request, Response, ManagedError +from main import router + +@router.route("/execute", methods=["POST"]) +def execute(): + req = Request(flask_request) + data = req.data + credentials = req.credentials + # … call your API … + return Response(data={"result": "ok"}) ``` -## Response Formatting +For validation, not-found, and auth errors, use `ManagedError` helpers as in the [package examples](https://github.com/stacksyncdata/workflows-cdk/blob/prod/README.md#error-handling). -Standardized response formatting: +--- -```python -from workflows_cdk import Response - -# Success response -return Response.success( - data={"result": "ok"}, - message="Operation completed", - metadata={"timestamp": "2024-02-17"} -) - -# Error response -return Response.error( - error="Something went wrong", - status_code=400 -) -``` +## Documentation & resources + +- [Build a custom connector](https://docs.stacksync.com/workflow-automation/developers/build-a-custom-connector) +- [Workflows app connector](https://docs.stacksync.com/workflows/app-connector) +- [Official connector template](https://github.com/stacksyncdata/workflows-app-connector-template) +- [Stacksync docs](https://docs.stacksync.com/) + +--- ## License -This project is licensed under the Stacksync Connector License (SCL) v1.0. +This project is licensed under the **Stacksync Connector License (SCL) v1.0**. diff --git a/requirements.txt b/requirements.txt index 51a7c35..29332d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,13 @@ flask -werkzeug +werkzeug==2.2 pyopenssl==24.1.0 flask-cors>=4.0.0 python-dotenv>=1.0.0 gunicorn==22.0.0 -authlib==1.1.0 sentry-sdk[Flask] pydantic>=2.0.0 -pyyaml>=6.0.0 \ No newline at end of file +pyyaml>=6.0.0 +click>=8.0.0 +rich>=13.0.0 +openai>=1.0.0 +anthropic>=0.30.0 diff --git a/setup.py b/setup.py index 6a61628..021f7ef 100644 --- a/setup.py +++ b/setup.py @@ -7,12 +7,11 @@ setup( name="workflows_cdk", - version="0.1.0", + version="0.2.0", description="A CDK for developing Stacksync Workflows Connectors", author="Stacksync", author_email="oliviero@stacksync.com", install_requires=[ - # Core dependencies "flask", "werkzeug==2.2", "pyopenssl==24.1.0", @@ -21,10 +20,25 @@ "gunicorn==22.0.0", "sentry-sdk[Flask]", "pydantic>=2.0.0", - "pyyaml>=6.0.0" + "pyyaml>=6.0.0", + "click>=8.0.0", + "rich>=13.0.0", + "openai>=1.0.0", + "anthropic>=0.30.0", ], + entry_points={ + "console_scripts": [ + "workflows=workflows_cdk.cli.main:cli", + ], + }, python_requires=">=3.10", packages=find_packages(where="src"), package_dir={"": "src"}, include_package_data=True, + package_data={ + "workflows_cdk": [ + "registry/capabilities/*.yaml", + "templates/library/*.yaml", + ], + }, ) diff --git a/src/workflows_cdk/ai/__init__.py b/src/workflows_cdk/ai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/workflows_cdk/ai/clarifier.py b/src/workflows_cdk/ai/clarifier.py new file mode 100644 index 0000000..b3137fb --- /dev/null +++ b/src/workflows_cdk/ai/clarifier.py @@ -0,0 +1,63 @@ +""" +One-shot clarification engine. + +Takes the ``ambiguities`` list from a ConnectorSpec, renders them as +individual prompts, collects the user's answers, and returns a plain-text +string that can be fed back into the LLM refinement prompt. +""" + +from __future__ import annotations + +from rich.console import Console +from rich.prompt import Prompt + +from ..spec.connector_spec import AmbiguitySpec, ConnectorSpec + +console = Console() + + +def render_clarification(spec: ConnectorSpec) -> str: + """Display ambiguities one at a time and collect answers interactively. + + Returns a plain-text summary of the user's choices, ready to be passed + to ``Planner.refine()``. + """ + ambiguities = spec.ambiguities + if not ambiguities: + return "" + + ambiguities = ambiguities[:3] + count = len(ambiguities) + + console.print( + f"\n[bold]I need {count} detail{'s' if count != 1 else ''} " + f"before generating:[/bold]" + ) + + answers: list[str] = [] + for idx, amb in enumerate(ambiguities, 1): + console.print(f"\n[bold cyan]({idx}/{count})[/bold cyan] {amb.question}") + + options = amb.options or [] + default = amb.default or (options[0] if options else None) + + if options: + for i, opt in enumerate(options, 1): + marker = "[bold green]*[/bold green] " if opt == default else " " + console.print(f" {marker}{i}. {opt}") + console.print(f" [dim]Press Enter for default: {default}[/dim]") + + raw = console.input("[bold]> [/bold]").strip() + if not raw: + chosen = default + elif raw.isdigit() and 1 <= int(raw) <= len(options): + chosen = options[int(raw) - 1] + else: + chosen = raw + else: + raw = console.input(f"[bold]> [/bold][dim]({default})[/dim] ").strip() + chosen = raw if raw else default + + answers.append(f"{amb.question}: {chosen}") + + return "\n".join(answers) diff --git a/src/workflows_cdk/ai/intent_parser.py b/src/workflows_cdk/ai/intent_parser.py new file mode 100644 index 0000000..6871450 --- /dev/null +++ b/src/workflows_cdk/ai/intent_parser.py @@ -0,0 +1,52 @@ +""" +Lightweight pre-processing of the user prompt *before* calling the LLM. + +Extracts candidate app slugs and action verbs so the planner can narrow +which capability manifests to inject into the system prompt context window. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from ..registry.registry import CapabilityRegistry + +_TRIGGER_VERBS = frozenset({ + "when", "on", "listen", "react", "trigger", "watch", "monitor", +}) + +_SEARCH_VERBS = frozenset({ + "list", "search", "find", "get", "fetch", "query", "lookup", +}) + + +@dataclass +class ParsedIntent: + raw: str + detected_slugs: list[str] = field(default_factory=list) + has_trigger_intent: bool = False + has_search_intent: bool = False + tokens: list[str] = field(default_factory=list) + + +def parse_intent(description: str, registry: CapabilityRegistry) -> ParsedIntent: + """Extract structured signals from the raw user prompt.""" + tokens = description.lower().split() + token_set = set(tokens) + + detected_slugs: list[str] = [] + for slug in registry.slugs(): + manifest = registry.get(slug) + if manifest is None: + continue + name_lower = manifest.app.name.lower() + if slug in token_set or name_lower in description.lower(): + detected_slugs.append(slug) + + return ParsedIntent( + raw=description, + detected_slugs=detected_slugs, + has_trigger_intent=bool(token_set & _TRIGGER_VERBS), + has_search_intent=bool(token_set & _SEARCH_VERBS), + tokens=tokens, + ) diff --git a/src/workflows_cdk/ai/planner.py b/src/workflows_cdk/ai/planner.py new file mode 100644 index 0000000..f400f98 --- /dev/null +++ b/src/workflows_cdk/ai/planner.py @@ -0,0 +1,327 @@ +""" +Planner orchestrator — the single entry point for AI-powered connector creation. + +Flow: + 1. Parse intent from user description (extract slugs, verbs) + 2. Build the LLM prompt with relevant capability manifests + 3. Call the configured LLM provider with structured output + 4. Return a validated ConnectorSpec (the CLI handles clarification / compilation) + +Provider selection (Anthropic preferred): + - Set ``ANTHROPIC_API_KEY`` to use Claude (default model: claude-sonnet-4-6) + - Set ``OPENAI_API_KEY`` to use OpenAI (default model: gpt-5-nano) + - If both are set, Anthropic wins unless ``WORKFLOWS_AI_PROVIDER=openai`` + +Structured output: + - OpenAI → Responses API with ``text.format`` json_schema (strict) + - Anthropic → ``output_config.format`` json_schema (SDK auto-transforms) + Both guarantee schema-compliant JSON — no post-hoc parsing needed. +""" + +from __future__ import annotations + +import json +import logging +import os +import time +from typing import Any, Literal + +from ..registry.registry import CapabilityRegistry +from ..spec.connector_spec import ConnectorSpec +from .intent_parser import parse_intent +from .prompts import PLANNER_SYSTEM_PROMPT, REFINEMENT_PROMPT + +logger = logging.getLogger(__name__) + +Provider = Literal["openai", "anthropic"] + +DEFAULT_OPENAI_MODEL = "gpt-5-nano" +DEFAULT_ANTHROPIC_MODEL = "claude-sonnet-4-6" + + +class PlannerError(Exception): + pass + + +# --------------------------------------------------------------------------- +# Provider detection (Anthropic default) +# --------------------------------------------------------------------------- + +def _detect_provider() -> Provider: + explicit = os.environ.get("WORKFLOWS_AI_PROVIDER", "").lower() + if explicit in ("openai", "anthropic"): + return explicit # type: ignore[return-value] + + if os.environ.get("ANTHROPIC_API_KEY"): + return "anthropic" + if os.environ.get("OPENAI_API_KEY"): + return "openai" + + raise PlannerError( + "No AI provider configured. " + "Set ANTHROPIC_API_KEY or OPENAI_API_KEY, " + "or pass --no-ai to use template matching instead." + ) + + +# --------------------------------------------------------------------------- +# JSON Schema from Pydantic +# --------------------------------------------------------------------------- + +def _build_json_schema() -> dict[str, Any]: + """Generate a JSON Schema from ConnectorSpec with ``additionalProperties: + false`` on all objects and all properties promoted to ``required``. + + Both OpenAI and Anthropic require this for their structured-output modes. + """ + schema = ConnectorSpec.model_json_schema() + _prepare_strict(schema) + return schema + + +_PYDANTIC_ONLY_KEYWORDS = { + "minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", + "minLength", "maxLength", "pattern", "minItems", "maxItems", + "uniqueItems", +} + + +def _prepare_strict(node: dict[str, Any]) -> None: + """Recursively add ``additionalProperties: false``, promote all properties + to ``required``, and strip Pydantic validation keywords that neither + provider supports in structured-output schemas. The constraints are still + enforced by ``ConnectorSpec.model_validate()`` after parsing.""" + for kw in _PYDANTIC_ONLY_KEYWORDS: + node.pop(kw, None) + + if node.get("type") == "object": + node["additionalProperties"] = False + if "properties" in node: + node.setdefault("required", list(node["properties"].keys())) + + for key in ("properties", "$defs"): + container = node.get(key) + if isinstance(container, dict): + for v in container.values(): + if isinstance(v, dict): + _prepare_strict(v) + + items = node.get("items") + if isinstance(items, dict): + _prepare_strict(items) + + for combo_key in ("anyOf", "oneOf", "allOf"): + combo = node.get(combo_key) + if isinstance(combo, list): + for item in combo: + if isinstance(item, dict): + _prepare_strict(item) + + +# --------------------------------------------------------------------------- +# Planner +# --------------------------------------------------------------------------- + +class Planner: + """Stateless orchestrator: description in, ConnectorSpec out.""" + + def __init__(self, registry: CapabilityRegistry) -> None: + self.registry = registry + + def build_prompt(self, description: str) -> tuple[str, str]: + """Phase 1: parse intent and build the system prompt (instant). + + Returns (system_prompt, user_message). + """ + intent = parse_intent(description, self.registry) + + if intent.detected_slugs: + summaries = [ + self.registry.get(slug).summary_for_llm() + for slug in intent.detected_slugs + if self.registry.get(slug) is not None + ] + else: + summaries = self.registry.summaries_for_llm() + + capabilities_json = json.dumps(summaries or self.registry.summaries_for_llm(), indent=2) + system = PLANNER_SYSTEM_PROMPT.format( + capabilities_json=capabilities_json, + ) + return system, description + + def call_llm(self, system: str, user: str) -> ConnectorSpec: + """Phase 2: call the LLM and parse the spec (slow).""" + raw = _call_llm(system=system, user=user) + return _parse_spec(raw) + + def plan(self, description: str) -> ConnectorSpec: + system, user = self.build_prompt(description) + return self.call_llm(system, user) + + def refine(self, draft: ConnectorSpec, user_answers: str) -> ConnectorSpec: + system = REFINEMENT_PROMPT.format( + draft_spec_json=draft.model_dump_json(indent=2), + user_answers=user_answers, + ) + raw = _call_llm(system=system, user=user_answers) + return _parse_spec(raw) + + +# --------------------------------------------------------------------------- +# LLM dispatch +# --------------------------------------------------------------------------- + +def _call_llm(*, system: str, user: str) -> str: + provider = _detect_provider() + if provider == "anthropic": + return _call_anthropic(system=system, user=user) + return _call_openai(system=system, user=user) + + +# --------------------------------------------------------------------------- +# Anthropic / Claude — output_config.format json_schema +# --------------------------------------------------------------------------- + +def _call_anthropic(*, system: str, user: str) -> str: + try: + import anthropic + except ImportError: + raise PlannerError( + "The 'anthropic' package is required for Claude. " + "Install it with: pip install workflows-cdk" + ) + + api_key = os.environ.get("ANTHROPIC_API_KEY") + if not api_key: + raise PlannerError("ANTHROPIC_API_KEY environment variable is not set.") + + client = anthropic.Anthropic(api_key=api_key, timeout=120.0) + model = os.environ.get("WORKFLOWS_AI_MODEL", DEFAULT_ANTHROPIC_MODEL) + + last_exc: Exception | None = None + for attempt in range(2): + try: + resp = client.messages.create( + model=model, + max_tokens=8192, + system=system, + messages=[{"role": "user", "content": user}], + temperature=0.2, + ) + break + except Exception as exc: + last_exc = exc + if attempt == 0 and "connection" in str(exc).lower(): + logger.debug("Anthropic connection error, retrying in 2s…") + time.sleep(2) + continue + raise PlannerError(f"Anthropic API error: {exc}") from exc + else: + raise PlannerError(f"Anthropic API error: {last_exc}") from last_exc + + text_blocks = [b.text for b in resp.content if b.type == "text"] + if not text_blocks: + raise PlannerError("Claude returned empty response") + return text_blocks[0] + + +# --------------------------------------------------------------------------- +# OpenAI — Responses API with json_schema structured output +# --------------------------------------------------------------------------- + +def _call_openai(*, system: str, user: str) -> str: + try: + import openai + except ImportError: + raise PlannerError( + "The 'openai' package is required for AI planning. " + "Install it with: pip install workflows-cdk" + ) + + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + raise PlannerError("OPENAI_API_KEY environment variable is not set.") + + client = openai.OpenAI(api_key=api_key) + model = os.environ.get("WORKFLOWS_AI_MODEL", DEFAULT_OPENAI_MODEL) + schema = _build_json_schema() + + try: + resp = client.responses.create( + model=model, + instructions=system, + input=user, + text={ + "format": { + "type": "json_schema", + "json_schema": { + "name": "connector_spec", + "schema": schema, + "strict": True, + }, + }, + }, + temperature=0.2, + ) + return resp.output_text + except Exception as exc: + logger.debug("Responses API unavailable (%s), falling back to chat completions", exc) + + try: + return _chat_completions_fallback(client, model, system, user, schema) + except Exception as exc: + raise PlannerError(f"OpenAI API error: {exc}") from exc + + +def _chat_completions_fallback( + client: "openai.OpenAI", + model: str, + system: str, + user: str, + schema: dict[str, Any], +) -> str: + resp = client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "connector_spec", + "schema": schema, + "strict": True, + }, + }, + temperature=0.2, + ) + content = resp.choices[0].message.content + if content is None: + raise PlannerError("LLM returned empty response") + return content + + +# --------------------------------------------------------------------------- +# Spec parsing +# --------------------------------------------------------------------------- + +def _parse_spec(raw: str) -> ConnectorSpec: + """Parse LLM output into a ConnectorSpec. + + With structured output the JSON is already valid, but we keep the + fence-stripping as a safety net for fallback paths. + """ + text = raw.strip() + if text.startswith("```"): + lines = text.split("\n") + lines = [l for l in lines if not l.strip().startswith("```")] + text = "\n".join(lines) + + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + raise PlannerError(f"LLM returned invalid JSON: {exc}\n\nRaw output:\n{raw}") + + return ConnectorSpec.model_validate(data) diff --git a/src/workflows_cdk/ai/prompts.py b/src/workflows_cdk/ai/prompts.py new file mode 100644 index 0000000..eedc614 --- /dev/null +++ b/src/workflows_cdk/ai/prompts.py @@ -0,0 +1,209 @@ +""" +LLM system prompts for the connector planner. + +Design follows the GPT-5 prompting guide: + - Structured XML sections for instruction adherence + - Explicit output contract with JSON schema + - No contradictory instructions + - Single-turn persistence directive + - Clarification rules capped at 1 round / 3 sub-questions +""" + +PLANNER_SYSTEM_PROMPT = """\ + +You are the Stacksync connector planner. You receive a short natural-language +description (under 30 words) of a desired connector and produce a structured +ConnectorSpec JSON that will be compiled into a runnable project. + +You MUST generate working Python implementation code for each action and trigger. +The implementation uses the ``requests`` library to make real API calls. + + + +{capabilities_json} + + + +Return ONLY valid JSON (no markdown fences, no commentary) matching this schema: + +{{ + "app_type": "string — slug, e.g. slack", + "app_name": "string — human-readable, e.g. Slack Connector", + "version": "string — default v1", + "actions": [ + {{ + "name": "string — snake_case action id", + "category": "action | search | transform", + "description": "string — one-sentence description", + "required_fields": [{{ "name": "string", "type": "string|number|boolean|object|array", "description": "string", "widget": "string (optional)", "choices": [{{}}] (optional), "depends_on": "string (optional)", "dynamic_content": false }}], + "optional_fields": [{{ "name": "string", "type": "string|number|boolean|object|array", "description": "string", "widget": "string (optional)", "choices": [{{}}] (optional), "depends_on": "string (optional)", "dynamic_content": false }}], + "implementation": "string — Python code (see )" + }} + ], + "triggers": [ + {{ + "name": "string — snake_case trigger id", + "event": "string — dot-notation event name", + "description": "string", + "payload_fields": [{{ "name": "string", "type": "string|number|boolean|object|array", "description": "string" }}], + "implementation": "string — Python code (see )" + }} + ], + "auth": {{ + "type": "oauth2 | api_key | basic | none", + "scopes": ["string"], + "fields": [] + }}, + "confidence": "float 0.0–1.0", + "ambiguities": [ + {{ + "question": "string", + "options": ["string"], + "default": "string | null" + }} + ] +}} + + + +Fields (required_fields, optional_fields, payload_fields) define the UI form that +the user fills in the Stacksync workflow builder. They are CONFIGURATION inputs, +not the data that comes out of an API. + +For actions: fields = what the user must provide to run the action. + Example "send_slack_message" action: + required_fields: channel (which Slack channel), text (message text) + optional_fields: username (display name override) + +For triggers: payload_fields = what the user configures for the trigger. + Example "new_hubspot_contact" trigger: + payload_fields: slack_channel (where to send), message_format (what to include) + NOT: firstname, lastname, email (those come from the API at runtime) + +CRITICAL ALIGNMENT RULE: Every field name used via data.get("field_name") in +the implementation code MUST exist in required_fields, optional_fields, or +payload_fields. If the implementation reads a field, add it to the field list. +Conversely, every field in the list should be used in the implementation. + +EXTENDED FIELD ATTRIBUTES (optional, use when appropriate): + - "widget": Override the default UI widget. Common values: "input", "textarea", + "checkbox", "SelectWidget", "password". Default is inferred from type. + - "choices": Array of {{id, label}} objects for static dropdown options. + Use when the field is type "object" and options are known at generation time. + Example: [{{"id": "twitter", "label": "Twitter"}}, {{"id": "linkedin", "label": "LinkedIn"}}] + - "dynamic_content": Set to true when the field's dropdown choices must be + loaded at runtime via the /content endpoint (e.g. fetched from an external API). + - "depends_on": The id of another field that this field's value depends on. + When set, the Stacksync UI will reload either /content or /schema when the + depended-on field changes. Only set this when there is a real dependency. + +ENDPOINT DECISION RULES: + - /execute is ALWAYS generated. + - /content is generated ONLY if at least one field has dynamic_content=true. + - /schema is generated ONLY if at least one field has depends_on set AND + dynamic_content is NOT true for that field (schema reload vs content reload). + - Do NOT set dynamic_content or depends_on unless the description clearly implies + runtime-dependent choices or form structure changes. + + + +The "implementation" field is a string of Python code that forms the BODY of the +/execute endpoint function. The following variables are already available: + + - ``data`` — dict of user-submitted form fields (from the schema above) + - ``credentials`` — dict with auth tokens (e.g. credentials.get("access_token")) + - ``requests`` — the requests library (already imported at file top) + - ``Response`` — the CDK Response class (use Response(data=...) or Response.error) + - ``ManagedError`` — the CDK error class + +Your implementation MUST: + 1. Extract fields from ``data`` using data.get("field_name") + 2. Build headers using credentials (e.g. Bearer token or API key) + 3. Make the real HTTP call using requests.get / requests.post / etc. + 4. Return Response(data=result) on success or raise ManagedError on failure + 5. Use proper indentation (4 spaces per level, starting at zero indent) + 6. Handle errors with try/except and raise ManagedError(message) + 7. NEVER include ``import`` statements — requests, json, time are already + available. NEVER include function def or decorators. + +Example — a Slack send_message action implementation: + + channel = data.get("channel") + text = data.get("text") + + headers = {{"Authorization": f"Bearer {{credentials.get('access_token')}}", "Content-Type": "application/json"}} + payload = {{"channel": channel, "text": text}} + + try: + resp = requests.post("https://slack.com/api/chat.postMessage", headers=headers, json=payload) + result = resp.json() + if not result.get("ok"): + raise ManagedError(result.get("error", "Unknown Slack error")) + except ManagedError: + raise + except Exception as e: + raise ManagedError(f"Slack API error: {{str(e)}}") + + return Response(data=result) + +The above would be stored as a single string with \\n for newlines. + + + +- If confidence >= 0.85: return the spec with ambiguities=[]. Done. +- If confidence < 0.85: return the spec AND populate ambiguities with at most + 3 items. Each item has: question, options, default. +- NEVER produce more than 1 clarification round. +- NEVER ask about things you can infer from context or from the capability + manifest defaults. +- If the user mentions an app that is NOT in , set + confidence to 0.5 and include all actions you can reasonably infer from + the app's public API. Do NOT mention the registry in clarification questions. + + + +- Generate at most 3 actions + 1 trigger unless the user explicitly asks for + more. Pick the most useful operations for the requested app. +- Map actions ONLY to capabilities listed in when the + app is known. +- Default auth type to what the capability manifest specifies. +- Default action category to "action" unless the description contains "list", + "search", "find" (use "search") or "when", "on", "trigger", "listen", + "react" (use trigger). +- Triggers go in the "triggers" array, NOT in "actions". +- Every required_field must have name, type, and description. +- Do NOT invent scopes or fields that are not in the manifest. +- Every action and trigger MUST have a non-empty "implementation" string. + + + +Complete the full spec in a single JSON response. Do not produce partial +results. Do not add text outside the JSON object. + +""" + +REFINEMENT_PROMPT = """\ + +You are the Stacksync connector planner. The user has answered your +clarification questions. Merge their answers into the draft spec and return +the final ConnectorSpec JSON. + + + +{draft_spec_json} + + + +{user_answers} + + + +Return ONLY the final valid JSON ConnectorSpec. Set confidence to 1.0 and +ambiguities to []. Follow the same schema as before. Every action and trigger +MUST have a non-empty "implementation" string with real Python code. + + + +Produce the complete spec in a single response. No commentary, no markdown. + +""" diff --git a/src/workflows_cdk/ai/validator.py b/src/workflows_cdk/ai/validator.py new file mode 100644 index 0000000..504dcef --- /dev/null +++ b/src/workflows_cdk/ai/validator.py @@ -0,0 +1,156 @@ +""" +Deterministic post-LLM validator. + +Runs *after* the planner produces a ConnectorSpec and checks it against the +capability registry. Returns blocking errors and non-blocking warnings so +the CLI can decide whether to proceed. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from ..registry.registry import CapabilityRegistry +from ..spec.connector_spec import ConnectorSpec + +_VALID_FIELD_TYPES = frozenset({"string", "number", "boolean", "object", "array"}) +_VALID_AUTH_TYPES = frozenset({"oauth2", "api_key", "basic", "none"}) +_VALID_CATEGORIES = frozenset({"action", "trigger", "search", "transform"}) + + +@dataclass +class ValidationResult: + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + return len(self.errors) == 0 + + +def validate_spec(spec: ConnectorSpec, registry: CapabilityRegistry) -> ValidationResult: + """Validate a ConnectorSpec against the registry. Pure, no side-effects.""" + result = ValidationResult() + + if not spec.app_type: + result.errors.append("app_type is empty") + if not spec.app_name: + result.errors.append("app_name is empty") + + _validate_auth(spec, result) + _validate_actions(spec, registry, result) + _validate_triggers(spec, registry, result) + _validate_no_route_collisions(spec, result) + _validate_stacksync_contracts(spec, result) + + return result + + +def _validate_auth(spec: ConnectorSpec, result: ValidationResult) -> None: + if spec.auth.type not in _VALID_AUTH_TYPES: + result.errors.append( + f"Unknown auth type '{spec.auth.type}'. " + f"Valid: {', '.join(sorted(_VALID_AUTH_TYPES))}" + ) + + +def _validate_actions( + spec: ConnectorSpec, + registry: CapabilityRegistry, + result: ValidationResult, +) -> None: + manifest = registry.get(spec.app_type) + seen_names: set[str] = set() + + for action in spec.actions: + if action.name in seen_names: + result.errors.append(f"Duplicate action name: '{action.name}'") + seen_names.add(action.name) + + if action.category not in _VALID_CATEGORIES: + result.errors.append( + f"Action '{action.name}' has invalid category '{action.category}'" + ) + + for f in action.required_fields + action.optional_fields: + if f.type not in _VALID_FIELD_TYPES: + result.warnings.append( + f"Action '{action.name}', field '{f.name}': " + f"unknown type '{f.type}', defaulting to 'string'" + ) + + # Registry hints help the LLM but are not a gate for validation. + + +def _validate_triggers( + spec: ConnectorSpec, + registry: CapabilityRegistry, + result: ValidationResult, +) -> None: + manifest = registry.get(spec.app_type) + seen_names: set[str] = set() + + for trigger in spec.triggers: + if trigger.name in seen_names: + result.errors.append(f"Duplicate trigger name: '{trigger.name}'") + seen_names.add(trigger.name) + + # Registry hints help the LLM but are not a gate for validation. + + +def _validate_no_route_collisions( + spec: ConnectorSpec, + result: ValidationResult, +) -> None: + paths: set[str] = set() + for action in spec.actions: + path = f"/{action.name}/{spec.version}/execute" + if path in paths: + result.errors.append(f"Route collision: {path}") + paths.add(path) + + for trigger in spec.triggers: + path = f"/{trigger.name}/{spec.version}/execute" + if path in paths: + result.errors.append(f"Route collision: {path}") + paths.add(path) + + +def _validate_stacksync_contracts( + spec: ConnectorSpec, + result: ValidationResult, +) -> None: + """Stacksync-specific checks that go beyond generic type/name validation.""" + if spec.auth.type in ("oauth2", "api_key") and not spec.app_type: + result.errors.append( + "Auth requires a connection but app_type is empty " + "(needed for allowed_app_types in schema)" + ) + + for action in spec.actions: + all_fields = list(action.required_fields) + list(action.optional_fields) + _check_field_contracts(action.name, all_fields, result) + + for trigger in spec.triggers: + _check_field_contracts(trigger.name, list(trigger.payload_fields), result) + + +def _check_field_contracts( + module_name: str, + fields: list, + result: ValidationResult, +) -> None: + all_ids = {f.name for f in fields} + + for f in fields: + if f.depends_on and f.depends_on not in all_ids: + result.errors.append( + f"Module '{module_name}', field '{f.name}': " + f"depends_on='{f.depends_on}' references non-existent field" + ) + + if f.type == "object" and not f.choices and not f.dynamic_content: + result.warnings.append( + f"Module '{module_name}', field '{f.name}': " + f"type 'object' typically needs choices or dynamic_content" + ) diff --git a/src/workflows_cdk/cli/__init__.py b/src/workflows_cdk/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/workflows_cdk/cli/main.py b/src/workflows_cdk/cli/main.py new file mode 100644 index 0000000..927a989 --- /dev/null +++ b/src/workflows_cdk/cli/main.py @@ -0,0 +1,1494 @@ +""" +CLI entry point for the Stacksync Workflows CDK. + +Registered as a console_script in setup.py so ``pip install workflows-cdk`` +makes the ``workflows`` command available globally. +""" + +from __future__ import annotations + +import json +import os +import re +import shlex +import shutil +import signal +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +import webbrowser +from pathlib import Path +from typing import Any, Optional + +import click +from dotenv import load_dotenv +from rich.console import Console +from rich.panel import Panel +from rich.prompt import Confirm, Prompt +from rich.tree import Tree + +from ..ai.validator import validate_spec +from ..registry.registry import CapabilityRegistry +from ..spec.compiler import ( + MODULES_DIR, + compile_connector, + detect_port, + needs_content_endpoint, + needs_schema_endpoint, +) +from ..spec.connector_spec import ConnectorSpec +from ..templates.matcher import match_template + +console = Console() + +_AUTH_DISPLAY = { + "oauth2": "OAuth2 via managed connection", + "api_key": "API key via connection field", + "basic": "Basic auth", + "none": "No auth", +} + +_BANNER = ( + "[bold cyan]" + " ____ _ _\n" + "/ ___|| |_ __ _ ___| | _____ _ _ _ __ ___\n" + "\\___ \\| __/ _` |/ __| |/ / __| | | | '_ \\ / __|\n" + " ___) | || (_| | (__| <\\__ \\ |_| | | | | (__\n" + "|____/ \\__\\__,_|\\___|_|\\_\\___/\\__, |_| |_|\\___|\n" + " |___/" + "[/bold cyan]" +) + + +def _print_banner() -> None: + console.print(_BANNER) + console.print("[bold green]Workflows CDK[/bold green]") + console.print("[dim]https://docs.stacksync.com/workflows/app-connector[/dim]") + console.print() + + +def _env_file() -> Path: + return Path.cwd() / ".env" + + +def _ensure_dotenv() -> None: + load_dotenv(_env_file()) + + +def _has_api_key() -> bool: + return bool( + os.environ.get("ANTHROPIC_API_KEY") or os.environ.get("OPENAI_API_KEY") + ) + + +def _run_setup() -> bool: + """Interactive first-time setup. Returns True if a key was configured.""" + console.print( + Panel( + "[bold]Welcome to Workflows CDK[/bold]\n\n" + "No API key found. Let's set one up.\n" + "You can reconfigure anytime with [cyan]workflows setup[/cyan].", + border_style="blue", + ) + ) + + provider = Prompt.ask( + "\n[bold]Which AI provider?[/bold]", + choices=["anthropic", "openai"], + default="anthropic", + ) + + if provider == "anthropic": + key_name = "ANTHROPIC_API_KEY" + hint = "sk-ant-..." + else: + key_name = "OPENAI_API_KEY" + hint = "sk-..." + + api_key = Prompt.ask(f"\n[bold]Paste your {key_name}[/bold] ({hint})") + api_key = api_key.strip() + + if not api_key: + console.print("[red]No key provided. Aborting setup.[/red]") + return False + + os.environ[key_name] = api_key + _save_to_env(key_name, api_key) + + console.print(f"\n[green bold]Done![/green bold] {key_name} saved to [cyan]{_env_file()}[/cyan]") + console.print("[dim]You can also export it in your shell or edit .env directly.[/dim]\n") + return True + + +def _save_to_env(key_name: str, value: str) -> None: + """Append or update a key in the .env file.""" + env = _env_file() + lines: list[str] = [] + found = False + + if env.exists(): + for line in env.read_text().splitlines(): + stripped = line.lstrip("# ").split("=", 1)[0].strip() + if stripped == key_name: + lines.append(f"{key_name}={value}") + found = True + else: + lines.append(line) + + if not found: + lines.append(f"{key_name}={value}") + + env.write_text("\n".join(lines) + "\n") + + +# --------------------------------------------------------------------------- +# CLI group +# --------------------------------------------------------------------------- + +_DOCS_URL = "https://docs.stacksync.com/workflow-automation/developers/build-a-custom-connector" +_NGROK_DOWNLOAD_URL = "https://ngrok.com/download" + +# Processes started by this CLI session (Expose with ngrok / background connector). +_managed_ngrok_proc: subprocess.Popen | None = None +_managed_connector_launcher: subprocess.Popen | None = None +_managed_connector_docker_script: bool = False +_managed_connector_project_dir: Path | None = None + + +def _docker_app_name(project_dir: Path) -> str: + """Container name used by generated run_dev.sh (workflows-app-).""" + return f"workflows-app-{project_dir.resolve().name}" + + +def _terminate_process_group(proc: subprocess.Popen, *, timeout: float = 8.0) -> None: + """Stop a process started with start_new_session=True (best-effort).""" + if proc.poll() is not None: + return + if sys.platform == "win32": + proc.terminate() + else: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + except (ProcessLookupError, OSError): + proc.terminate() + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + + +def _dispose_managed_ngrok_silent() -> None: + """Stop ngrok started by this CLI, if still running.""" + global _managed_ngrok_proc + if _managed_ngrok_proc is not None: + _terminate_process_group(_managed_ngrok_proc) + _managed_ngrok_proc = None + + +def _dispose_managed_connector_silent() -> None: + """Stop background connector launcher and Docker container if we started them.""" + global _managed_connector_launcher, _managed_connector_docker_script + global _managed_connector_project_dir + + if _managed_connector_launcher is not None: + _terminate_process_group(_managed_connector_launcher) + _managed_connector_launcher = None + + if _managed_connector_docker_script and _managed_connector_project_dir is not None: + name = _docker_app_name(_managed_connector_project_dir) + try: + subprocess.run( + ["docker", "rm", "-f", name], + capture_output=True, + timeout=30, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + pass + + _managed_connector_docker_script = False + _managed_connector_project_dir = None + + +def _managed_ngrok_still_running() -> bool: + global _managed_ngrok_proc + if _managed_ngrok_proc is None: + return False + if _managed_ngrok_proc.poll() is not None: + _managed_ngrok_proc = None + return False + return True + + +def _managed_connector_still_running() -> bool: + global _managed_connector_launcher + if _managed_connector_launcher is None: + return False + if _managed_connector_launcher.poll() is not None: + _managed_connector_launcher = None + return False + return True + + +def _prompt_stop_managed_services_if_any() -> None: + """If this CLI started ngrok or a background connector, offer to stop them.""" + ngrok_live = _managed_ngrok_still_running() + conn_live = _managed_connector_still_running() + + if not ngrok_live and not conn_live: + return + + console.print() + console.rule("[bold cyan]Background services[/bold cyan]", style="cyan") + if ngrok_live: + if Confirm.ask( + "Stop the [bold]ngrok tunnel[/bold] that this CLI started?", + default=True, + ): + _dispose_managed_ngrok_silent() + console.print("[dim]ngrok stopped.[/dim]") + else: + console.print("[dim]Left ngrok running.[/dim]") + if conn_live: + if Confirm.ask( + "Stop the [bold]connector[/bold] this CLI started in the background " + "(Docker or python)?", + default=True, + ): + _dispose_managed_connector_silent() + console.print("[dim]Connector stopped.[/dim]") + else: + console.print("[dim]Left the connector running.[/dim]") + console.print() + + +def _detect_region(project_dir: Path) -> str: + """Read REGION from the connector's .env (matches run_dev.sh / template).""" + env_path = project_dir / ".env" + if env_path.exists(): + for raw in env_path.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith("REGION="): + val = line.split("=", 1)[1].strip().strip('"').strip("'") + return val if val else "besg" + return "besg" + + +def _ngrok_local_api_tunnels() -> dict[str, Any] | None: + """Return ngrok agent local API JSON, or None if unreachable.""" + try: + req = urllib.request.Request( + "http://127.0.0.1:4040/api/tunnels", + headers={"Accept": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=1.5) as resp: + return json.loads(resp.read().decode()) + except (urllib.error.URLError, OSError, json.JSONDecodeError, ValueError): + return None + + +def _port_from_upstream_addr(addr: str) -> int | None: + """Parse local port from ngrok config.addr (e.g. http://127.0.0.1:2003).""" + s = addr.strip() + if not s or s.lower() in ("undefined", "none", "null"): + return None + m = re.search(r":(\d{1,5})\s*$", s) + if m: + p = int(m.group(1)) + return p if 1 <= p <= 65535 else None + if s.isdigit(): + p = int(s) + return p if 1 <= p <= 65535 else None + return None + + +def _tunnel_local_port(tunnel: dict[str, Any]) -> int | None: + """Return the upstream port this tunnel forwards to, or None if unknown / invalid.""" + conf = tunnel.get("config") or {} + addr = conf.get("addr") + if isinstance(addr, dict): + p = addr.get("port") + if p is not None: + try: + pi = int(p) + return pi if 1 <= pi <= 65535 else None + except (TypeError, ValueError): + pass + url = addr.get("URL") or addr.get("url") + if isinstance(url, str): + return _port_from_upstream_addr(url) + return None + if isinstance(addr, str): + return _port_from_upstream_addr(addr) + return None + + +def _tunnel_targets_port(tunnel: dict[str, Any], port: int) -> bool: + return _tunnel_local_port(tunnel) == port + + +def _pick_https_public_url_for_port(tunnels_data: dict[str, Any], port: int) -> str | None: + """Return the public HTTPS URL only for a tunnel whose upstream port matches *port*. + + Does not fall back to unrelated tunnels (avoids false positives when another + broken tunnel exists with undefined upstream). + """ + tunnels = tunnels_data.get("tunnels") or [] + for t in tunnels: + if t.get("proto") == "https" and _tunnel_targets_port(t, port): + url = t.get("public_url") + if url: + return str(url) + for t in tunnels: + if _tunnel_targets_port(t, port): + url = t.get("public_url") + if url: + return str(url) + return None + + +def _port_is_listening(port: int, host: str = "127.0.0.1") -> bool: + """True if something accepts TCP connections on host:port.""" + try: + with socket.create_connection((host, port), timeout=0.75): + return True + except OSError: + return False + + +def _wait_for_port(port: int, *, timeout: float, host: str = "127.0.0.1") -> bool: + """Poll until the port accepts connections or *timeout* seconds elapse.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if _port_is_listening(port, host=host): + return True + time.sleep(0.4) + return False + + +def _start_main_py_background(project_dir: Path, main_py: Path) -> subprocess.Popen | None: + try: + return subprocess.Popen( + [sys.executable, str(main_py)], + cwd=str(project_dir), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + except OSError as exc: + console.print(f"[red]Could not start python main.py: {exc}[/red]") + return None + + +def _start_connector_background( + project_dir: Path, port: int +) -> tuple[bool, subprocess.Popen | None, bool]: + """Start ./run_dev.sh (with a pseudo-TTY when needed) or python main.py in the background. + + Returns: + (success, launcher_popen, docker_via_script) — *docker_via_script* is True when + the launcher is ``script`` wrapping ``run_dev.sh`` (Docker), for cleanup via ``docker rm``. + """ + run_dev = project_dir / "run_dev.sh" + main_py = project_dir / "main.py" + + if run_dev.exists(): + script_bin = shutil.which("script") + if script_bin and sys.platform != "win32": + try: + console.print( + "[dim]Launching [cyan]./run_dev.sh[/cyan] in the background " + "(Docker; first run may build for several minutes)…[/dim]" + ) + if sys.platform == "darwin": + script_argv = [ + script_bin, "-q", "/dev/null", "bash", str(run_dev), + ] + else: + # util-linux `script`: -c runs a command with a pty + inner = f"bash {shlex.quote(str(run_dev.name))}" + script_argv = [ + script_bin, "-q", "-c", inner, "/dev/null", + ] + proc = subprocess.Popen( + script_argv, + cwd=str(project_dir), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + return True, proc, True + except OSError as exc: + console.print(f"[yellow]Could not start run_dev.sh via script: {exc}[/yellow]") + + if main_py.exists(): + console.print( + "[yellow]Docker [bold]run_dev.sh[/bold] needs a pseudo-TTY for [bold]-it[/bold]. " + "Starting [cyan]python main.py[/cyan] in the background instead.[/yellow]\n" + "[dim]For full Docker dev, run [cyan]./run_dev.sh[/cyan] in a separate terminal.[/dim]" + ) + py = _start_main_py_background(project_dir, main_py) + return py is not None, py, False + + console.print( + "[red]Cannot start the connector: [cyan]script[/cyan] is not available for headless " + "[cyan]./run_dev.sh[/cyan], and [cyan]main.py[/cyan] is missing.[/red]" + ) + return False, None, False + + if main_py.exists(): + console.print("[dim]Starting [cyan]python main.py[/cyan] in the background…[/dim]") + py = _start_main_py_background(project_dir, main_py) + return py is not None, py, False + + console.print( + f"[red]No [cyan]run_dev.sh[/cyan] or [cyan]main.py[/cyan] in {project_dir}[/red]" + ) + return False, None, False + + +def _ensure_connector_running_for_ngrok(project_dir: Path, port: int) -> bool: + """Ensure localhost:*port* is serving the connector before exposing with ngrok.""" + global _managed_connector_launcher, _managed_connector_docker_script + global _managed_connector_project_dir + + if _port_is_listening(port): + console.print( + f"[dim]Connector already listening on [cyan]localhost:{port}[/cyan].[/dim]" + ) + return True + + console.print( + "\n[bold]Starting the connector locally first[/bold] " + "[dim](ngrok needs your app on this port).[/dim]" + ) + ok, launcher, docker_script = _start_connector_background(project_dir, port) + if not ok or launcher is None: + return False + + _managed_connector_launcher = launcher + _managed_connector_docker_script = docker_script + _managed_connector_project_dir = project_dir + + console.print( + f"[dim]Waiting for [cyan]localhost:{port}[/cyan] " + f"(up to ~7 min on first Docker build)…[/dim]" + ) + if not _wait_for_port(port, timeout=420.0): + _dispose_managed_connector_silent() + console.print( + Panel( + f"[red]Nothing accepted connections on port {port} in time.[/red]\n\n" + "Start the connector manually:\n" + f" [cyan]cd {project_dir}[/cyan]\n" + " [cyan]./run_dev.sh[/cyan] [dim]or[/dim] [cyan]python main.py[/cyan]\n\n" + "Then run **Expose with ngrok** again.", + title="[red]Connector did not become ready[/red]", + border_style="red", + ) + ) + return False + + console.print(f"[green]Connector is reachable on port {port}.[/green]") + return True + + +def _expose_with_ngrok(project_dir: Path, port: int) -> None: + """Ensure ngrok is available; start a tunnel or reuse an existing one.""" + ngrok_bin = shutil.which("ngrok") + if not ngrok_bin: + console.print(Panel( + "[bold]ngrok is not installed or not on your PATH.[/bold]\n\n" + " [bold]macOS (Homebrew):[/bold]\n" + " brew install ngrok/ngrok/ngrok\n\n" + " Or download the agent from the official site.\n" + " After installing, run [cyan]ngrok config add-authtoken [/cyan] once.", + title="[yellow]ngrok not found[/yellow]", + border_style="yellow", + )) + if Confirm.ask("Open the ngrok download page?", default=True): + webbrowser.open(_NGROK_DOWNLOAD_URL) + return + + if not _ensure_connector_running_for_ngrok(project_dir, port): + return + + region = _detect_region(project_dir) + region_hint = ( + f"Use Stacksync region [bold]{region}[/bold] in Developer Studio and workflows." + ) + + data = _ngrok_local_api_tunnels() + if data is not None: + existing = _pick_https_public_url_for_port(data, port) + if existing: + console.print( + f"\n[green]A tunnel to localhost:{port} is already running.[/green]\n" + f" Public URL: [bold cyan]{existing}[/bold cyan]\n" + ) + console.print(f"[dim]{region_hint}[/dim]") + return + + global _managed_ngrok_proc + _dispose_managed_ngrok_silent() + + console.print(f"\n[bold]Starting ngrok[/bold] [dim](forwarding to localhost:{port})…[/dim]") + try: + proc = subprocess.Popen( + [ngrok_bin, "http", str(port)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + except OSError as exc: + console.print(f"[red]Could not start ngrok: {exc}[/red]") + return + + deadline = time.monotonic() + 12.0 + public_url: str | None = None + while time.monotonic() < deadline: + if proc.poll() is not None: + console.print( + Panel( + "[red]ngrok exited immediately.[/red]\n\n" + "Common causes: missing authtoken or port already in use.\n" + " [cyan]ngrok config add-authtoken YOUR_TOKEN[/cyan]\n" + f"Or run [cyan]ngrok http {port}[/cyan] in a terminal to see the full error.", + title="[red]ngrok failed[/red]", + border_style="red", + ) + ) + return + tunnels = _ngrok_local_api_tunnels() + if tunnels is not None: + public_url = _pick_https_public_url_for_port(tunnels, port) + if public_url: + break + time.sleep(0.35) + + if public_url: + _managed_ngrok_proc = proc + console.print( + f"\n[green bold]Tunnel is up.[/green bold]\n" + f" Public URL: [bold cyan]{public_url}[/bold cyan]\n" + ) + console.print( + f"[dim]{region_hint} " + f"Paste the URL into Developer Studio. " + f"Inspect tunnels at http://127.0.0.1:4040[/dim]" + ) + else: + _managed_ngrok_proc = proc + console.print( + f"\n[yellow]ngrok was started in the background but the public URL could not " + f"be read automatically.[/yellow]\n" + f" Open [cyan]http://127.0.0.1:4040[/cyan] in your browser to copy the HTTPS URL.\n" + ) + + +@click.group(invoke_without_command=True) +@click.version_option(package_name="workflows_cdk") +@click.pass_context +def cli(ctx: click.Context) -> None: + """Stacksync Workflows CLI -- AI-assisted connector and module generation.""" + ctx.ensure_object(dict) + if ctx.invoked_subcommand is None: + _start_menu(ctx) + + +def _banner_once(ctx: click.Context) -> None: + """Print the banner only once per CLI invocation.""" + state = ctx.ensure_object(dict) + if not state.get("banner_shown"): + _print_banner() + state["banner_shown"] = True + + +def _pause_return_to_menu(*, label: str) -> None: + """Wait for Enter so the user can read output before the menu redraws.""" + console.print() + console.rule(f"[dim]Press Enter to return to the {label}[/dim]", style="dim") + try: + console.input() + except (KeyboardInterrupt, EOFError): + _prompt_stop_managed_services_if_any() + console.print() + + +def _hint_path_instead_of_menu_number(choice: str, *, menu_range: str = "1–8") -> None: + """If the user typed a path at the menu prompt, explain the flow.""" + s = choice.strip() + if len(s) < 2: + return + if s.isdigit() and len(s) == 1: + return + pathish = ( + s.startswith((".", "/", "~")) + or "/" in s + or "\\" in s + or s.endswith(("-connector", "_connector")) + ) + if pathish: + console.print( + "[yellow]That looks like a path, not a menu number.[/yellow]\n" + f"[dim]Pick an option from the menu ([cyan]{menu_range}[/cyan]) first.[/dim]" + ) + + +def _is_connector_root(path: Path) -> bool: + """True if *path* looks like a connector root (has app_config.yaml).""" + return (path / "app_config.yaml").is_file() + + +def _discover_connectors_near_cwd(*, max_results: int = 30) -> list[Path]: + """List connector roots: current directory, then each immediate subfolder with app_config.yaml.""" + cwd = Path.cwd().resolve() + ordered: list[Path] = [] + seen: set[Path] = set() + + def add(candidate: Path) -> None: + rp = candidate.resolve() + if rp in seen or not _is_connector_root(rp): + return + seen.add(rp) + if len(ordered) < max_results: + ordered.append(rp) + + add(cwd) + try: + for child in sorted(cwd.iterdir(), key=lambda p: p.name.lower()): + if child.is_dir() and not child.name.startswith("."): + add(child) + except OSError: + pass + return ordered + + +def _connector_choice_label(path: Path) -> str: + try: + rel = path.resolve().relative_to(Path.cwd().resolve()) + s = str(rel) + return "." if s in (".", "") else s + except ValueError: + return str(path) + + +def _read_custom_connector_path() -> Path | None: + raw = console.input( + "[bold]Path to connector project[/bold] [dim](default: .)[/dim]\n> " + ).strip() or "." + project = Path(raw).expanduser().resolve() + if not project.is_dir(): + console.print(f"[red]Not a directory:[/red] {project}") + return None + return project + + +def _prompt_connector_project_dir() -> Path | None: + """Pick a connector from discovered folders, paste a path, or type a custom path.""" + discovered = _discover_connectors_near_cwd() + + if not discovered: + console.print( + "[dim]No app_config.yaml in . or immediate subfolders — enter path manually.[/dim]" + ) + return _read_custom_connector_path() + + console.print() + console.print( + "[bold]Connector projects[/bold] " + "[dim](current dir or subfolders with app_config.yaml)[/dim]" + ) + for i, p in enumerate(discovered, start=1): + console.print(f" [cyan][{i}][/cyan] {_connector_choice_label(p)}") + console.print(f" [cyan][c][/cyan] Enter a custom path") + + if len(discovered) == 1: + hint = "[bold]Select [cyan]1[/cyan] or [cyan]c[/cyan] [dim](Enter = 1)[/dim]:[/bold] " + else: + hint = ( + f"[bold]Select a number [cyan](1–{len(discovered)})[/cyan] " + f"or [cyan]c[/cyan] [dim](Enter = 1)[/dim]:[/bold] " + ) + sel = console.input(f"\n{hint}").strip() + + if not sel: + return discovered[0] + low = sel.lower() + if low in ("c", "custom"): + return _read_custom_connector_path() + if sel.isdigit(): + n = int(sel) + if 1 <= n <= len(discovered): + return discovered[n - 1] + console.print("[yellow]Invalid number.[/yellow]") + return None + + project = Path(sel).expanduser().resolve() + if project.is_dir(): + return project + console.print(f"[red]Not a directory:[/red] {project}") + return None + + +def _start_menu(ctx: click.Context) -> None: + """Branded interactive menu when no subcommand is given.""" + _banner_once(ctx) + while True: + console.print(Panel( + " [bold cyan][1][/bold cyan] Create a connector " + "[dim]Generate modules from a description[/dim]\n" + " [bold cyan][2][/bold cyan] Update a connector " + "[dim]Add modules to an existing project[/dim]\n" + " [bold cyan][3][/bold cyan] Validate a project " + "[dim]Pick connector or custom path[/dim]\n" + " [bold cyan][4][/bold cyan] Run connector locally " + "[dim]Pick project, then run_dev.sh[/dim]\n" + " [bold cyan][5][/bold cyan] Expose with ngrok " + "[dim]Pick project, then tunnel[/dim]\n" + " [bold cyan][6][/bold cyan] View documentation " + "[dim]Open the Stacksync developer docs[/dim]\n" + " [bold cyan][7][/bold cyan] Setup AI provider " + "[dim]Configure your Anthropic / OpenAI key[/dim]\n" + " [bold cyan][8][/bold cyan] Exit", + title="[bold]What would you like to do?[/bold]", + border_style="cyan", + padding=(1, 2), + )) + try: + choice = console.input("[bold]Select [cyan][1-8][/cyan]:[/bold] ").strip() + except (KeyboardInterrupt, EOFError): + _prompt_stop_managed_services_if_any() + return + + if choice == "8" or choice == "": + _prompt_stop_managed_services_if_any() + return + + try: + if choice == "1": + console.rule("[bold cyan]Create a connector[/bold cyan]", style="cyan") + console.print() + desc = console.input( + "[bold]Describe your connector:[/bold] " + "[dim](e.g. \"Klaviyo connector with API key\")[/dim]\n> " + ).strip() + if desc: + ctx.invoke(create, description=desc) + elif choice == "2": + console.rule("[bold cyan]Update a connector[/bold cyan]", style="cyan") + console.print() + proj_path = _prompt_connector_project_dir() + if proj_path is None: + continue + desc = console.input( + "[bold]What to add:[/bold] " + "[dim](e.g. \"add a delete_contact action\")[/dim]\n> " + ).strip() + if desc: + ctx.invoke( + create, + description=desc, + output=str(proj_path), + module_only=True, + ) + else: + console.print("[dim]No description provided.[/dim]") + elif choice == "3": + console.rule("[bold cyan]Validate a project[/bold cyan]", style="cyan") + console.print() + project = _prompt_connector_project_dir() + if project is not None: + ctx.invoke(validate, path=str(project)) + elif choice == "4": + console.rule("[bold cyan]Run connector locally[/bold cyan]", style="cyan") + console.print() + project = _prompt_connector_project_dir() + if project is not None: + _run_connector(project) + elif choice == "5": + console.rule("[bold cyan]Expose with ngrok[/bold cyan]", style="cyan") + console.print() + project = _prompt_connector_project_dir() + if project is not None: + port = detect_port(project) + _expose_with_ngrok(project, port) + elif choice == "6": + console.rule("[bold cyan]Documentation[/bold cyan]", style="cyan") + console.print(f"[dim]Opening {_DOCS_URL} in your browser…[/dim]") + webbrowser.open(_DOCS_URL) + console.print("[green]Done.[/green]") + elif choice == "7": + console.rule("[bold cyan]Setup AI provider[/bold cyan]", style="cyan") + console.print() + ctx.invoke(setup) + else: + console.print( + "[yellow]Invalid choice.[/yellow] " + "[dim]Enter a number [cyan]1[/cyan]–[cyan]8[/cyan] " + "(see the list above).[/dim]" + ) + _hint_path_instead_of_menu_number(choice, menu_range="1–8") + except SystemExit: + pass + + _pause_return_to_menu(label="main menu") + + +# --------------------------------------------------------------------------- +# workflows setup +# --------------------------------------------------------------------------- + +@cli.command() +@click.pass_context +def setup(ctx: click.Context) -> None: + """Configure your AI provider and API key.""" + _banner_once(ctx) + _ensure_dotenv() + _run_setup() + + +# --------------------------------------------------------------------------- +# workflows create +# --------------------------------------------------------------------------- + +@cli.command() +@click.argument("description") +@click.option( + "-o", "--output", + default=".", + type=click.Path(file_okay=False), + help="Parent directory for the generated project.", +) +@click.option("--dry-run", is_flag=True, help="Preview without writing files.") +@click.option( + "--no-ai", + is_flag=True, + help="Use template matching only (no LLM call).", +) +@click.option( + "--module-only", + is_flag=True, + help="Generate only module files into an existing connector project.", +) +@click.pass_context +def create( + ctx: click.Context, + description: str, + output: str, + dry_run: bool, + no_ai: bool, + module_only: bool, +) -> None: + """Generate a Stacksync module or connector project from a natural-language description.""" + _banner_once(ctx) + _ensure_dotenv() + registry = CapabilityRegistry() + + spec: ConnectorSpec | None = None + + if no_ai: + spec = _template_path(description, registry) + else: + if not _has_api_key(): + configured = _run_setup() + if not configured: + console.print("[yellow]No API key. Falling back to template matching.[/yellow]") + spec = _template_path(description, registry) + + if spec is None: + spec = _ai_path(description, registry) + + if spec is None: + console.print( + "[red]Could not generate a module spec from that description.[/red]\n" + "Try being more specific, e.g.:\n" + ' workflows create "Get LinkedIn posts using an API key"' + ) + raise SystemExit(1) + + validation = validate_spec(spec, registry) + if validation.warnings: + for w in validation.warnings: + console.print(f"[yellow] warning:[/yellow] {w}") + if not validation.ok: + for e in validation.errors: + console.print(f"[red] error:[/red] {e}") + console.print("\n[red]Spec has blocking errors. Aborting.[/red]") + raise SystemExit(1) + + _show_preview(spec, module_only=module_only) + + if dry_run: + console.print("\n[dim]--dry-run: no files written.[/dim]") + return + + if not click.confirm("\nContinue?", default=True): + console.print("[dim]Aborted.[/dim]") + return + + output_dir = Path(output).resolve() + + version = _handle_overwrite(spec, output_dir, module_only) + if version is None: + return + if version != spec.version: + spec.version = version + + project_dir, _rationale = compile_connector(spec, output_dir, module_only=module_only) + + port = detect_port(project_dir) + _show_post_gen(spec, project_dir, port, module_only=module_only) + _interactive_menu(project_dir, port) + + +# --------------------------------------------------------------------------- +# workflows validate +# --------------------------------------------------------------------------- + +@cli.command() +@click.option( + "-p", "--path", + default=".", + type=click.Path(exists=True, file_okay=False), + help="Path to a connector project directory.", +) +@click.pass_context +def validate(ctx: click.Context, path: str) -> None: + """Validate a generated connector/module for Stacksync compatibility.""" + _banner_once(ctx) + project = Path(path).resolve() + console.print(f"\nValidating connector at [bold]{project}[/bold]...\n") + + ok = True + app_cfg = project / "app_config.yaml" + if app_cfg.exists(): + port = detect_port(project) + console.print(f" app_config.yaml: [green]OK[/green] (port: {port})") + else: + console.print(" app_config.yaml: [red]MISSING[/red]") + ok = False + + modules_root = project / MODULES_DIR + if not modules_root.is_dir(): + console.print(f" {MODULES_DIR}/: [red]MISSING[/red]") + ok = False + return + + for module_dir in sorted(modules_root.iterdir()): + if not module_dir.is_dir(): + continue + for version_dir in sorted(module_dir.iterdir()): + if not version_dir.is_dir(): + continue + rel = version_dir.relative_to(project) + console.print(f" {rel}/:") + + cfg = version_dir / "module_config.yaml" + if cfg.exists(): + console.print(" module_config.yaml: [green]OK[/green]") + else: + console.print(" module_config.yaml: [red]MISSING[/red]") + ok = False + + schema_path = version_dir / "schema.json" + if schema_path.exists(): + try: + schema = json.loads(schema_path.read_text()) + ver = schema.get("metadata", {}).get("workflows_module_schema_version", "?") + n_fields = len(schema.get("fields", [])) + console.print(f" schema.json: [green]OK[/green] ({n_fields} fields, Module Schema v{ver})") + except json.JSONDecodeError: + console.print(" schema.json: [red]INVALID JSON[/red]") + ok = False + else: + console.print(" schema.json: [red]MISSING[/red]") + ok = False + + route = version_dir / "route.py" + if route.exists(): + code = route.read_text() + has_execute = bool(re.search(r'"/execute"', code)) + has_content = bool(re.search(r'"/content"', code)) + has_schema = bool(re.search(r'"/schema"', code)) + parts = ["/execute" if has_execute else "[red]/execute MISSING[/red]"] + if has_content: + parts.append("/content") + if has_schema: + parts.append("/schema") + console.print(f" route.py: [green]OK[/green] ({', '.join(parts)})") + if not has_execute: + ok = False + else: + console.print(" route.py: [red]MISSING[/red]") + ok = False + + console.print() + if ok: + console.print("[green bold]All checks passed.[/green bold]\n") + else: + console.print("[red bold]Some checks failed.[/red bold]\n") + raise SystemExit(1) + + +# --------------------------------------------------------------------------- +# workflows list +# --------------------------------------------------------------------------- + +@cli.command("list") +def list_capabilities() -> None: + """List all known app capabilities in the registry.""" + registry = CapabilityRegistry() + if not registry.slugs(): + console.print("[dim]No capabilities found.[/dim]") + return + + table_lines: list[str] = [] + for slug in registry.slugs(): + m = registry.get(slug) + if m is None: + continue + actions = ", ".join(m.action_names()) or "(none)" + triggers = ", ".join(m.trigger_names()) or "(none)" + table_lines.append( + f" [bold]{slug:12s}[/bold] " + f"actions: {actions} | triggers: {triggers}" + ) + + console.print(Panel( + "\n".join(table_lines), + title="[bold]Available capabilities[/bold]", + border_style="blue", + )) + + +# --------------------------------------------------------------------------- +# workflows inspect +# --------------------------------------------------------------------------- + +@cli.command() +@click.argument("app_slug") +def inspect(app_slug: str) -> None: + """Show detailed capabilities for a specific app.""" + registry = CapabilityRegistry() + manifest = registry.get(app_slug) + if manifest is None: + console.print(f"[red]Unknown app: '{app_slug}'[/red]") + console.print(f"Available: {', '.join(registry.slugs())}") + raise SystemExit(1) + + tree = Tree(f"[bold]{manifest.app.name}[/bold] ({manifest.app.slug})") + tree.add(f"[dim]{manifest.app.description}[/dim]") + tree.add(f"Auth: {manifest.app.auth.type}") + + if manifest.actions: + actions_branch = tree.add("[bold]Actions[/bold]") + for a in manifest.actions: + fields = ", ".join(f.name for f in a.required_fields) or "(none)" + actions_branch.add(f"{a.name} [{a.category}] -- {a.description} (fields: {fields})") + + if manifest.triggers: + triggers_branch = tree.add("[bold]Triggers[/bold]") + for t in manifest.triggers: + triggers_branch.add(f"{t.name} ({t.event}) -- {t.description}") + + console.print(tree) + + +# --------------------------------------------------------------------------- +# workflows guide +# --------------------------------------------------------------------------- + +@cli.group() +def guide() -> None: + """Step-by-step guidance for the Stacksync connector workflow.""" + + +@guide.command("run") +def guide_run() -> None: + """How to start the connector locally.""" + port = _detect_port_cwd() + console.print(Panel( + f"Starting your connector locally:\n\n" + f" 1. cd \n" + f" 2. pip install -r requirements.txt\n" + f" 3. ./run_dev.sh\n\n" + f" Or without Docker:\n" + f" python main.py\n\n" + f" Your connector will start on port {port} " + f"(configurable in app_config.yaml).", + title="[bold]Run locally[/bold]", + border_style="blue", + )) + + +@guide.command("ngrok") +def guide_ngrok() -> None: + """How to expose your connector with ngrok.""" + port = _detect_port_cwd() + console.print(Panel( + f"Exposing your connector with ngrok:\n\n" + f" Detected connector port: {port}\n\n" + f" 1. Open a new terminal\n" + f" 2. Run:\n" + f" ngrok http {port}\n" + f" 3. Copy the public HTTPS URL\n" + f" 4. Paste it in Developer Studio (see: workflows guide register)", + title="[bold]ngrok setup[/bold]", + border_style="blue", + )) + + +@guide.command("register") +def guide_register() -> None: + """How to register the connector in Developer Studio.""" + console.print(Panel( + "Registering your connector in Stacksync Developer Studio:\n\n" + " 1. Open Developer Studio in your browser\n" + " 2. Create a new Custom Connector\n" + " 3. Paste your ngrok URL as the connector base URL\n" + " 4. Configure auth settings to match your connector\n" + " 5. Save and activate the connector\n\n" + " Important: Your connector and workflows must be in the same region.", + title="[bold]Register connector[/bold]", + border_style="blue", + )) + + +@guide.command("test") +def guide_test() -> None: + """How to test your module in a Stacksync workflow.""" + console.print(Panel( + "Testing your module in a Stacksync workflow:\n\n" + " 1. Create a new workflow in Stacksync (same region as your connector)\n" + " 2. Add a step using your custom connector\n" + " 3. Select the action/module you generated\n" + " 4. Fill in the form fields\n" + " 5. Run the workflow and check the response\n\n" + " Tip: Check the connector logs in your terminal for debugging.", + title="[bold]Test in workflow[/bold]", + border_style="blue", + )) + + +# --------------------------------------------------------------------------- +# Internal helpers +# --------------------------------------------------------------------------- + +def _detect_port_cwd() -> int: + """Try to read port from app_config.yaml in the current directory.""" + return detect_port(Path.cwd()) + + +def _template_path( + description: str, + registry: CapabilityRegistry, +) -> ConnectorSpec | None: + result = match_template(description) + if result is not None: + kw = ", ".join(result.matched_keywords) + console.print( + f"[green]Matched template '{result.template_name}' " + f"(keywords: {kw})[/green]" + ) + return result.spec + return None + + +def _ai_path( + description: str, + registry: CapabilityRegistry, +) -> ConnectorSpec | None: + try: + from ..ai.planner import Planner, PlannerError + except ImportError: + console.print( + "[yellow]AI planner unavailable (missing dependency). " + "Falling back to template matching.[/yellow]" + ) + return _template_path(description, registry) + + try: + planner = Planner(registry) + except (ImportError, PlannerError) as exc: + console.print(f"[yellow]AI planner init failed: {exc}[/yellow]") + console.print("[yellow]Falling back to template matching.[/yellow]") + return _template_path(description, registry) + except Exception: + console.print_exception(max_frames=4) + console.print("[yellow]Unexpected error initializing planner. Falling back to template matching.[/yellow]") + return _template_path(description, registry) + + # Phase 1: parse intent and build prompt (instant) + console.print(" [bold][1/3][/bold] Parsing intent…", end=" ") + try: + system, user_msg = planner.build_prompt(description) + except Exception as exc: + console.print("[red]failed[/red]") + console.print(f"[yellow]{exc}[/yellow]") + return _template_path(description, registry) + console.print("[green]done[/green]") + + # Phase 2: call LLM (slow) + t0 = time.monotonic() + try: + with console.status(" [bold][2/3][/bold] Calling AI provider…"): + spec = planner.call_llm(system, user_msg) + except PlannerError as exc: + console.print(f" [bold][2/3][/bold] Calling AI provider… [red]failed[/red]") + console.print(f"[yellow]{exc}[/yellow]") + console.print("[yellow]Falling back to template matching.[/yellow]") + return _template_path(description, registry) + except KeyboardInterrupt: + console.print("\n[dim]Cancelled.[/dim]") + return None + except Exception: + console.print(f" [bold][2/3][/bold] Calling AI provider… [red]failed[/red]") + console.print_exception(max_frames=6) + console.print("[yellow]AI generation failed. Falling back to template matching.[/yellow]") + return _template_path(description, registry) + elapsed = time.monotonic() - t0 + console.print(f" [bold][2/3][/bold] Calling AI provider… [green]done[/green] [dim]({elapsed:.0f}s)[/dim]") + + # Phase 3: validate (instant) + console.print(" [bold][3/3][/bold] Validating spec…", end=" ") + console.print("[green]done[/green]") + + if spec.needs_clarification: + from ..ai.clarifier import render_clarification + + answers = render_clarification(spec) + if answers: + try: + with console.status(" Refining with your answers…"): + spec = planner.refine(spec, answers) + except PlannerError as exc: + console.print(f"[yellow]Refinement failed: {exc}[/yellow]") + except Exception: + console.print_exception(max_frames=6) + console.print("[yellow]Refinement failed, using initial spec.[/yellow]") + + return spec + + +def _show_preview(spec: ConnectorSpec, *, module_only: bool = False) -> None: + """Compact preview: modules, fields summary, auth, endpoints.""" + lines: list[str] = [] + + action_names = [a.name for a in spec.actions] + trigger_names = [t.name for t in spec.triggers] + module_summary = ", ".join(action_names + trigger_names) + n_a, n_t = len(action_names), len(trigger_names) + counts = [] + if n_a: + counts.append(f"{n_a} action{'s' if n_a != 1 else ''}") + if n_t: + counts.append(f"{n_t} trigger{'s' if n_t != 1 else ''}") + lines.append(f"[bold]Modules:[/bold] {module_summary} ({', '.join(counts)})") + + all_fields = [] + for a in spec.actions: + all_fields.extend(a.required_fields) + all_fields.extend(a.optional_fields) + for t in spec.triggers: + all_fields.extend(t.payload_fields) + + max_show = 8 + if all_fields: + shown = all_fields[:max_show] + field_names = ", ".join(f.name for f in shown) + extra = f" (+{len(all_fields) - max_show} more)" if len(all_fields) > max_show else "" + lines.append(f"[bold]Fields:[/bold] {field_names}{extra}") + + auth_display = _AUTH_DISPLAY.get(spec.auth.type, spec.auth.type) + lines.append(f"[bold]Auth:[/bold] {auth_display}") + + endpoints = ["/execute"] + if needs_content_endpoint(all_fields): + endpoints.append("/content") + if needs_schema_endpoint(all_fields): + endpoints.append("/schema") + lines.append(f"[bold]Endpoints:[/bold] {', '.join(endpoints)}") + + console.print(Panel( + "\n".join(lines), + title="[bold]Preview[/bold]", + border_style="green", + )) + + +def _handle_overwrite( + spec: ConnectorSpec, + output_dir: Path, + module_only: bool, +) -> Optional[str]: + """Check for existing module paths and handle overwrite/versioning. + + Returns the version string to use, or None to abort. + """ + project_dir = output_dir if module_only else (output_dir / spec.directory_name) + for a in spec.actions: + target = project_dir / MODULES_DIR / a.name / spec.version + if target.exists(): + return _prompt_overwrite(target, spec.version) + for t in spec.triggers: + target = project_dir / MODULES_DIR / t.name / spec.version + if target.exists(): + return _prompt_overwrite(target, spec.version) + return spec.version + + +def _prompt_overwrite(path: Path, current_version: str) -> Optional[str]: + console.print(f"\n[yellow]A module already exists at {path}[/yellow]") + choice = Prompt.ask( + "Choose an option", + choices=["abort", "overwrite", "new-version"], + default="abort", + ) + if choice == "abort": + console.print("[dim]Aborted.[/dim]") + return None + if choice == "overwrite": + return current_version + num = int(current_version.lstrip("v")) + 1 if current_version.lstrip("v").isdigit() else 2 + return f"v{num}" + + +def _show_post_gen( + spec: ConnectorSpec, + project_dir: Path, + port: int, + *, + module_only: bool = False, +) -> None: + """Display post-generation summary and next steps.""" + files: list[str] = [] + for a in spec.actions: + base = f"{MODULES_DIR}/{a.name}/{spec.version}" + files.extend([f"{base}/module_config.yaml", f"{base}/schema.json", f"{base}/route.py"]) + for t in spec.triggers: + base = f"{MODULES_DIR}/{t.name}/{spec.version}" + files.extend([f"{base}/module_config.yaml", f"{base}/schema.json", f"{base}/route.py"]) + + file_list = "\n".join(f" {f}" for f in files) + + if module_only: + region = _detect_region(project_dir) + body = ( + f"[green bold]Module generated successfully[/green bold]\n\n" + f"Files created:\n{file_list}\n\n" + f"Next: restart your connector and test the new module " + f"(Stacksync region: [bold]{region}[/bold])." + ) + else: + region = _detect_region(project_dir) + body = ( + f"[green bold]Module generated successfully[/green bold]\n\n" + f"Files created:\n{file_list}\n\n" + f"[bold]Next steps:[/bold]\n" + f" 1. Start the connector locally:\n" + f" ./run_dev.sh\n" + f" 2. Expose your local backend:\n" + f" ngrok http {port}\n" + f" 3. Copy the ngrok URL into Stacksync Developer Studio " + f"(region: [bold]{region}[/bold])\n" + f" 4. Create a workflow in the same region ([bold]{region}[/bold])\n" + f" 5. Add your new action and test it" + ) + + console.print(Panel(body, border_style="green")) + + +def _interactive_menu(project_dir: Path, port: int) -> None: + """Offer guided next-step actions after generation.""" + region = _detect_region(project_dir) + while True: + console.print(Panel( + f" [bold cyan][1][/bold cyan] Run the connector " + f"[dim]Start locally via run_dev.sh[/dim]\n" + f" [bold cyan][2][/bold cyan] Expose with ngrok " + f"[dim]run_dev.sh then ngrok http {port}[/dim]\n" + f" [bold cyan][3][/bold cyan] Open documentation " + f"[dim]Stacksync developer docs[/dim]\n" + f" [bold cyan][4][/bold cyan] Exit\n\n" + f" [dim]Stacksync region:[/dim] [bold]{region}[/bold]", + title="[bold]What would you like to do next?[/bold]", + border_style="cyan", + padding=(1, 2), + )) + try: + choice = console.input("[bold]Select [cyan][1-4][/cyan]:[/bold] ").strip() + except (KeyboardInterrupt, EOFError): + _prompt_stop_managed_services_if_any() + return + + if choice == "4" or choice == "": + _prompt_stop_managed_services_if_any() + return + + if choice == "1": + console.rule("[bold cyan]Run the connector[/bold cyan]", style="cyan") + console.print() + _run_connector(project_dir) + elif choice == "2": + console.rule("[bold cyan]Expose with ngrok[/bold cyan]", style="cyan") + console.print() + _expose_with_ngrok(project_dir, port) + elif choice == "3": + console.rule("[bold cyan]Documentation[/bold cyan]", style="cyan") + console.print(f"[dim]Opening {_DOCS_URL} in your browser…[/dim]") + webbrowser.open(_DOCS_URL) + console.print("[green]Done.[/green]") + else: + console.print( + "[yellow]Invalid choice.[/yellow] " + "[dim]Enter [cyan]1[/cyan]–[cyan]4[/cyan].[/dim]" + ) + _hint_path_instead_of_menu_number(choice, menu_range="1–4") + + _pause_return_to_menu(label="next steps menu") + + +def _run_connector(project_dir: Path) -> None: + """Try to start the connector from *project_dir*.""" + run_dev = project_dir / "run_dev.sh" + main_py = project_dir / "main.py" + + if run_dev.exists(): + console.print(f"\n[bold]Starting connector via run_dev.sh…[/bold]") + console.print( + "[dim]Press Ctrl+C to stop the connector. " + "If this CLI started ngrok or a background connector earlier, " + "you will be asked whether to stop those next.[/dim]\n" + ) + try: + subprocess.run(["bash", str(run_dev)], cwd=str(project_dir)) + except KeyboardInterrupt: + console.print("\n[dim]Interrupted.[/dim]") + _prompt_stop_managed_services_if_any() + elif main_py.exists(): + console.print(f"\n[bold]Starting connector via python main.py…[/bold]") + console.print( + "[dim]Press Ctrl+C to stop the connector. " + "If this CLI started ngrok or a background connector earlier, " + "you will be asked whether to stop those next.[/dim]\n" + ) + try: + subprocess.run(["python", str(main_py)], cwd=str(project_dir)) + except KeyboardInterrupt: + console.print("\n[dim]Interrupted.[/dim]") + _prompt_stop_managed_services_if_any() + else: + console.print( + f"\n[yellow]No run_dev.sh or main.py found in {project_dir}.[/yellow]\n" + f" cd {project_dir}\n" + f" pip install -r requirements.txt\n" + f" python main.py" + ) diff --git a/src/workflows_cdk/core/dynamic_routing.py b/src/workflows_cdk/core/dynamic_routing.py index 41174f6..36e7376 100644 --- a/src/workflows_cdk/core/dynamic_routing.py +++ b/src/workflows_cdk/core/dynamic_routing.py @@ -31,6 +31,14 @@ +def _safe_getcwd() -> Optional[str]: + """Return cwd if valid; None if the process cwd was deleted (raises FileNotFoundError).""" + try: + return os.path.abspath(os.getcwd()) + except (FileNotFoundError, OSError): + return None + + def load_app_config(app_dir: str) -> Dict[str, Any]: """Load app config with proper file handle cleanup.""" config_path = os.path.join(app_dir, "app_config.yaml") @@ -204,23 +212,59 @@ def __init__(self, app: Optional[Flask] = None, *, self.routes: List[Dict[str, Any]] = [] # List to store module metadata self.modules_list: List[Dict[str, Any]] = [] - # Flask application instance - self.app: Optional[Flask] = None + # Flask app set early so ``_project_root()`` can use ``app.root_path`` (stable vs getcwd). + self.app: Optional[Flask] = app self._router_instance = self + self._project_root_cache: Optional[str] = None self.environment = get_environment() - - # Initial app config loading - self.app_config = load_app_config(os.getcwd()) + + # Initial app config loading (avoids bare os.getcwd() — can raise FileNotFoundError) + self.app_config = load_app_config(self._project_root()) self.config = config or {} self.sentry_dsn = sentry_dsn self.cors_origins = cors_origins or ["*"] - + # Apply configuration settings self.refresh_app_config_variables(self.app_config) if app is not None: self.init_app(app) + def _project_root(self) -> str: + """Directory containing ``app_config.yaml`` and the routes tree. + + ``os.getcwd()`` can raise ``FileNotFoundError`` if the process cwd was deleted. Prefer + ``WORKFLOWS_PROJECT_ROOT`` / ``STACKSYNC_CONNECTOR_ROOT``, then Flask ``root_path`` + (use ``Flask(__name__)`` in ``main.py``), then cwd, then ``/usr/src/app`` in Docker. + """ + if self._project_root_cache is not None: + return self._project_root_cache + for key in ("WORKFLOWS_PROJECT_ROOT", "STACKSYNC_CONNECTOR_ROOT"): + raw = os.environ.get(key) + if raw: + p = os.path.abspath(raw) + if os.path.isdir(p): + self._project_root_cache = p + return p + flask_app = self.app + if flask_app is not None: + rp = getattr(flask_app, "root_path", None) + if rp: + p = os.path.abspath(rp) + if os.path.isdir(p): + self._project_root_cache = p + return p + cwd = _safe_getcwd() + if cwd and os.path.isdir(cwd): + self._project_root_cache = cwd + return cwd + docker_default = "/usr/src/app" + if os.path.isdir(docker_default): + self._project_root_cache = docker_default + return docker_default + self._project_root_cache = os.path.abspath(".") + return self._project_root_cache + def refresh_app_config_variables(self, app_config: Dict[str, Any]) -> None: """Apply configuration settings to the router instance. @@ -301,7 +345,7 @@ def _create_route_info(self, function: Callable, rule: Optional[str] = None, opt raise ValueError("Could not determine the module path for the function") - routes_directory = Path(os.path.join(os.getcwd(), self.routes_directory)) + routes_directory = Path(os.path.join(self._project_root(), self.routes_directory)) module_file_path = Path(function_module.__file__).resolve() # Check if the module is in the routes directory and generate metadata @@ -369,13 +413,13 @@ def _scan_routes_directory(self) -> list: Returns: list: List of route file paths (Path objects) """ - current_working_directory = os.getcwd() - routes_directory = Path(os.path.join(current_working_directory, self.routes_directory)) - + project_root = self._project_root() + routes_directory = Path(os.path.join(project_root, self.routes_directory)) + if not routes_directory.exists(): print_error(f"Routes directory not found at: {routes_directory}") return [] - + # Find all Python files in routes directory and its subdirectories # Only include .py files that are directly inside a version directory (v1, v2, vX, etc.) version_dir_pattern = re.compile(r"^v[\w\d]+$") @@ -402,9 +446,9 @@ def _collect_route_information(self) -> tuple: Returns: tuple: (routes_info, modules_list) containing metadata """ - current_working_directory = os.getcwd() - routes_directory = Path(os.path.join(current_working_directory, self.routes_directory)) - + project_root = self._project_root() + routes_directory = Path(os.path.join(project_root, self.routes_directory)) + # Lists to store collected data routes_info = [] modules_list = [] @@ -495,22 +539,22 @@ def discover_routes(self) -> None: Key issue: Each discovery loads Python modules that stay in memory forever. In serverless, this grows with each container reuse. """ - current_working_directory = os.getcwd() - + project_root = self._project_root() + # Collect route info without any imports first routes_info, module_list = self._collect_route_information() - + # Add module metadata (but clear old references first) for module in module_list: if not any(m.get("module_id") == module["module_id"] for m in self.modules_list): self.modules_list.append(module) - + if not routes_info: return - # Add working directory to path for imports - if current_working_directory not in sys.path: - sys.path.insert(0, current_working_directory) + # Add project root to path for imports + if project_root not in sys.path: + sys.path.insert(0, project_root) # Use clean import context to prevent module accumulation with clean_module_import(): @@ -518,7 +562,7 @@ def discover_routes(self) -> None: original_router_module = sys.modules.get('main', None) temporary_main_module = types.ModuleType('main') setattr(temporary_main_module, 'router', self) - setattr(temporary_main_module, '__file__', os.path.join(current_working_directory, 'main.py')) + setattr(temporary_main_module, '__file__', os.path.join(project_root, 'main.py')) sys.modules['main'] = temporary_main_module try: @@ -568,8 +612,8 @@ def discover_routes(self) -> None: sys.modules.pop('main', None) # Remove the project root from sys.path - if current_working_directory in sys.path: - sys.path.remove(current_working_directory) + if project_root in sys.path: + sys.path.remove(project_root) def register_error_handlers(self, app: Flask) -> None: """Register error handlers for the application.""" @@ -616,7 +660,7 @@ def health_check(): def app_config(): try: # Reload app config to get fresh settings - fresh_app_config = load_app_config(os.getcwd()) + fresh_app_config = load_app_config(self._project_root()) # Apply the fresh configuration self.refresh_app_config_variables(fresh_app_config) @@ -681,7 +725,7 @@ def routes(): def register_schema_routes(self, app: Flask) -> None: """Register schema routes without loading schema content into memory.""" - routes_path = os.path.join(os.getcwd(), self.routes_directory) + routes_path = os.path.join(self._project_root(), self.routes_directory) # Only proceed if auto-registration is enabled if self.app_settings.get("automatically_register_schema_routes", True): @@ -705,8 +749,8 @@ def _handle_dynamic_schema_request(self, dynamic_path: str) -> FlaskResponse: """ try: # Construct the full path to the schema file - current_working_directory = os.getcwd() - routes_directory = os.path.join(current_working_directory, self.routes_directory) + project_root = self._project_root() + routes_directory = os.path.join(project_root, self.routes_directory) schema_file_path = os.path.join(routes_directory, dynamic_path, 'schema.json') # Check if the schema file exists @@ -757,8 +801,8 @@ def _create_dynamic_schema_handler(self, route_path: str) -> Callable[[], FlaskR def dynamic_schema_handler() -> FlaskResponse: try: # Get absolute path to schema file - current_working_directory = os.getcwd() - routes_directory = os.path.join(current_working_directory, self.routes_directory) + project_root = self._project_root() + routes_directory = os.path.join(project_root, self.routes_directory) schema_file_path = os.path.join(routes_directory, route_path.lstrip('/'), 'schema.json') # Check if file exists @@ -825,10 +869,11 @@ def _register_schema_route(self, route_path: str, app: Optional[Flask] = None) - def init_app(self, app: Flask) -> None: """Initialize the router with a Flask app and register all discovered routes.""" self.app = app - + self._project_root_cache = None + self.clear_accumulated_data() - - self.app_config = load_app_config(os.getcwd()) + + self.app_config = load_app_config(self._project_root()) self.refresh_app_config_variables(self.app_config) # Update Flask configuration diff --git a/src/workflows_cdk/registry/__init__.py b/src/workflows_cdk/registry/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/workflows_cdk/registry/capabilities/hubspot.yaml b/src/workflows_cdk/registry/capabilities/hubspot.yaml new file mode 100644 index 0000000..8e1f060 --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/hubspot.yaml @@ -0,0 +1,76 @@ +app: + name: HubSpot + slug: hubspot + description: "CRM, marketing, and sales automation platform" + auth: + type: oauth2 + scopes: + - "crm.objects.contacts.read" + - "crm.objects.contacts.write" + - "crm.objects.deals.read" + - "crm.objects.deals.write" + - "crm.objects.companies.read" + - "crm.objects.companies.write" + +actions: + - name: create_contact + category: action + description: "Create a new CRM contact" + required_fields: + - { name: email, type: string, description: "Contact email address" } + optional_fields: + - { name: firstname, type: string, description: "First name" } + - { name: lastname, type: string, description: "Last name" } + - { name: phone, type: string, description: "Phone number" } + - { name: company, type: string, description: "Company name" } + + - name: update_contact + category: action + description: "Update an existing CRM contact" + required_fields: + - { name: contact_id, type: string, description: "HubSpot contact ID" } + optional_fields: + - { name: email, type: string, description: "Email address" } + - { name: firstname, type: string, description: "First name" } + - { name: lastname, type: string, description: "Last name" } + - { name: phone, type: string, description: "Phone number" } + + - name: search_contacts + category: search + description: "Search for contacts by property values" + required_fields: + - { name: query, type: string, description: "Search query" } + + - name: create_deal + category: action + description: "Create a new deal in the pipeline" + required_fields: + - { name: dealname, type: string, description: "Deal name" } + - { name: amount, type: number, description: "Deal amount" } + optional_fields: + - { name: pipeline, type: string, description: "Pipeline ID" } + - { name: dealstage, type: string, description: "Stage ID" } + +triggers: + - name: contact_created + event: contact.created + description: "Fires when a new contact is created in HubSpot" + payload_fields: + - { name: id, type: string } + - { name: email, type: string } + - { name: firstname, type: string } + - { name: lastname, type: string } + + - name: deal_updated + event: deal.updated + description: "Fires when a deal is updated" + payload_fields: + - { name: id, type: string } + - { name: dealname, type: string } + - { name: dealstage, type: string } + - { name: amount, type: number } + +examples: + - "HubSpot connector to manage contacts and deals" + - "Create and update HubSpot contacts" + - "HubSpot CRM: create contact, search, update, manage deals" diff --git a/src/workflows_cdk/registry/capabilities/openai.yaml b/src/workflows_cdk/registry/capabilities/openai.yaml new file mode 100644 index 0000000..8b12489 --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/openai.yaml @@ -0,0 +1,51 @@ +app: + name: OpenAI + slug: openai + description: "AI models for text generation, embeddings, and classification" + auth: + type: api_key + scopes: [] + fields: + - { name: api_key, type: string, description: "OpenAI API key" } + +actions: + - name: chat_completion + category: action + description: "Generate a chat completion using GPT models" + required_fields: + - { name: prompt, type: string, description: "User message / prompt" } + optional_fields: + - { name: model, type: string, description: "Model name (default gpt-4o)" } + - { name: system_prompt, type: string, description: "System instruction" } + - { name: temperature, type: number, description: "Sampling temperature 0-2" } + - { name: max_tokens, type: number, description: "Max tokens in response" } + + - name: create_embedding + category: transform + description: "Create a vector embedding from text" + required_fields: + - { name: input, type: string, description: "Text to embed" } + optional_fields: + - { name: model, type: string, description: "Embedding model name" } + + - name: classify_text + category: transform + description: "Classify text into categories using a prompt" + required_fields: + - { name: text, type: string, description: "Text to classify" } + - { name: categories, type: array, description: "List of possible categories" } + + - name: summarize + category: transform + description: "Summarize a block of text" + required_fields: + - { name: text, type: string, description: "Text to summarize" } + optional_fields: + - { name: max_length, type: number, description: "Target summary length in words" } + +triggers: [] + +examples: + - "OpenAI connector for text generation" + - "AI enrichment: classify and summarize text" + - "OpenAI: generate completions, create embeddings, classify" diff --git a/src/workflows_cdk/registry/capabilities/postgres.yaml b/src/workflows_cdk/registry/capabilities/postgres.yaml new file mode 100644 index 0000000..52ddfd5 --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/postgres.yaml @@ -0,0 +1,60 @@ +app: + name: PostgreSQL + slug: postgres + description: "Relational database for structured data storage and queries" + auth: + type: basic + scopes: [] + fields: + - { name: host, type: string, description: "Database host" } + - { name: port, type: number, description: "Database port" } + - { name: database, type: string, description: "Database name" } + - { name: user, type: string, description: "Database user" } + - { name: password, type: string, description: "Database password" } + +actions: + - name: execute_query + category: action + description: "Execute a SQL query and return results" + required_fields: + - { name: query, type: string, description: "SQL query to execute" } + optional_fields: + - { name: params, type: array, description: "Query parameters for prepared statements" } + + - name: insert_row + category: action + description: "Insert a row into a table" + required_fields: + - { name: table, type: string, description: "Table name" } + - { name: data, type: object, description: "Column-value pairs to insert" } + + - name: update_rows + category: action + description: "Update rows matching a condition" + required_fields: + - { name: table, type: string, description: "Table name" } + - { name: data, type: object, description: "Column-value pairs to update" } + - { name: where, type: string, description: "WHERE clause (without the WHERE keyword)" } + + - name: search_rows + category: search + description: "Search rows in a table" + required_fields: + - { name: table, type: string, description: "Table name" } + optional_fields: + - { name: where, type: string, description: "WHERE clause" } + - { name: limit, type: number, description: "Max rows to return" } + - { name: order_by, type: string, description: "ORDER BY clause" } + +triggers: + - name: new_row + event: row.inserted + description: "Fires when a new row is inserted (polling-based)" + payload_fields: + - { name: table, type: string } + - { name: row, type: object } + +examples: + - "PostgreSQL connector for querying and inserting data" + - "Postgres: execute queries, insert rows, search tables" + - "Database connector to read and write Postgres rows" diff --git a/src/workflows_cdk/registry/capabilities/salesforce.yaml b/src/workflows_cdk/registry/capabilities/salesforce.yaml new file mode 100644 index 0000000..b89353a --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/salesforce.yaml @@ -0,0 +1,70 @@ +app: + name: Salesforce + slug: salesforce + description: "Enterprise CRM and sales management platform" + auth: + type: oauth2 + scopes: + - "api" + - "refresh_token" + +actions: + - name: create_lead + category: action + description: "Create a new lead in Salesforce" + required_fields: + - { name: LastName, type: string, description: "Lead last name" } + - { name: Company, type: string, description: "Company name" } + optional_fields: + - { name: FirstName, type: string, description: "First name" } + - { name: Email, type: string, description: "Email address" } + - { name: Phone, type: string, description: "Phone number" } + + - name: create_opportunity + category: action + description: "Create a new sales opportunity" + required_fields: + - { name: Name, type: string, description: "Opportunity name" } + - { name: StageName, type: string, description: "Current stage" } + - { name: CloseDate, type: string, description: "Expected close date (YYYY-MM-DD)" } + optional_fields: + - { name: Amount, type: number, description: "Deal amount" } + - { name: AccountId, type: string, description: "Associated account ID" } + + - name: search_records + category: search + description: "Search Salesforce records using SOQL" + required_fields: + - { name: query, type: string, description: "SOQL query string" } + + - name: update_record + category: action + description: "Update an existing Salesforce record" + required_fields: + - { name: object_type, type: string, description: "Salesforce object type (Lead, Contact, etc.)" } + - { name: record_id, type: string, description: "Record ID" } + - { name: fields, type: object, description: "Fields to update" } + +triggers: + - name: new_lead + event: lead.created + description: "Fires when a new lead is created" + payload_fields: + - { name: Id, type: string } + - { name: LastName, type: string } + - { name: Email, type: string } + - { name: Company, type: string } + + - name: opportunity_closed + event: opportunity.closed + description: "Fires when an opportunity is closed (won or lost)" + payload_fields: + - { name: Id, type: string } + - { name: Name, type: string } + - { name: StageName, type: string } + - { name: Amount, type: number } + +examples: + - "Salesforce connector to manage leads and opportunities" + - "Create Salesforce leads and search records" + - "Salesforce CRM: leads, opportunities, record updates" diff --git a/src/workflows_cdk/registry/capabilities/slack.yaml b/src/workflows_cdk/registry/capabilities/slack.yaml new file mode 100644 index 0000000..aedad13 --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/slack.yaml @@ -0,0 +1,66 @@ +app: + name: Slack + slug: slack + description: "Team messaging and collaboration platform" + auth: + type: oauth2 + scopes: + - "chat:write" + - "channels:read" + - "channels:history" + - "reactions:read" + - "reactions:write" + - "users:read" + +actions: + - name: send_message + category: action + description: "Send a message to a Slack channel or DM" + required_fields: + - { name: channel, type: string, description: "Channel name or ID" } + - { name: text, type: string, description: "Message body" } + optional_fields: + - { name: thread_ts, type: string, description: "Thread timestamp for replies" } + + - name: list_channels + category: search + description: "List all channels in the workspace" + required_fields: [] + + - name: add_reaction + category: action + description: "Add an emoji reaction to a message" + required_fields: + - { name: channel, type: string, description: "Channel containing the message" } + - { name: timestamp, type: string, description: "Message timestamp" } + - { name: emoji, type: string, description: "Emoji name without colons" } + + - name: list_users + category: search + description: "List all users in the workspace" + required_fields: [] + +triggers: + - name: new_message + event: message.created + description: "Fires when a new message is posted in a channel" + payload_fields: + - { name: channel, type: string } + - { name: text, type: string } + - { name: user, type: string } + - { name: ts, type: string } + + - name: new_reaction + event: reaction.added + description: "Fires when a reaction is added to a message" + payload_fields: + - { name: channel, type: string } + - { name: user, type: string } + - { name: reaction, type: string } + - { name: ts, type: string } + +examples: + - "Slack connector that sends messages to channels" + - "Listen for new Slack messages" + - "Slack: send messages, list channels, react to new messages" + - "Post notifications to Slack" diff --git a/src/workflows_cdk/registry/capabilities/stripe.yaml b/src/workflows_cdk/registry/capabilities/stripe.yaml new file mode 100644 index 0000000..8baa7f6 --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/stripe.yaml @@ -0,0 +1,62 @@ +app: + name: Stripe + slug: stripe + description: "Online payment processing platform" + auth: + type: api_key + scopes: [] + fields: + - { name: api_key, type: string, description: "Stripe secret API key" } + +actions: + - name: create_customer + category: action + description: "Create a new Stripe customer" + required_fields: + - { name: email, type: string, description: "Customer email address" } + optional_fields: + - { name: name, type: string, description: "Customer full name" } + - { name: metadata, type: object, description: "Key-value metadata" } + + - name: create_charge + category: action + description: "Create a payment charge" + required_fields: + - { name: amount, type: number, description: "Amount in cents" } + - { name: currency, type: string, description: "Three-letter ISO currency code" } + - { name: customer, type: string, description: "Customer ID" } + + - name: list_customers + category: search + description: "List Stripe customers" + required_fields: [] + optional_fields: + - { name: limit, type: number, description: "Max results to return" } + + - name: get_balance + category: search + description: "Retrieve the current account balance" + required_fields: [] + +triggers: + - name: customer_created + event: customer.created + description: "Fires when a new customer is created" + payload_fields: + - { name: id, type: string } + - { name: email, type: string } + - { name: name, type: string } + + - name: payment_succeeded + event: payment_intent.succeeded + description: "Fires when a payment succeeds" + payload_fields: + - { name: id, type: string } + - { name: amount, type: number } + - { name: currency, type: string } + - { name: customer, type: string } + +examples: + - "Stripe connector to create customers and process payments" + - "Listen for new Stripe payments" + - "Stripe: create customer, charge, list customers" diff --git a/src/workflows_cdk/registry/manifest.py b/src/workflows_cdk/registry/manifest.py new file mode 100644 index 0000000..93416e7 --- /dev/null +++ b/src/workflows_cdk/registry/manifest.py @@ -0,0 +1,87 @@ +""" +Pydantic models that mirror the capability.yaml manifest format. + +Each YAML file in ``registry/capabilities/`` is loaded into an +``AppManifest`` instance so both the LLM planner and the deterministic +validator work against a single, typed source of truth. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Literal, Optional + +import yaml +from pydantic import BaseModel, Field + + +class ManifestField(BaseModel): + name: str + type: Literal["string", "number", "boolean", "object", "array"] = "string" + description: str = "" + + +class ManifestAuth(BaseModel): + type: Literal["oauth2", "api_key", "basic", "none"] = "oauth2" + scopes: list[str] = Field(default_factory=list) + fields: list[ManifestField] = Field(default_factory=list) + + +class ManifestAction(BaseModel): + name: str + category: Literal["action", "trigger", "search", "transform"] = "action" + description: str = "" + required_fields: list[ManifestField] = Field(default_factory=list) + optional_fields: list[ManifestField] = Field(default_factory=list) + + +class ManifestTrigger(BaseModel): + name: str + event: str = "" + description: str = "" + payload_fields: list[ManifestField] = Field(default_factory=list) + + +class ManifestApp(BaseModel): + name: str + slug: str + description: str = "" + auth: ManifestAuth = Field(default_factory=ManifestAuth) + + +class AppManifest(BaseModel): + """Top-level model for a single ``capability.yaml`` file.""" + + app: ManifestApp + actions: list[ManifestAction] = Field(default_factory=list) + triggers: list[ManifestTrigger] = Field(default_factory=list) + examples: list[str] = Field(default_factory=list) + + @classmethod + def from_yaml(cls, path: Path) -> "AppManifest": + with open(path, "r") as fh: + raw = yaml.safe_load(fh) or {} + return cls.model_validate(raw) + + def action_names(self) -> list[str]: + return [a.name for a in self.actions] + + def trigger_names(self) -> list[str]: + return [t.name for t in self.triggers] + + def summary_for_llm(self) -> dict: + """Compact JSON-serialisable summary injected into the LLM prompt.""" + return { + "app": self.app.slug, + "name": self.app.name, + "description": self.app.description, + "auth_type": self.app.auth.type, + "actions": [ + {"name": a.name, "category": a.category, "description": a.description} + for a in self.actions + ], + "triggers": [ + {"name": t.name, "event": t.event, "description": t.description} + for t in self.triggers + ], + } diff --git a/src/workflows_cdk/registry/registry.py b/src/workflows_cdk/registry/registry.py new file mode 100644 index 0000000..56e9517 --- /dev/null +++ b/src/workflows_cdk/registry/registry.py @@ -0,0 +1,77 @@ +""" +Capability registry -- loads all YAML manifests and provides lookup / search. + +Usage:: + + registry = CapabilityRegistry() # auto-loads built-in manifests + registry.load_directory(Path("./custom_capabilities")) # extend with user-provided + slack = registry.get("slack") # by slug + hits = registry.search("send message slack") # keyword search +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Optional + +from .manifest import AppManifest + +logger = logging.getLogger(__name__) + +_BUILTIN_DIR = Path(__file__).parent / "capabilities" + + +class CapabilityRegistry: + """In-memory index of every known app manifest.""" + + def __init__(self, *, load_builtins: bool = True) -> None: + self._manifests: dict[str, AppManifest] = {} + if load_builtins: + self.load_directory(_BUILTIN_DIR) + + def load_directory(self, directory: Path) -> None: + if not directory.is_dir(): + logger.warning("Capabilities directory not found: %s", directory) + return + for path in sorted(directory.glob("*.yaml")): + try: + manifest = AppManifest.from_yaml(path) + self._manifests[manifest.app.slug] = manifest + except Exception: + logger.warning("Failed to load manifest %s", path, exc_info=True) + + def get(self, slug: str) -> Optional[AppManifest]: + return self._manifests.get(slug) + + def all(self) -> list[AppManifest]: + return list(self._manifests.values()) + + def slugs(self) -> list[str]: + return sorted(self._manifests.keys()) + + def search(self, query: str) -> list[AppManifest]: + """Rank manifests by simple keyword overlap with *query*.""" + tokens = set(query.lower().split()) + scored: list[tuple[int, AppManifest]] = [] + + for manifest in self._manifests.values(): + corpus = " ".join([ + manifest.app.name.lower(), + manifest.app.slug.lower(), + manifest.app.description.lower(), + " ".join(manifest.examples).lower(), + " ".join(a.name.replace("_", " ") for a in manifest.actions), + " ".join(a.description.lower() for a in manifest.actions), + " ".join(t.name.replace("_", " ") for t in manifest.triggers), + ]) + score = sum(1 for t in tokens if t in corpus) + if score > 0: + scored.append((score, manifest)) + + scored.sort(key=lambda pair: pair[0], reverse=True) + return [m for _, m in scored] + + def summaries_for_llm(self) -> list[dict]: + """All manifests in the compact dict format consumed by the planner prompt.""" + return [m.summary_for_llm() for m in self._manifests.values()] diff --git a/src/workflows_cdk/spec/__init__.py b/src/workflows_cdk/spec/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/workflows_cdk/spec/compiler.py b/src/workflows_cdk/spec/compiler.py new file mode 100644 index 0000000..da4ffba --- /dev/null +++ b/src/workflows_cdk/spec/compiler.py @@ -0,0 +1,759 @@ +""" +Compiler: turns a ``ConnectorSpec`` into a scaffolded CDK project on disk. + +The generated project is immediately runnable with ``./run_dev.sh`` and +follows every convention documented in the Stacksync Workflows platform +(module schema v1.0.0, /execute + /content + /schema endpoints, Docker +deployment, file-based route discovery under ``src/modules/``). +""" + +from __future__ import annotations + +import json +import textwrap +from pathlib import Path +from typing import Optional + +import yaml + +from .connector_spec import ActionSpec, AuthSpec, ConnectorSpec, FieldSpec, TriggerSpec + +MODULES_DIR = "src/modules" +DEFAULT_PORT = 2003 + + +def needs_content_endpoint(fields: list[FieldSpec]) -> bool: + """True when any field requires dynamic dropdown choices via /content.""" + return any(f.dynamic_content for f in fields) + + +def needs_schema_endpoint(fields: list[FieldSpec]) -> bool: + """True when any field declares a depends_on relationship requiring /schema reload.""" + return any(f.depends_on for f in fields) + + +def detect_port(project_dir: Path) -> int: + """Read the port from an existing app_config.yaml, falling back to DEFAULT_PORT.""" + cfg_path = project_dir / "app_config.yaml" + if cfg_path.exists(): + try: + cfg = yaml.safe_load(cfg_path.read_text()) or {} + return int( + cfg.get("local_development_settings", {}).get("port", DEFAULT_PORT) + ) + except Exception: + pass + return DEFAULT_PORT + + +def compile_connector( + spec: ConnectorSpec, + output_dir: Path, + *, + module_only: bool = False, +) -> tuple[Path, list[str]]: + """Write a CDK project (or just module files) to disk. + + Returns ``(project_dir, rationale_lines)`` where *rationale_lines* explains + what was generated and why, suitable for terminal display. + """ + # module_only: *output_dir* is the existing connector root (do not append directory_name). + project_dir = output_dir if module_only else (output_dir / spec.directory_name) + project_dir.mkdir(parents=True, exist_ok=True) + rationale: list[str] = [] + + if not module_only: + _write_main_py(spec, project_dir) + _write_app_config(spec, project_dir) + _write_requirements(project_dir) + _write_deployment_files(spec, project_dir) + _write_gitignore(project_dir) + _write_dot_env(project_dir) + _write_readme(spec, project_dir) + + for action in spec.actions: + r = _write_action_route(spec, action, project_dir) + rationale.extend(r) + + for trigger in spec.triggers: + r = _write_trigger_route(spec, trigger, project_dir) + rationale.extend(r) + + return project_dir, rationale + + +def preview_tree(spec: ConnectorSpec) -> str: + """Return a textual file-tree preview without writing anything.""" + lines = [f"{spec.directory_name}/"] + lines.append(" main.py") + lines.append(" app_config.yaml") + lines.append(" requirements.txt") + lines.append(" .env") + lines.append(" .gitignore") + lines.append(" README.md") + lines.append(" Dockerfile") + lines.append(" run_dev.sh") + lines.append(" config/") + lines.append(" Dockerfile.dev") + lines.append(" entrypoint.sh") + lines.append(" gunicorn_config.py") + lines.append(f" {MODULES_DIR}/") + + for action in spec.actions: + lines.append(f" {action.name}/") + lines.append(f" {spec.version}/") + lines.append(f" route.py") + lines.append(f" schema.json") + lines.append(f" module_config.yaml") + + for trigger in spec.triggers: + lines.append(f" {trigger.name}/") + lines.append(f" {spec.version}/") + lines.append(f" route.py") + lines.append(f" schema.json") + lines.append(f" module_config.yaml") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Static project files +# --------------------------------------------------------------------------- + +def _write_main_py(spec: ConnectorSpec, project_dir: Path) -> None: + """Use ``Flask(__name__)`` so ``app.root_path`` is the project dir (stable under Docker/gunicorn).""" + content = textwrap.dedent("""\ + from flask import Flask + from workflows_cdk import Router + + app = Flask(__name__) + router = Router(app) + + if __name__ == "__main__": + router.run_app(app) + """) + (project_dir / "main.py").write_text(content) + + +def _write_app_config(spec: ConnectorSpec, project_dir: Path) -> None: + config = { + "app_settings": { + "app_type": spec.app_type, + "app_name": spec.app_name, + "app_icon_svg_url": "", + "app_description": f"Stacksync connector for {spec.app_name}", + "routes_directory": MODULES_DIR, + }, + "local_development_settings": { + "sentry_dsn": "", + "cors_origins": ["*"], + "routes_directory": MODULES_DIR, + "debug": True, + "host": "0.0.0.0", + "port": DEFAULT_PORT, + }, + } + (project_dir / "app_config.yaml").write_text( + yaml.dump(config, default_flow_style=False, sort_keys=False) + ) + + +def _write_requirements(project_dir: Path) -> None: + content = textwrap.dedent("""\ + # Flask + flask + werkzeug + requests + # Server + gunicorn==22.0.0 + # Monitoring + sentry-sdk[Flask] + # Core + pydantic>=2.0.0 + PyYAML>=6.0.1 + """) + (project_dir / "requirements.txt").write_text(content) + + +def _write_gitignore(project_dir: Path) -> None: + content = textwrap.dedent("""\ + __pycache__/ + *.pyc + *.pyo + .env + .venv/ + venv/ + *.egg-info/ + dist/ + build/ + .pytest_cache/ + """) + (project_dir / ".gitignore").write_text(content) + + +def _write_dot_env(project_dir: Path) -> None: + content = textwrap.dedent("""\ + ENVIRONMENT=dev + REGION=besg + """) + path = project_dir / ".env" + if not path.exists(): + path.write_text(content) + + +def _write_readme(spec: ConnectorSpec, project_dir: Path) -> None: + name = spec.app_name + actions = ", ".join(a.name for a in spec.actions) or "(none)" + triggers = ", ".join(t.name for t in spec.triggers) or "(none)" + content = textwrap.dedent(f"""\ + # {name} + + Stacksync connector generated with the Workflows CDK. + + ## Quick start + + ```bash + # Start with Docker + ./run_dev.sh + + # Or without Docker + pip install -r requirements.txt + python main.py + ``` + + The connector runs on port {DEFAULT_PORT} by default (see `app_config.yaml`). + + ## Modules + + - **Actions:** {actions} + - **Triggers:** {triggers} + + ## Project structure + + ``` + app_config.yaml Connector configuration + main.py Flask entry point + src/modules/ Module folders (route.py, schema.json, module_config.yaml) + config/ Docker & Gunicorn configs + ``` + + ## Documentation + + https://docs.stacksync.com/workflow-automation/developers/build-a-custom-connector + """) + (project_dir / "README.md").write_text(content) + + +# --------------------------------------------------------------------------- +# Deployment files +# --------------------------------------------------------------------------- + +def _write_deployment_files(spec: ConnectorSpec, project_dir: Path) -> None: + config_dir = project_dir / "config" + config_dir.mkdir(parents=True, exist_ok=True) + + (project_dir / "Dockerfile").write_text(textwrap.dedent("""\ + FROM python:3.11-slim + ARG ENVIRONMENT + + WORKDIR /usr/src/app + + ENV WORKFLOWS_PROJECT_ROOT=/usr/src/app + + RUN pip install --upgrade pip + + RUN apt-get update && \\ + apt-get install -y git curl && \\ + rm -rf /var/lib/apt/lists/* + + RUN pip install git+https://github.com/stacksyncdata/workflows-cdk.git@prod + + COPY requirements.txt ./ + RUN pip3 install -r requirements.txt + + COPY . . + + EXPOSE 8080 + + RUN chmod +x ./entrypoint.sh + + ENTRYPOINT ["sh", "entrypoint.sh"] + """)) + + (config_dir / "Dockerfile.dev").write_text(textwrap.dedent("""\ + FROM python:3.11-slim + + WORKDIR /usr/src/app + + ENV WORKFLOWS_PROJECT_ROOT=/usr/src/app + + RUN apt-get update && \\ + apt-get install -y git curl && \\ + rm -rf /var/lib/apt/lists/* + + RUN pip install --upgrade pip + + RUN pip install git+https://github.com/stacksyncdata/workflows-cdk.git@prod + + COPY requirements.txt ./ + RUN pip3 install -r requirements.txt + + COPY . . + + RUN chmod +x ./config/entrypoint.sh + + EXPOSE 8080 + + ENTRYPOINT ["sh", "./config/entrypoint.sh"] + """)) + + (config_dir / "entrypoint.sh").write_text( + "cd /usr/src/app\nexec gunicorn --config /usr/src/app/config/gunicorn_config.py main:app\n" + ) + + (project_dir / "entrypoint.sh").write_text( + "cd /usr/src/app\nexec gunicorn --config /usr/src/app/config/gunicorn_config.py main:app\n" + ) + + (config_dir / "gunicorn_config.py").write_text(textwrap.dedent("""\ + bind = "0.0.0.0:8080" + chdir = "/usr/src/app" + accesslog = "-" + errorlog = "-" + capture_output = True + enable_stdio_inheritance = True + + workers = 2 + threads = 1 + timeout = 360 + """)) + + port = DEFAULT_PORT + (project_dir / "run_dev.sh").write_text(textwrap.dedent(f"""\ + #!/bin/bash + + REBUILD=false + for arg in "$@"; do + if [ "$arg" == "--build" ]; then + REBUILD=true + fi + done + + echo "" + echo "Stacksync App Connector" + echo "Documentation: https://docs.stacksync.com/workflows/app-connector" + echo "" + + if [ ! -d "config" ]; then + mkdir -p config + fi + + PORT={port} + DOCKERFILE_PATH="config/Dockerfile.dev" + REPO_NAME=$(basename "$PWD") + APP_NAME="workflows-app-${{REPO_NAME}}" + + echo "Preparing ${{APP_NAME}}..." + + IMAGE_EXISTS=$(docker images -q ${{APP_NAME}} 2> /dev/null) + + if [ -z "$IMAGE_EXISTS" ] || [ "$REBUILD" == "true" ]; then + if [ "$REBUILD" == "true" ]; then + echo "Forcing rebuild of Docker image: ${{APP_NAME}}" + docker build --no-cache -t ${{APP_NAME}} -f ${{DOCKERFILE_PATH}} . + else + echo "Docker image not found. Building: ${{APP_NAME}}" + docker build -t ${{APP_NAME}} -f ${{DOCKERFILE_PATH}} . + fi + else + echo "Docker image ${{APP_NAME}} already exists. Skipping build." + echo "Use --build flag to force a rebuild." + fi + + if [ $? -ne 0 ]; then + echo "Docker build failed. Exiting..." + exit 1 + fi + + echo "Starting container on port ${{PORT}}..." + # Remove any previous container with the same name (avoids "name already in use") + docker rm -f "${{APP_NAME}}" 2>/dev/null || true + docker run --rm -p ${{PORT}}:8080 -it -e ENVIRONMENT=dev -e REGION=besg --name=${{APP_NAME}} -v $PWD:/usr/src/app/ ${{APP_NAME}} + """)) + + (project_dir / "run_dev.bat").write_text(textwrap.dedent(f"""\ + @echo off + setlocal enabledelayedexpansion + + set "REBUILD=false" + for %%a in (%*) do ( + if "%%a"=="--build" set "REBUILD=true" + ) + + echo. + echo Stacksync App Connector + echo Documentation: https://docs.stacksync.com/workflows/app-connector + echo. + + if not exist "config" mkdir config + + set "PORT={port}" + set "DOCKERFILE_PATH=config\\Dockerfile.dev" + + for %%I in ("!CD!") do set "DIRNAME=%%~nxI" + set "APP_NAME=workflows-app-!DIRNAME!" + + echo Preparing !APP_NAME!... + + set "IMAGE_EXISTS=" + for /f %%i in ('docker images -q !APP_NAME! 2^>nul') do set "IMAGE_EXISTS=%%i" + + if "!IMAGE_EXISTS!"=="" ( + echo Docker image not found. Building: !APP_NAME! + docker build -t !APP_NAME! -f !DOCKERFILE_PATH! . + ) else if "!REBUILD!"=="true" ( + echo Forcing rebuild: !APP_NAME! + docker build --no-cache -t !APP_NAME! -f !DOCKERFILE_PATH! . + ) else ( + echo Docker image !APP_NAME! already exists. Use --build to rebuild. + ) + + if errorlevel 1 ( + echo Docker build failed. Exiting... + exit /b 1 + ) + + echo Starting container on port !PORT!... + docker rm -f !APP_NAME! 2>nul + docker run --rm -p !PORT!:8080 -it -e ENVIRONMENT=dev -e REGION=besg --name=!APP_NAME! -v %%CD%%:/usr/src/app/ !APP_NAME! + """)) + + +# --------------------------------------------------------------------------- +# Module config +# --------------------------------------------------------------------------- + +def _write_module_config(name: str, description: str, route_dir: Path) -> None: + """Write module_config.yaml for a single module version.""" + human_name = name.replace("_", " ").title() + config = { + "module_settings": { + "module_name": human_name, + "module_description": description or f"{human_name} Module", + } + } + (route_dir / "module_config.yaml").write_text( + yaml.dump(config, default_flow_style=False, sort_keys=False) + ) + + +# --------------------------------------------------------------------------- +# Module Schema (Stacksync format) +# --------------------------------------------------------------------------- + +_WIDGET_MAP = { + "string": "input", + "number": "input", + "boolean": "checkbox", + "object": "SelectWidget", + "array": "input", +} + +_CONN_MGMT_MAP = { + "oauth2": ["managed"], + "api_key": ["user_managed"], +} + + +def _build_module_schema( + fields: list[FieldSpec], + auth: AuthSpec, + app_type: str, +) -> dict: + """Build a Stacksync Module Schema (v1.0.0) from field specs.""" + schema_fields: list[dict] = [] + + if auth.type in ("oauth2", "api_key"): + mgmt_types = _CONN_MGMT_MAP.get(auth.type, ["managed"]) + conn_field: dict = { + "type": "connection", + "id": "api_connection", + "label": "Connection", + "allowed_app_types": [app_type], + "allowed_connection_management_types": mgmt_types, + "validation": {"required": True}, + } + schema_fields.append(conn_field) + + for f in fields: + widget = f.widget or _WIDGET_MAP.get(f.type, "input") + + field_def: dict = { + "id": f.name, + "type": f.type, + "label": f.name.replace("_", " ").title(), + "description": f.description, + "ui_options": {"ui_widget": widget}, + } + + if f.required: + field_def["validation"] = {"required": True} + + if f.description: + field_def["ui_options"]["helper_text"] = f.description + + if f.choices: + field_def["choices"] = f.choices + + if f.dynamic_content: + field_def.setdefault("on_action", {})["load_content"] = True + if f.depends_on: + field_def["content_object_depends_on_fields"] = [f.depends_on] + + if f.depends_on and not f.dynamic_content: + field_def.setdefault("on_action", {})["load_schema"] = True + + schema_fields.append(field_def) + + field_ids = [sf["id"] for sf in schema_fields] + + return { + "metadata": {"workflows_module_schema_version": "1.0.0"}, + "fields": schema_fields, + "ui_options": {"ui_order": field_ids}, + } + + +# --------------------------------------------------------------------------- +# Route generation +# --------------------------------------------------------------------------- + +def _extract_imports(code: str) -> tuple[list[str], str]: + """Separate import statements from implementation code. + + Returns (import_lines, remaining_code). + """ + import_lines: list[str] = [] + body_lines: list[str] = [] + for line in code.splitlines(): + stripped = line.strip() + if stripped.startswith("import ") or stripped.startswith("from "): + import_lines.append(stripped) + else: + body_lines.append(line) + remaining = "\n".join(body_lines).strip() + return import_lines, remaining + + +def _indent(code: str, level: int = 1) -> str: + """Indent each line of *code* by *level* * 4 spaces.""" + prefix = " " * level + return "\n".join(prefix + line if line.strip() else "" for line in code.splitlines()) + + +def _write_action_route( + spec: ConnectorSpec, + action: ActionSpec, + project_dir: Path, +) -> list[str]: + """Write route.py, schema.json, module_config.yaml and return rationale lines.""" + route_dir = project_dir / MODULES_DIR / action.name / spec.version + route_dir.mkdir(parents=True, exist_ok=True) + + all_fields = list(action.required_fields) + list(action.optional_fields) + want_content = needs_content_endpoint(all_fields) + want_schema = needs_schema_endpoint(all_fields) + + rationale = _build_rationale(action.name, all_fields, want_content, want_schema) + + extra_imports: list[str] = [] + if action.implementation.strip(): + extra_imports, clean_impl = _extract_imports(action.implementation) + execute_body = _indent(clean_impl) + else: + execute_body = _build_stub_body(spec, action.name, all_fields, action.category) + + route_code = _build_route_file( + execute_body, extra_imports, + include_content=want_content, + include_schema=want_schema, + ) + (route_dir / "route.py").write_text(route_code) + + schema = _build_module_schema(all_fields, spec.auth, spec.app_type) + (route_dir / "schema.json").write_text(json.dumps(schema, indent=2) + "\n") + + _write_module_config(action.name, action.description, route_dir) + return rationale + + +def _write_trigger_route( + spec: ConnectorSpec, + trigger: TriggerSpec, + project_dir: Path, +) -> list[str]: + """Write trigger route files and return rationale lines.""" + route_dir = project_dir / MODULES_DIR / trigger.name / spec.version + route_dir.mkdir(parents=True, exist_ok=True) + + fields = list(trigger.payload_fields) + want_content = needs_content_endpoint(fields) + want_schema = needs_schema_endpoint(fields) + + rationale = _build_rationale(trigger.name, fields, want_content, want_schema) + + extra_imports: list[str] = [] + if trigger.implementation.strip(): + extra_imports, clean_impl = _extract_imports(trigger.implementation) + execute_body = _indent(clean_impl) + else: + execute_body = _build_trigger_stub_body(spec, trigger) + + route_code = _build_route_file( + execute_body, extra_imports, + include_content=want_content, + include_schema=want_schema, + ) + (route_dir / "route.py").write_text(route_code) + + schema = _build_module_schema(fields, spec.auth, spec.app_type) + (route_dir / "schema.json").write_text(json.dumps(schema, indent=2) + "\n") + + _write_module_config(trigger.name, trigger.description, route_dir) + return rationale + + +def _build_rationale( + name: str, + fields: list[FieldSpec], + want_content: bool, + want_schema: bool, +) -> list[str]: + lines = [f" Adding /execute for '{name}' (every runnable module needs it)"] + + dynamic_names = [f.name for f in fields if f.dynamic_content] + if want_content: + lines.append( + f" Adding /content (dynamic dropdown data for field" + f"{'s' if len(dynamic_names) > 1 else ''} " + f"'{', '.join(dynamic_names)}')" + ) + else: + lines.append(" Skipping /content (no dynamic fields were requested)") + + dep_names = [f.name for f in fields if f.depends_on] + if want_schema: + lines.append( + f" Adding /schema (schema reload needed when " + f"'{', '.join(dep_names)}' changes)" + ) + else: + lines.append(" Skipping /schema (no schema reload needed)") + + return lines + + +def _build_route_file( + execute_body: str, + extra_imports: list[str] | None = None, + *, + include_content: bool = False, + include_schema: bool = False, +) -> str: + """Assemble a route.py with /execute and conditionally /content and /schema.""" + base_imports = [ + "import json", + "", + "import requests", + "from flask import request as flask_request", + "from workflows_cdk import Request, Response, ManagedError", + "from main import router", + ] + if extra_imports: + already = {line.strip() for line in base_imports if line.strip()} + for imp in extra_imports: + if imp not in already: + base_imports.insert(1, imp) + + import_block = "\n".join(base_imports) + + parts = [ + import_block, + "\n\n", + "@router.route(\"/execute\", methods=[\"POST\"])\n", + "def execute():\n", + " req = Request(flask_request)\n", + " data = req.data\n", + " credentials = req.credentials\n", + "\n", + execute_body + "\n", + ] + + if include_content: + parts.extend([ + "\n\n", + "@router.route(\"/content\", methods=[\"POST\"])\n", + "def content():\n", + " req = Request(flask_request)\n", + " data = req.data\n", + "\n", + " if not data:\n", + ' return Response(data={"message": "Missing request data"}, status_code=400)\n', + "\n", + ' form_data = data.get("form_data", {})\n', + ' content_object_names = data.get("content_object_names", [])\n', + "\n", + " if isinstance(content_object_names, list) and content_object_names and isinstance(content_object_names[0], dict):\n", + ' content_object_names = [obj.get("id") for obj in content_object_names if "id" in obj]\n', + "\n", + " content_objects = []\n", + "\n", + ' return Response(data={"content_objects": content_objects})\n', + ]) + + if include_schema: + parts.extend([ + "\n\n", + "@router.route(\"/schema\", methods=[\"POST\"])\n", + "def schema():\n", + " req = Request(flask_request)\n", + ' schema_path = __file__.replace("route.py", "schema.json")\n', + " with open(schema_path) as f:\n", + " base_schema = json.load(f)\n", + " return Response(data=base_schema)\n", + ]) + + return "".join(parts) + + +def _build_stub_body( + spec: ConnectorSpec, + action_name: str, + fields: list[FieldSpec], + category: str, +) -> str: + """Fallback stub when the LLM didn't generate implementation code.""" + lines: list[str] = [] + for f in fields: + lines.append(f' {f.name} = data.get("{f.name}")') + + lines.append("") + lines.append(f' # TODO: implement {spec.app_name} API call for "{action_name}"') + lines.append("") + + if category == "search": + lines.append(' return Response(data={"results": []})') + else: + lines.append(' return Response(data={"success": True})') + + return "\n".join(lines) + + +def _build_trigger_stub_body(spec: ConnectorSpec, trigger: TriggerSpec) -> str: + lines = [ + f' # TODO: implement polling / webhook logic for "{trigger.name}"', + f" # Event: {trigger.event}", + "", + ' return Response(data={"events": []})', + ] + return "\n".join(lines) diff --git a/src/workflows_cdk/spec/connector_spec.py b/src/workflows_cdk/spec/connector_spec.py new file mode 100644 index 0000000..a54c869 --- /dev/null +++ b/src/workflows_cdk/spec/connector_spec.py @@ -0,0 +1,82 @@ +""" +Intermediate connector specification format. + +The LLM planner produces a ConnectorSpec; the deterministic validator checks it; +the compiler turns it into a scaffolded CDK project on disk. No component +downstream of the planner ever sees raw LLM text -- only this typed model. +""" + +from __future__ import annotations + +from typing import Literal, Optional + +from pydantic import BaseModel, Field + + +class FieldSpec(BaseModel): + name: str + type: Literal["string", "number", "boolean", "object", "array"] = "string" + description: str = "" + required: bool = True + widget: str = "" + choices: list[dict] = Field(default_factory=list) + depends_on: str = "" + dynamic_content: bool = False + + +class AuthSpec(BaseModel): + type: Literal["oauth2", "api_key", "basic", "none"] = "oauth2" + scopes: list[str] = Field(default_factory=list) + fields: list[FieldSpec] = Field(default_factory=list) + + +class ActionSpec(BaseModel): + name: str + category: Literal["action", "trigger", "search", "transform"] = "action" + description: str = "" + required_fields: list[FieldSpec] = Field(default_factory=list) + optional_fields: list[FieldSpec] = Field(default_factory=list) + implementation: str = Field( + default="", + description="Python code for the /execute endpoint body (after data extraction)", + ) + + +class TriggerSpec(BaseModel): + name: str + event: str = "" + description: str = "" + payload_fields: list[FieldSpec] = Field(default_factory=list) + implementation: str = Field( + default="", + description="Python code for the /execute endpoint body (polling/webhook logic)", + ) + + +class AmbiguitySpec(BaseModel): + """A single point the planner could not resolve from the prompt alone.""" + + question: str + options: list[str] = Field(default_factory=list) + default: Optional[str] = None + + +class ConnectorSpec(BaseModel): + """Full specification for a connector to be scaffolded.""" + + app_type: str = Field(description="Slug identifier, e.g. 'slack'") + app_name: str = Field(description="Human-readable name, e.g. 'Slack Connector'") + version: str = "v1" + actions: list[ActionSpec] = Field(default_factory=list) + triggers: list[TriggerSpec] = Field(default_factory=list) + auth: AuthSpec = Field(default_factory=AuthSpec) + confidence: float = Field(default=1.0, ge=0.0, le=1.0) + ambiguities: list[AmbiguitySpec] = Field(default_factory=list) + + @property + def needs_clarification(self) -> bool: + return self.confidence < 0.85 and len(self.ambiguities) > 0 + + @property + def directory_name(self) -> str: + return f"{self.app_type}-connector" diff --git a/src/workflows_cdk/templates/__init__.py b/src/workflows_cdk/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/workflows_cdk/templates/library/crm_sync.yaml b/src/workflows_cdk/templates/library/crm_sync.yaml new file mode 100644 index 0000000..77d28f9 --- /dev/null +++ b/src/workflows_cdk/templates/library/crm_sync.yaml @@ -0,0 +1,54 @@ +app_type: hubspot +app_name: HubSpot Connector +version: v1 + +keywords: + - hubspot + - crm + - contact + - lead + - deal + - sales + - customer + +auth: + type: oauth2 + scopes: + - "crm.objects.contacts.read" + - "crm.objects.contacts.write" + - "crm.objects.deals.read" + - "crm.objects.deals.write" + +actions: + - name: create_contact + category: action + description: "Create a new CRM contact" + required_fields: + - { name: email, type: string, description: "Contact email address" } + optional_fields: + - { name: firstname, type: string, description: "First name" } + - { name: lastname, type: string, description: "Last name" } + + - name: update_contact + category: action + description: "Update an existing contact" + required_fields: + - { name: contact_id, type: string, description: "HubSpot contact ID" } + optional_fields: + - { name: email, type: string, description: "Email address" } + - { name: firstname, type: string, description: "First name" } + - { name: lastname, type: string, description: "Last name" } + + - name: search_contacts + category: search + description: "Search for contacts" + required_fields: + - { name: query, type: string, description: "Search query" } + +triggers: + - name: contact_created + event: contact.created + description: "Fires when a new contact is created" + payload_fields: + - { name: id, type: string } + - { name: email, type: string } diff --git a/src/workflows_cdk/templates/library/data_enrichment.yaml b/src/workflows_cdk/templates/library/data_enrichment.yaml new file mode 100644 index 0000000..81bb594 --- /dev/null +++ b/src/workflows_cdk/templates/library/data_enrichment.yaml @@ -0,0 +1,43 @@ +app_type: openai +app_name: OpenAI Connector +version: v1 + +keywords: + - openai + - ai + - gpt + - enrich + - enrichment + - classify + - summarize + - embedding + - generate + +auth: + type: api_key + scopes: [] + +actions: + - name: chat_completion + category: action + description: "Generate a chat completion using GPT" + required_fields: + - { name: prompt, type: string, description: "User message / prompt" } + optional_fields: + - { name: model, type: string, description: "Model name (default gpt-4o)" } + - { name: system_prompt, type: string, description: "System instruction" } + + - name: classify_text + category: transform + description: "Classify text into categories" + required_fields: + - { name: text, type: string, description: "Text to classify" } + - { name: categories, type: array, description: "List of possible categories" } + + - name: summarize + category: transform + description: "Summarize a block of text" + required_fields: + - { name: text, type: string, description: "Text to summarize" } + +triggers: [] diff --git a/src/workflows_cdk/templates/library/payment_processor.yaml b/src/workflows_cdk/templates/library/payment_processor.yaml new file mode 100644 index 0000000..5db58c0 --- /dev/null +++ b/src/workflows_cdk/templates/library/payment_processor.yaml @@ -0,0 +1,47 @@ +app_type: stripe +app_name: Stripe Connector +version: v1 + +keywords: + - stripe + - payment + - charge + - customer + - billing + - invoice + - subscription + +auth: + type: api_key + scopes: [] + +actions: + - name: create_customer + category: action + description: "Create a new Stripe customer" + required_fields: + - { name: email, type: string, description: "Customer email address" } + optional_fields: + - { name: name, type: string, description: "Customer full name" } + + - name: create_charge + category: action + description: "Create a payment charge" + required_fields: + - { name: amount, type: number, description: "Amount in cents" } + - { name: currency, type: string, description: "Three-letter ISO currency code" } + - { name: customer, type: string, description: "Customer ID" } + + - name: list_customers + category: search + description: "List Stripe customers" + required_fields: [] + +triggers: + - name: payment_succeeded + event: payment_intent.succeeded + description: "Fires when a payment succeeds" + payload_fields: + - { name: id, type: string } + - { name: amount, type: number } + - { name: currency, type: string } diff --git a/src/workflows_cdk/templates/library/slack_notifier.yaml b/src/workflows_cdk/templates/library/slack_notifier.yaml new file mode 100644 index 0000000..7ddd279 --- /dev/null +++ b/src/workflows_cdk/templates/library/slack_notifier.yaml @@ -0,0 +1,40 @@ +app_type: slack +app_name: Slack Connector +version: v1 + +keywords: + - slack + - message + - channel + - notify + - notification + - chat + - alert + +auth: + type: oauth2 + scopes: + - "chat:write" + - "channels:read" + +actions: + - name: send_message + category: action + description: "Send a message to a Slack channel or DM" + required_fields: + - { name: channel, type: string, description: "Channel name or ID" } + - { name: text, type: string, description: "Message body" } + + - name: list_channels + category: search + description: "List all channels in the workspace" + required_fields: [] + +triggers: + - name: new_message + event: message.created + description: "Fires when a new message is posted" + payload_fields: + - { name: channel, type: string } + - { name: text, type: string } + - { name: user, type: string } diff --git a/src/workflows_cdk/templates/matcher.py b/src/workflows_cdk/templates/matcher.py new file mode 100644 index 0000000..7e0f83c --- /dev/null +++ b/src/workflows_cdk/templates/matcher.py @@ -0,0 +1,135 @@ +""" +Template matcher -- resolves a user description to a pre-built ConnectorSpec +without calling any LLM. Used as the ``--no-ai`` fast path and as a fallback +when the OpenAI key is not configured. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Optional + +import yaml + +from ..spec.connector_spec import ( + ActionSpec, + AuthSpec, + ConnectorSpec, + FieldSpec, + TriggerSpec, +) + +logger = logging.getLogger(__name__) + +_LIBRARY_DIR = Path(__file__).parent / "library" + + +class _Template: + """Internal wrapper around a single template YAML file.""" + + def __init__(self, data: dict, path: Path) -> None: + self.data = data + self.path = path + self.keywords: list[str] = [ + k.lower() for k in data.get("keywords", []) + ] + self.slug: str = data.get("app_type", path.stem) + + def score(self, query_tokens: set[str]) -> int: + corpus = set(self.keywords) | {self.slug} + return len(query_tokens & corpus) + + def to_connector_spec(self) -> ConnectorSpec: + d = self.data + auth_raw = d.get("auth", {}) + return ConnectorSpec( + app_type=d["app_type"], + app_name=d["app_name"], + version=d.get("version", "v1"), + actions=[_parse_action(a) for a in d.get("actions", [])], + triggers=[_parse_trigger(t) for t in d.get("triggers", [])], + auth=AuthSpec( + type=auth_raw.get("type", "oauth2"), + scopes=auth_raw.get("scopes", []), + ), + confidence=1.0, + ) + + +def _parse_field(raw: dict) -> FieldSpec: + return FieldSpec( + name=raw["name"], + type=raw.get("type", "string"), + description=raw.get("description", ""), + ) + + +def _parse_action(raw: dict) -> ActionSpec: + return ActionSpec( + name=raw["name"], + category=raw.get("category", "action"), + description=raw.get("description", ""), + required_fields=[_parse_field(f) for f in raw.get("required_fields", [])], + optional_fields=[_parse_field(f) for f in raw.get("optional_fields", [])], + ) + + +def _parse_trigger(raw: dict) -> TriggerSpec: + return TriggerSpec( + name=raw["name"], + event=raw.get("event", ""), + description=raw.get("description", ""), + payload_fields=[_parse_field(f) for f in raw.get("payload_fields", [])], + ) + + +def _load_templates() -> list[_Template]: + templates: list[_Template] = [] + if not _LIBRARY_DIR.is_dir(): + return templates + for path in sorted(_LIBRARY_DIR.glob("*.yaml")): + try: + with open(path) as fh: + data = yaml.safe_load(fh) or {} + templates.append(_Template(data, path)) + except Exception: + logger.warning("Failed to load template %s", path, exc_info=True) + return templates + + +class MatchResult: + """Wraps a template match with explanation metadata.""" + + def __init__(self, spec: ConnectorSpec, template_name: str, matched_keywords: list[str]) -> None: + self.spec = spec + self.template_name = template_name + self.matched_keywords = matched_keywords + + +def match_template(description: str) -> Optional[MatchResult]: + """Return the best-matching template as a ``MatchResult``, or *None*.""" + tokens = set(description.lower().split()) + templates = _load_templates() + if not templates: + return None + + best: Optional[_Template] = None + best_score = 0 + best_overlap: list[str] = [] + for tpl in templates: + s = tpl.score(tokens) + if s > best_score: + best_score = s + best = tpl + corpus = set(tpl.keywords) | {tpl.slug} + best_overlap = sorted(tokens & corpus) + + if best is None or best_score == 0: + return None + + return MatchResult( + spec=best.to_connector_spec(), + template_name=best.slug, + matched_keywords=best_overlap, + )