From 2bc01a4dc2732bbbfa335ac1f5c3380eb559f05c Mon Sep 17 00:00:00 2001 From: Jason Benichou Date: Fri, 3 Apr 2026 18:50:12 +0200 Subject: [PATCH 1/2] Add AI-powered connector generation via CLI Replace the manual connector creation workflow with a single-command AI-powered approach: `workflows create ""` generates a complete, deployable connector project in ~15 seconds. New modules: - ai/: LLM planner (Anthropic/OpenAI), intent parser, deterministic validator, clarification engine, and structured prompts - cli/: `workflows` CLI with create, setup, list, inspect commands and interactive first-time API key configuration - registry/: capability manifests (Slack, HubSpot, Stripe, Salesforce, OpenAI, PostgreSQL) used for LLM context and validation - spec/: ConnectorSpec Pydantic model and compiler that generates Stacksync-compatible projects (src/modules/ structure, Module Schema format, /execute + /content + /schema endpoints, deployment files) - templates/: pre-built specs for --no-ai fallback mode Generated connectors include real API call implementations, proper error handling, Dockerfiles, gunicorn config, and all deployment files matching the Stacksync connector template structure. Also syncs requirements.txt with setup.py. Made-with: Cursor --- .env.example | 17 + .gitignore | 5 +- README.md | 76 +++ requirements.txt | 9 +- setup.py | 20 +- src/workflows_cdk/ai/__init__.py | 0 src/workflows_cdk/ai/clarifier.py | 103 +++ src/workflows_cdk/ai/intent_parser.py | 52 ++ src/workflows_cdk/ai/planner.py | 306 +++++++++ src/workflows_cdk/ai/prompts.py | 187 ++++++ src/workflows_cdk/ai/validator.py | 133 ++++ src/workflows_cdk/cli/__init__.py | 0 src/workflows_cdk/cli/main.py | 316 ++++++++++ src/workflows_cdk/registry/__init__.py | 0 .../registry/capabilities/hubspot.yaml | 76 +++ .../registry/capabilities/openai.yaml | 51 ++ .../registry/capabilities/postgres.yaml | 60 ++ .../registry/capabilities/salesforce.yaml | 70 +++ .../registry/capabilities/slack.yaml | 66 ++ .../registry/capabilities/stripe.yaml | 62 ++ src/workflows_cdk/registry/manifest.py | 87 +++ src/workflows_cdk/registry/registry.py | 77 +++ src/workflows_cdk/spec/__init__.py | 0 src/workflows_cdk/spec/compiler.py | 588 ++++++++++++++++++ src/workflows_cdk/spec/connector_spec.py | 78 +++ src/workflows_cdk/templates/__init__.py | 0 .../templates/library/crm_sync.yaml | 54 ++ .../templates/library/data_enrichment.yaml | 43 ++ .../templates/library/payment_processor.yaml | 47 ++ .../templates/library/slack_notifier.yaml | 40 ++ src/workflows_cdk/templates/matcher.py | 119 ++++ 31 files changed, 2735 insertions(+), 7 deletions(-) create mode 100644 .env.example create mode 100644 src/workflows_cdk/ai/__init__.py create mode 100644 src/workflows_cdk/ai/clarifier.py create mode 100644 src/workflows_cdk/ai/intent_parser.py create mode 100644 src/workflows_cdk/ai/planner.py create mode 100644 src/workflows_cdk/ai/prompts.py create mode 100644 src/workflows_cdk/ai/validator.py create mode 100644 src/workflows_cdk/cli/__init__.py create mode 100644 src/workflows_cdk/cli/main.py create mode 100644 src/workflows_cdk/registry/__init__.py create mode 100644 src/workflows_cdk/registry/capabilities/hubspot.yaml create mode 100644 src/workflows_cdk/registry/capabilities/openai.yaml create mode 100644 src/workflows_cdk/registry/capabilities/postgres.yaml create mode 100644 src/workflows_cdk/registry/capabilities/salesforce.yaml create mode 100644 src/workflows_cdk/registry/capabilities/slack.yaml create mode 100644 src/workflows_cdk/registry/capabilities/stripe.yaml create mode 100644 src/workflows_cdk/registry/manifest.py create mode 100644 src/workflows_cdk/registry/registry.py create mode 100644 src/workflows_cdk/spec/__init__.py create mode 100644 src/workflows_cdk/spec/compiler.py create mode 100644 src/workflows_cdk/spec/connector_spec.py create mode 100644 src/workflows_cdk/templates/__init__.py create mode 100644 src/workflows_cdk/templates/library/crm_sync.yaml create mode 100644 src/workflows_cdk/templates/library/data_enrichment.yaml create mode 100644 src/workflows_cdk/templates/library/payment_processor.yaml create mode 100644 src/workflows_cdk/templates/library/slack_notifier.yaml create mode 100644 src/workflows_cdk/templates/matcher.py diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..9afd1fb --- /dev/null +++ b/.env.example @@ -0,0 +1,17 @@ +# ─── AI Provider (choose one) ─────────────────────────────────────────────── +# Set ONE of these keys. If both are set, Anthropic is used by default. + +ANTHROPIC_API_KEY=sk-ant-... +# OPENAI_API_KEY=sk-... + +# ─── Provider selection (optional) ────────────────────────────────────────── +# Force a specific provider when both keys are present: "anthropic" or "openai" +# WORKFLOWS_AI_PROVIDER=anthropic + +# ─── Model override (optional) ────────────────────────────────────────────── +# Default: claude-sonnet-4-6 (Anthropic) or gpt-5-nano (OpenAI) +# WORKFLOWS_AI_MODEL=claude-sonnet-4-6 + +# ─── Runtime ──────────────────────────────────────────────────────────────── +ENVIRONMENT=dev +# SENTRY_DSN= diff --git a/.gitignore b/.gitignore index ad509d6..c0c89fb 100644 --- a/.gitignore +++ b/.gitignore @@ -129,4 +129,7 @@ dmypy.json .pyre/ # .vscode -.vscode/ \ No newline at end of file +.vscode/ + +# Generated connector outputs (test artifacts) +*-connector/ \ No newline at end of file diff --git a/README.md b/README.md index 5567f02..947f249 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,8 @@ A powerful CDK (Connector Development Kit) for building Stacksync Workflows Conn - 📦 Standardized request/response handling - 🛠️ Error management with standardized error handling - 🔄 Environment-aware configuration +- 🤖 AI-powered connector scaffolding via CLI (OpenAI or Anthropic) +- 📚 Built-in capability registry for Slack, Stripe, HubSpot, Salesforce, OpenAI, PostgreSQL ## Installation @@ -16,8 +18,82 @@ A powerful CDK (Connector Development Kit) for building Stacksync Workflows Conn pip install workflows-cdk ``` +## Configuration + +On first run, the CLI will ask you to pick a provider and paste your API key: + +``` +$ workflows create "Slack connector" + +╭─ Welcome to Workflows CDK ──────────────────────────╮ +│ No API key found. Let's set one up. │ +│ You can reconfigure anytime with workflows setup. │ +╰──────────────────────────────────────────────────────╯ + +Which AI provider? [anthropic/openai] (anthropic): +Paste your ANTHROPIC_API_KEY (sk-ant-...): sk-ant-xxxxx + +Done! ANTHROPIC_API_KEY saved to .env +``` + +You can also configure manually: + +```bash +# Interactive setup / reconfigure +workflows setup + +# Or set keys directly via .env +echo 'ANTHROPIC_API_KEY=sk-ant-your-key-here' > .env + +# Or export in your shell +export ANTHROPIC_API_KEY=sk-ant-your-key-here +``` + +Both `.env` and environment variables work. Environment variables take priority. + +| Variable | Required | Description | +|---|---|---| +| `ANTHROPIC_API_KEY` | One of these | Anthropic API key — uses Claude Sonnet 4.6 by default | +| `OPENAI_API_KEY` | One of these | OpenAI API key — uses GPT-5 Nano by default | +| `WORKFLOWS_AI_PROVIDER` | No | Force a provider when both keys are set: `anthropic` or `openai` | +| `WORKFLOWS_AI_MODEL` | No | Override the default model (default: `claude-sonnet-4-6` or `gpt-5-nano`) | +| `ENVIRONMENT` | No | Runtime environment: `dev`, `stage`, or `prod` | + +No API key? Use `--no-ai` to create connectors via built-in template matching instead. + ## Quick Start +### AI-powered (recommended) + +Create a connector in one command: + +```bash +workflows create "Slack connector: send messages, list channels" +``` + +A complete project appears in `./slack-connector/` with working Flask routes, schemas, and config -- ready to run with `python main.py`. + +More examples: + +```bash +# Preview without writing files +workflows create "HubSpot CRM: create contacts, search, manage deals" --dry-run + +# Template matching only (no API key needed) +workflows create "Slack: send messages, list channels" --no-ai + +# Specify output directory +workflows create "PostgreSQL: query, insert, search rows" -o ./connectors + +# List available capabilities +workflows list + +# Inspect a specific app +workflows inspect slack +``` + +### Manual setup + 1. Create a new project directory: ```bash diff --git a/requirements.txt b/requirements.txt index 51a7c35..29332d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,13 @@ flask -werkzeug +werkzeug==2.2 pyopenssl==24.1.0 flask-cors>=4.0.0 python-dotenv>=1.0.0 gunicorn==22.0.0 -authlib==1.1.0 sentry-sdk[Flask] pydantic>=2.0.0 -pyyaml>=6.0.0 \ No newline at end of file +pyyaml>=6.0.0 +click>=8.0.0 +rich>=13.0.0 +openai>=1.0.0 +anthropic>=0.30.0 diff --git a/setup.py b/setup.py index 6a61628..021f7ef 100644 --- a/setup.py +++ b/setup.py @@ -7,12 +7,11 @@ setup( name="workflows_cdk", - version="0.1.0", + version="0.2.0", description="A CDK for developing Stacksync Workflows Connectors", author="Stacksync", author_email="oliviero@stacksync.com", install_requires=[ - # Core dependencies "flask", "werkzeug==2.2", "pyopenssl==24.1.0", @@ -21,10 +20,25 @@ "gunicorn==22.0.0", "sentry-sdk[Flask]", "pydantic>=2.0.0", - "pyyaml>=6.0.0" + "pyyaml>=6.0.0", + "click>=8.0.0", + "rich>=13.0.0", + "openai>=1.0.0", + "anthropic>=0.30.0", ], + entry_points={ + "console_scripts": [ + "workflows=workflows_cdk.cli.main:cli", + ], + }, python_requires=">=3.10", packages=find_packages(where="src"), package_dir={"": "src"}, include_package_data=True, + package_data={ + "workflows_cdk": [ + "registry/capabilities/*.yaml", + "templates/library/*.yaml", + ], + }, ) diff --git a/src/workflows_cdk/ai/__init__.py b/src/workflows_cdk/ai/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/workflows_cdk/ai/clarifier.py b/src/workflows_cdk/ai/clarifier.py new file mode 100644 index 0000000..d44374d --- /dev/null +++ b/src/workflows_cdk/ai/clarifier.py @@ -0,0 +1,103 @@ +""" +One-shot clarification engine. + +Takes the ``ambiguities`` list from a ConnectorSpec, renders them as a single +compact terminal prompt, collects the user's answers, and returns a plain-text +string that can be fed back into the LLM refinement prompt. +""" + +from __future__ import annotations + +from rich.console import Console +from rich.panel import Panel +from rich.text import Text + +from ..spec.connector_spec import AmbiguitySpec, ConnectorSpec + +console = Console() + + +def render_clarification(spec: ConnectorSpec) -> str: + """Display ambiguities and collect answers interactively. + + Returns a plain-text summary of the user's choices, ready to be passed + to ``Planner.refine()``. + """ + ambiguities = spec.ambiguities + if not ambiguities: + return "" + + action_count = len(spec.actions) + trigger_count = len(spec.triggers) + + parts = [] + if action_count: + parts.append(f"{action_count} action{'s' if action_count != 1 else ''}") + if trigger_count: + parts.append(f"{trigger_count} trigger{'s' if trigger_count != 1 else ''}") + summary = " + ".join(parts) if parts else "a connector" + + header = ( + f"I can build this {spec.app_name} with {summary}.\n" + f"I need {len(ambiguities)} detail{'s' if len(ambiguities) != 1 else ''} " + f"before scaffolding:" + ) + + body_lines: list[str] = [] + for idx, amb in enumerate(ambiguities, 1): + options_str = _format_options(amb) + body_lines.append(f" {idx}. {amb.question}: {options_str}") + + body = "\n".join(body_lines) + + console.print() + console.print(Panel( + f"{header}\n\n{body}", + title="[bold]Clarification needed[/bold]", + border_style="yellow", + )) + + console.print( + "\n[dim]Press Enter to accept defaults (shown in brackets).[/dim]" + ) + raw = console.input("[bold]Your answers:[/bold] ").strip() + + if not raw: + return _defaults_summary(ambiguities) + + return _merge_answers(ambiguities, raw) + + +def _format_options(amb: AmbiguitySpec) -> str: + if not amb.options: + return f"[{amb.default or '?'}]" + + parts: list[str] = [] + for opt in amb.options: + if opt == amb.default: + parts.append(f"[{opt}]") + else: + parts.append(opt) + return " / ".join(parts) + + +def _defaults_summary(ambiguities: list[AmbiguitySpec]) -> str: + lines: list[str] = [] + for amb in ambiguities: + default = amb.default or (amb.options[0] if amb.options else "unspecified") + lines.append(f"{amb.question}: {default}") + return "\n".join(lines) + + +def _merge_answers(ambiguities: list[AmbiguitySpec], raw: str) -> str: + """Best-effort parse of comma-separated or numbered answers.""" + tokens = [t.strip() for t in raw.replace(";", ",").split(",")] + + lines: list[str] = [] + for idx, amb in enumerate(ambiguities): + if idx < len(tokens) and tokens[idx]: + lines.append(f"{amb.question}: {tokens[idx]}") + else: + default = amb.default or (amb.options[0] if amb.options else "unspecified") + lines.append(f"{amb.question}: {default}") + return "\n".join(lines) diff --git a/src/workflows_cdk/ai/intent_parser.py b/src/workflows_cdk/ai/intent_parser.py new file mode 100644 index 0000000..6871450 --- /dev/null +++ b/src/workflows_cdk/ai/intent_parser.py @@ -0,0 +1,52 @@ +""" +Lightweight pre-processing of the user prompt *before* calling the LLM. + +Extracts candidate app slugs and action verbs so the planner can narrow +which capability manifests to inject into the system prompt context window. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from ..registry.registry import CapabilityRegistry + +_TRIGGER_VERBS = frozenset({ + "when", "on", "listen", "react", "trigger", "watch", "monitor", +}) + +_SEARCH_VERBS = frozenset({ + "list", "search", "find", "get", "fetch", "query", "lookup", +}) + + +@dataclass +class ParsedIntent: + raw: str + detected_slugs: list[str] = field(default_factory=list) + has_trigger_intent: bool = False + has_search_intent: bool = False + tokens: list[str] = field(default_factory=list) + + +def parse_intent(description: str, registry: CapabilityRegistry) -> ParsedIntent: + """Extract structured signals from the raw user prompt.""" + tokens = description.lower().split() + token_set = set(tokens) + + detected_slugs: list[str] = [] + for slug in registry.slugs(): + manifest = registry.get(slug) + if manifest is None: + continue + name_lower = manifest.app.name.lower() + if slug in token_set or name_lower in description.lower(): + detected_slugs.append(slug) + + return ParsedIntent( + raw=description, + detected_slugs=detected_slugs, + has_trigger_intent=bool(token_set & _TRIGGER_VERBS), + has_search_intent=bool(token_set & _SEARCH_VERBS), + tokens=tokens, + ) diff --git a/src/workflows_cdk/ai/planner.py b/src/workflows_cdk/ai/planner.py new file mode 100644 index 0000000..9548af0 --- /dev/null +++ b/src/workflows_cdk/ai/planner.py @@ -0,0 +1,306 @@ +""" +Planner orchestrator — the single entry point for AI-powered connector creation. + +Flow: + 1. Parse intent from user description (extract slugs, verbs) + 2. Build the LLM prompt with relevant capability manifests + 3. Call the configured LLM provider with structured output + 4. Return a validated ConnectorSpec (the CLI handles clarification / compilation) + +Provider selection (Anthropic preferred): + - Set ``ANTHROPIC_API_KEY`` to use Claude (default model: claude-sonnet-4-6) + - Set ``OPENAI_API_KEY`` to use OpenAI (default model: gpt-5-nano) + - If both are set, Anthropic wins unless ``WORKFLOWS_AI_PROVIDER=openai`` + +Structured output: + - OpenAI → Responses API with ``text.format`` json_schema (strict) + - Anthropic → ``output_config.format`` json_schema (SDK auto-transforms) + Both guarantee schema-compliant JSON — no post-hoc parsing needed. +""" + +from __future__ import annotations + +import json +import logging +import os +from typing import Any, Literal + +from ..registry.registry import CapabilityRegistry +from ..spec.connector_spec import ConnectorSpec +from .intent_parser import parse_intent +from .prompts import PLANNER_SYSTEM_PROMPT, REFINEMENT_PROMPT + +logger = logging.getLogger(__name__) + +Provider = Literal["openai", "anthropic"] + +DEFAULT_OPENAI_MODEL = "gpt-5-nano" +DEFAULT_ANTHROPIC_MODEL = "claude-sonnet-4-6" + + +class PlannerError(Exception): + pass + + +# --------------------------------------------------------------------------- +# Provider detection (Anthropic default) +# --------------------------------------------------------------------------- + +def _detect_provider() -> Provider: + explicit = os.environ.get("WORKFLOWS_AI_PROVIDER", "").lower() + if explicit in ("openai", "anthropic"): + return explicit # type: ignore[return-value] + + if os.environ.get("ANTHROPIC_API_KEY"): + return "anthropic" + if os.environ.get("OPENAI_API_KEY"): + return "openai" + + raise PlannerError( + "No AI provider configured. " + "Set ANTHROPIC_API_KEY or OPENAI_API_KEY, " + "or pass --no-ai to use template matching instead." + ) + + +# --------------------------------------------------------------------------- +# JSON Schema from Pydantic +# --------------------------------------------------------------------------- + +def _build_json_schema() -> dict[str, Any]: + """Generate a JSON Schema from ConnectorSpec with ``additionalProperties: + false`` on all objects and all properties promoted to ``required``. + + Both OpenAI and Anthropic require this for their structured-output modes. + """ + schema = ConnectorSpec.model_json_schema() + _prepare_strict(schema) + return schema + + +_PYDANTIC_ONLY_KEYWORDS = { + "minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum", + "minLength", "maxLength", "pattern", "minItems", "maxItems", + "uniqueItems", +} + + +def _prepare_strict(node: dict[str, Any]) -> None: + """Recursively add ``additionalProperties: false``, promote all properties + to ``required``, and strip Pydantic validation keywords that neither + provider supports in structured-output schemas. The constraints are still + enforced by ``ConnectorSpec.model_validate()`` after parsing.""" + for kw in _PYDANTIC_ONLY_KEYWORDS: + node.pop(kw, None) + + if node.get("type") == "object": + node["additionalProperties"] = False + if "properties" in node: + node.setdefault("required", list(node["properties"].keys())) + + for key in ("properties", "$defs"): + container = node.get(key) + if isinstance(container, dict): + for v in container.values(): + if isinstance(v, dict): + _prepare_strict(v) + + items = node.get("items") + if isinstance(items, dict): + _prepare_strict(items) + + for combo_key in ("anyOf", "oneOf", "allOf"): + combo = node.get(combo_key) + if isinstance(combo, list): + for item in combo: + if isinstance(item, dict): + _prepare_strict(item) + + +# --------------------------------------------------------------------------- +# Planner +# --------------------------------------------------------------------------- + +class Planner: + """Stateless orchestrator: description in, ConnectorSpec out.""" + + def __init__(self, registry: CapabilityRegistry) -> None: + self.registry = registry + + def plan(self, description: str) -> ConnectorSpec: + intent = parse_intent(description, self.registry) + + if intent.detected_slugs: + summaries = [ + self.registry.get(slug).summary_for_llm() + for slug in intent.detected_slugs + if self.registry.get(slug) is not None + ] + else: + summaries = self.registry.summaries_for_llm() + + capabilities_json = json.dumps(summaries or self.registry.summaries_for_llm(), indent=2) + system = PLANNER_SYSTEM_PROMPT.format( + capabilities_json=capabilities_json, + ) + + raw = _call_llm(system=system, user=description) + return _parse_spec(raw) + + def refine(self, draft: ConnectorSpec, user_answers: str) -> ConnectorSpec: + system = REFINEMENT_PROMPT.format( + draft_spec_json=draft.model_dump_json(indent=2), + user_answers=user_answers, + ) + raw = _call_llm(system=system, user=user_answers) + return _parse_spec(raw) + + +# --------------------------------------------------------------------------- +# LLM dispatch +# --------------------------------------------------------------------------- + +def _call_llm(*, system: str, user: str) -> str: + provider = _detect_provider() + if provider == "anthropic": + return _call_anthropic(system=system, user=user) + return _call_openai(system=system, user=user) + + +# --------------------------------------------------------------------------- +# Anthropic / Claude — output_config.format json_schema +# --------------------------------------------------------------------------- + +def _call_anthropic(*, system: str, user: str) -> str: + try: + import anthropic + except ImportError: + raise PlannerError( + "The 'anthropic' package is required for Claude. " + "Install it with: pip install workflows-cdk" + ) + + api_key = os.environ.get("ANTHROPIC_API_KEY") + if not api_key: + raise PlannerError("ANTHROPIC_API_KEY environment variable is not set.") + + client = anthropic.Anthropic(api_key=api_key) + model = os.environ.get("WORKFLOWS_AI_MODEL", DEFAULT_ANTHROPIC_MODEL) + schema = _build_json_schema() + + resp = client.messages.create( + model=model, + max_tokens=4096, + system=system, + messages=[{"role": "user", "content": user}], + output_config={ + "format": { + "type": "json_schema", + "schema": schema, + }, + }, + temperature=0.2, + ) + + text_blocks = [b.text for b in resp.content if b.type == "text"] + if not text_blocks: + raise PlannerError("Claude returned empty response") + return text_blocks[0] + + +# --------------------------------------------------------------------------- +# OpenAI — Responses API with json_schema structured output +# --------------------------------------------------------------------------- + +def _call_openai(*, system: str, user: str) -> str: + try: + import openai + except ImportError: + raise PlannerError( + "The 'openai' package is required for AI planning. " + "Install it with: pip install workflows-cdk" + ) + + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + raise PlannerError("OPENAI_API_KEY environment variable is not set.") + + client = openai.OpenAI(api_key=api_key) + model = os.environ.get("WORKFLOWS_AI_MODEL", DEFAULT_OPENAI_MODEL) + schema = _build_json_schema() + + try: + resp = client.responses.create( + model=model, + instructions=system, + input=user, + text={ + "format": { + "type": "json_schema", + "json_schema": { + "name": "connector_spec", + "schema": schema, + "strict": True, + }, + }, + }, + temperature=0.2, + ) + return resp.output_text + except Exception: + logger.debug("Responses API unavailable, falling back to chat completions") + + return _chat_completions_fallback(client, model, system, user, schema) + + +def _chat_completions_fallback( + client: "openai.OpenAI", + model: str, + system: str, + user: str, + schema: dict[str, Any], +) -> str: + resp = client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "connector_spec", + "schema": schema, + "strict": True, + }, + }, + temperature=0.2, + ) + content = resp.choices[0].message.content + if content is None: + raise PlannerError("LLM returned empty response") + return content + + +# --------------------------------------------------------------------------- +# Spec parsing +# --------------------------------------------------------------------------- + +def _parse_spec(raw: str) -> ConnectorSpec: + """Parse LLM output into a ConnectorSpec. + + With structured output the JSON is already valid, but we keep the + fence-stripping as a safety net for fallback paths. + """ + text = raw.strip() + if text.startswith("```"): + lines = text.split("\n") + lines = [l for l in lines if not l.strip().startswith("```")] + text = "\n".join(lines) + + try: + data = json.loads(text) + except json.JSONDecodeError as exc: + raise PlannerError(f"LLM returned invalid JSON: {exc}\n\nRaw output:\n{raw}") + + return ConnectorSpec.model_validate(data) diff --git a/src/workflows_cdk/ai/prompts.py b/src/workflows_cdk/ai/prompts.py new file mode 100644 index 0000000..58b7133 --- /dev/null +++ b/src/workflows_cdk/ai/prompts.py @@ -0,0 +1,187 @@ +""" +LLM system prompts for the connector planner. + +Design follows the GPT-5 prompting guide: + - Structured XML sections for instruction adherence + - Explicit output contract with JSON schema + - No contradictory instructions + - Single-turn persistence directive + - Clarification rules capped at 1 round / 3 sub-questions +""" + +PLANNER_SYSTEM_PROMPT = """\ + +You are the Stacksync connector planner. You receive a short natural-language +description (under 30 words) of a desired connector and produce a structured +ConnectorSpec JSON that will be compiled into a runnable project. + +You MUST generate working Python implementation code for each action and trigger. +The implementation uses the ``requests`` library to make real API calls. + + + +{capabilities_json} + + + +Return ONLY valid JSON (no markdown fences, no commentary) matching this schema: + +{{ + "app_type": "string — slug, e.g. slack", + "app_name": "string — human-readable, e.g. Slack Connector", + "version": "string — default v1", + "actions": [ + {{ + "name": "string — snake_case action id", + "category": "action | search | transform", + "description": "string — one-sentence description", + "required_fields": [{{ "name": "string", "type": "string|number|boolean|object|array", "description": "string" }}], + "optional_fields": [{{ "name": "string", "type": "string|number|boolean|object|array", "description": "string" }}], + "implementation": "string — Python code (see )" + }} + ], + "triggers": [ + {{ + "name": "string — snake_case trigger id", + "event": "string — dot-notation event name", + "description": "string", + "payload_fields": [{{ "name": "string", "type": "string|number|boolean|object|array", "description": "string" }}], + "implementation": "string — Python code (see )" + }} + ], + "auth": {{ + "type": "oauth2 | api_key | basic | none", + "scopes": ["string"], + "fields": [] + }}, + "confidence": "float 0.0–1.0", + "ambiguities": [ + {{ + "question": "string", + "options": ["string"], + "default": "string | null" + }} + ] +}} + + + +Fields (required_fields, optional_fields, payload_fields) define the UI form that +the user fills in the Stacksync workflow builder. They are CONFIGURATION inputs, +not the data that comes out of an API. + +For actions: fields = what the user must provide to run the action. + Example "send_slack_message" action: + required_fields: channel (which Slack channel), text (message text) + optional_fields: username (display name override) + +For triggers: payload_fields = what the user configures for the trigger. + Example "new_hubspot_contact" trigger: + payload_fields: slack_channel (where to send), message_format (what to include) + NOT: firstname, lastname, email (those come from the API at runtime) + +CRITICAL ALIGNMENT RULE: Every field name used via data.get("field_name") in +the implementation code MUST exist in required_fields, optional_fields, or +payload_fields. If the implementation reads a field, add it to the field list. +Conversely, every field in the list should be used in the implementation. + + + +The "implementation" field is a string of Python code that forms the BODY of the +/execute endpoint function. The following variables are already available: + + - ``data`` — dict of user-submitted form fields (from the schema above) + - ``credentials`` — dict with auth tokens (e.g. credentials.get("access_token")) + - ``requests`` — the requests library (already imported at file top) + - ``Response`` — the CDK Response class (use Response(data=...) or Response.error) + - ``ManagedError`` — the CDK error class + +Your implementation MUST: + 1. Extract fields from ``data`` using data.get("field_name") + 2. Build headers using credentials (e.g. Bearer token or API key) + 3. Make the real HTTP call using requests.get / requests.post / etc. + 4. Return Response(data=result) on success or raise ManagedError on failure + 5. Use proper indentation (4 spaces per level, starting at zero indent) + 6. Handle errors with try/except and raise ManagedError(message) + 7. NEVER include ``import`` statements — requests, json, time are already + available. NEVER include function def or decorators. + +Example — a Slack send_message action implementation: + + channel = data.get("channel") + text = data.get("text") + + headers = {{"Authorization": f"Bearer {{credentials.get('access_token')}}", "Content-Type": "application/json"}} + payload = {{"channel": channel, "text": text}} + + try: + resp = requests.post("https://slack.com/api/chat.postMessage", headers=headers, json=payload) + result = resp.json() + if not result.get("ok"): + raise ManagedError(result.get("error", "Unknown Slack error")) + except ManagedError: + raise + except Exception as e: + raise ManagedError(f"Slack API error: {{str(e)}}") + + return Response(data=result) + +The above would be stored as a single string with \\n for newlines. + + + +- If confidence >= 0.85: return the spec with ambiguities=[]. Done. +- If confidence < 0.85: return the spec AND populate ambiguities with at most + 3 items. Each item has: question, options, default. +- NEVER produce more than 1 clarification round. +- NEVER ask about things you can infer from context or from the capability + manifest defaults. +- If the user mentions an app that is NOT in , set + confidence to 0.5, include all actions you can reasonably infer, and add one + ambiguity noting the app is not in the built-in registry. + + + +- Map actions ONLY to capabilities listed in when the + app is known. +- Default auth type to what the capability manifest specifies. +- Default action category to "action" unless the description contains "list", + "search", "find" (use "search") or "when", "on", "trigger", "listen", + "react" (use trigger). +- Triggers go in the "triggers" array, NOT in "actions". +- Every required_field must have name, type, and description. +- Do NOT invent scopes or fields that are not in the manifest. +- Every action and trigger MUST have a non-empty "implementation" string. + + + +Complete the full spec in a single JSON response. Do not produce partial +results. Do not add text outside the JSON object. + +""" + +REFINEMENT_PROMPT = """\ + +You are the Stacksync connector planner. The user has answered your +clarification questions. Merge their answers into the draft spec and return +the final ConnectorSpec JSON. + + + +{draft_spec_json} + + + +{user_answers} + + + +Return ONLY the final valid JSON ConnectorSpec. Set confidence to 1.0 and +ambiguities to []. Follow the same schema as before. Every action and trigger +MUST have a non-empty "implementation" string with real Python code. + + + +Produce the complete spec in a single response. No commentary, no markdown. + +""" diff --git a/src/workflows_cdk/ai/validator.py b/src/workflows_cdk/ai/validator.py new file mode 100644 index 0000000..34f727b --- /dev/null +++ b/src/workflows_cdk/ai/validator.py @@ -0,0 +1,133 @@ +""" +Deterministic post-LLM validator. + +Runs *after* the planner produces a ConnectorSpec and checks it against the +capability registry. Returns blocking errors and non-blocking warnings so +the CLI can decide whether to proceed. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +from ..registry.registry import CapabilityRegistry +from ..spec.connector_spec import ConnectorSpec + +_VALID_FIELD_TYPES = frozenset({"string", "number", "boolean", "object", "array"}) +_VALID_AUTH_TYPES = frozenset({"oauth2", "api_key", "basic", "none"}) +_VALID_CATEGORIES = frozenset({"action", "trigger", "search", "transform"}) + + +@dataclass +class ValidationResult: + errors: list[str] = field(default_factory=list) + warnings: list[str] = field(default_factory=list) + + @property + def ok(self) -> bool: + return len(self.errors) == 0 + + +def validate_spec(spec: ConnectorSpec, registry: CapabilityRegistry) -> ValidationResult: + """Validate a ConnectorSpec against the registry. Pure, no side-effects.""" + result = ValidationResult() + + if not spec.app_type: + result.errors.append("app_type is empty") + if not spec.app_name: + result.errors.append("app_name is empty") + + _validate_auth(spec, result) + _validate_actions(spec, registry, result) + _validate_triggers(spec, registry, result) + _validate_no_route_collisions(spec, result) + + return result + + +def _validate_auth(spec: ConnectorSpec, result: ValidationResult) -> None: + if spec.auth.type not in _VALID_AUTH_TYPES: + result.errors.append( + f"Unknown auth type '{spec.auth.type}'. " + f"Valid: {', '.join(sorted(_VALID_AUTH_TYPES))}" + ) + + +def _validate_actions( + spec: ConnectorSpec, + registry: CapabilityRegistry, + result: ValidationResult, +) -> None: + manifest = registry.get(spec.app_type) + seen_names: set[str] = set() + + for action in spec.actions: + if action.name in seen_names: + result.errors.append(f"Duplicate action name: '{action.name}'") + seen_names.add(action.name) + + if action.category not in _VALID_CATEGORIES: + result.errors.append( + f"Action '{action.name}' has invalid category '{action.category}'" + ) + + for f in action.required_fields + action.optional_fields: + if f.type not in _VALID_FIELD_TYPES: + result.warnings.append( + f"Action '{action.name}', field '{f.name}': " + f"unknown type '{f.type}', defaulting to 'string'" + ) + + if manifest is not None: + known_actions = manifest.action_names() + if action.name not in known_actions: + result.warnings.append( + f"Action '{action.name}' is not in the {spec.app_type} " + f"capability manifest (known: {', '.join(known_actions)})" + ) + + if manifest is None and spec.app_type: + result.warnings.append( + f"App '{spec.app_type}' is not in the built-in registry. " + f"The generated connector will work but fields/auth are unverified." + ) + + +def _validate_triggers( + spec: ConnectorSpec, + registry: CapabilityRegistry, + result: ValidationResult, +) -> None: + manifest = registry.get(spec.app_type) + seen_names: set[str] = set() + + for trigger in spec.triggers: + if trigger.name in seen_names: + result.errors.append(f"Duplicate trigger name: '{trigger.name}'") + seen_names.add(trigger.name) + + if manifest is not None: + known_triggers = manifest.trigger_names() + if trigger.name not in known_triggers: + result.warnings.append( + f"Trigger '{trigger.name}' is not in the {spec.app_type} " + f"capability manifest (known: {', '.join(known_triggers)})" + ) + + +def _validate_no_route_collisions( + spec: ConnectorSpec, + result: ValidationResult, +) -> None: + paths: set[str] = set() + for action in spec.actions: + path = f"/{action.name}/{spec.version}/execute" + if path in paths: + result.errors.append(f"Route collision: {path}") + paths.add(path) + + for trigger in spec.triggers: + path = f"/{trigger.name}/{spec.version}/execute" + if path in paths: + result.errors.append(f"Route collision: {path}") + paths.add(path) diff --git a/src/workflows_cdk/cli/__init__.py b/src/workflows_cdk/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/workflows_cdk/cli/main.py b/src/workflows_cdk/cli/main.py new file mode 100644 index 0000000..c7360d2 --- /dev/null +++ b/src/workflows_cdk/cli/main.py @@ -0,0 +1,316 @@ +""" +CLI entry point: ``workflows create "Slack connector: send messages, list channels"`` + +Registered as a console_script in setup.py so ``pip install workflows-cdk`` +makes the ``workflows`` command available globally. +""" + +from __future__ import annotations + +import os +from pathlib import Path + +import click +from dotenv import load_dotenv +from rich.console import Console +from rich.panel import Panel +from rich.prompt import Prompt +from rich.tree import Tree + +load_dotenv() + +from ..ai.validator import validate_spec +from ..registry.registry import CapabilityRegistry +from ..spec.compiler import compile_connector, preview_tree +from ..spec.connector_spec import ConnectorSpec +from ..templates.matcher import match_template + +console = Console() + +ENV_FILE = Path.cwd() / ".env" + + +def _has_api_key() -> bool: + return bool( + os.environ.get("ANTHROPIC_API_KEY") or os.environ.get("OPENAI_API_KEY") + ) + + +def _run_setup() -> bool: + """Interactive first-time setup. Returns True if a key was configured.""" + console.print( + Panel( + "[bold]Welcome to Workflows CDK[/bold]\n\n" + "No API key found. Let's set one up.\n" + "You can reconfigure anytime with [cyan]workflows setup[/cyan].", + border_style="blue", + ) + ) + + provider = Prompt.ask( + "\n[bold]Which AI provider?[/bold]", + choices=["anthropic", "openai"], + default="anthropic", + ) + + if provider == "anthropic": + key_name = "ANTHROPIC_API_KEY" + hint = "sk-ant-..." + else: + key_name = "OPENAI_API_KEY" + hint = "sk-..." + + api_key = Prompt.ask(f"\n[bold]Paste your {key_name}[/bold] ({hint})") + api_key = api_key.strip() + + if not api_key: + console.print("[red]No key provided. Aborting setup.[/red]") + return False + + os.environ[key_name] = api_key + + _save_to_env(key_name, api_key) + + console.print(f"\n[green bold]Done![/green bold] {key_name} saved to [cyan]{ENV_FILE}[/cyan]") + console.print("[dim]You can also export it in your shell or edit .env directly.[/dim]\n") + return True + + +def _save_to_env(key_name: str, value: str) -> None: + """Append or update a key in the .env file.""" + lines: list[str] = [] + found = False + + if ENV_FILE.exists(): + for line in ENV_FILE.read_text().splitlines(): + stripped = line.lstrip("# ").split("=", 1)[0].strip() + if stripped == key_name: + lines.append(f"{key_name}={value}") + found = True + else: + lines.append(line) + + if not found: + lines.append(f"{key_name}={value}") + + ENV_FILE.write_text("\n".join(lines) + "\n") + + +# --------------------------------------------------------------------------- +# CLI group +# --------------------------------------------------------------------------- + +@click.group() +@click.version_option(package_name="workflows_cdk") +def cli() -> None: + """Stacksync Workflows CLI -- create connectors in one command.""" + + +# --------------------------------------------------------------------------- +# workflows setup +# --------------------------------------------------------------------------- + +@cli.command() +def setup() -> None: + """Configure your AI provider and API key.""" + _run_setup() + + +# --------------------------------------------------------------------------- +# workflows create +# --------------------------------------------------------------------------- + +@cli.command() +@click.argument("description") +@click.option( + "-o", "--output", + default=".", + type=click.Path(file_okay=False), + help="Parent directory for the generated project.", +) +@click.option("--dry-run", is_flag=True, help="Preview without writing files.") +@click.option( + "--no-ai", + is_flag=True, + help="Use template matching only (no LLM call).", +) +def create(description: str, output: str, dry_run: bool, no_ai: bool) -> None: + """Create a connector from a natural-language description (max ~30 words).""" + registry = CapabilityRegistry() + + spec: ConnectorSpec | None = None + + if no_ai: + spec = _template_path(description, registry) + else: + if not _has_api_key(): + configured = _run_setup() + if not configured: + console.print("[yellow]No API key. Falling back to template matching.[/yellow]") + spec = _template_path(description, registry) + + if spec is None: + spec = _ai_path(description, registry) + + if spec is None: + console.print( + "[red]Could not generate a connector spec from that description.[/red]\n" + "Try being more specific, e.g.:\n" + ' workflows create "Slack connector: send messages, list channels"' + ) + raise SystemExit(1) + + validation = validate_spec(spec, registry) + if validation.warnings: + for w in validation.warnings: + console.print(f"[yellow] warning:[/yellow] {w}") + if not validation.ok: + for e in validation.errors: + console.print(f"[red] error:[/red] {e}") + console.print("\n[red]Spec has blocking errors. Aborting.[/red]") + raise SystemExit(1) + + _show_preview(spec) + + if dry_run: + console.print("\n[dim]--dry-run: no files written.[/dim]") + return + + output_dir = Path(output).resolve() + project_dir = compile_connector(spec, output_dir) + console.print(f"\n[green bold]Connector created at:[/green bold] {project_dir}") + console.print("[dim]Run it with:[/dim] cd {0} && pip install -r requirements.txt && python main.py".format( + project_dir.name + )) + + +# --------------------------------------------------------------------------- +# workflows list +# --------------------------------------------------------------------------- + +@cli.command("list") +def list_capabilities() -> None: + """List all known app capabilities in the registry.""" + registry = CapabilityRegistry() + if not registry.slugs(): + console.print("[dim]No capabilities found.[/dim]") + return + + table_lines: list[str] = [] + for slug in registry.slugs(): + m = registry.get(slug) + if m is None: + continue + actions = ", ".join(m.action_names()) or "(none)" + triggers = ", ".join(m.trigger_names()) or "(none)" + table_lines.append( + f" [bold]{slug:12s}[/bold] " + f"actions: {actions} | triggers: {triggers}" + ) + + console.print(Panel( + "\n".join(table_lines), + title="[bold]Available capabilities[/bold]", + border_style="blue", + )) + + +# --------------------------------------------------------------------------- +# workflows inspect +# --------------------------------------------------------------------------- + +@cli.command() +@click.argument("app_slug") +def inspect(app_slug: str) -> None: + """Show detailed capabilities for a specific app.""" + registry = CapabilityRegistry() + manifest = registry.get(app_slug) + if manifest is None: + console.print(f"[red]Unknown app: '{app_slug}'[/red]") + console.print(f"Available: {', '.join(registry.slugs())}") + raise SystemExit(1) + + tree = Tree(f"[bold]{manifest.app.name}[/bold] ({manifest.app.slug})") + tree.add(f"[dim]{manifest.app.description}[/dim]") + tree.add(f"Auth: {manifest.app.auth.type}") + + if manifest.actions: + actions_branch = tree.add("[bold]Actions[/bold]") + for a in manifest.actions: + fields = ", ".join(f.name for f in a.required_fields) or "(none)" + actions_branch.add(f"{a.name} [{a.category}] -- {a.description} (fields: {fields})") + + if manifest.triggers: + triggers_branch = tree.add("[bold]Triggers[/bold]") + for t in manifest.triggers: + triggers_branch.add(f"{t.name} ({t.event}) -- {t.description}") + + console.print(tree) + + +# --------------------------------------------------------------------------- +# Internal paths +# --------------------------------------------------------------------------- + +def _template_path( + description: str, + registry: CapabilityRegistry, +) -> ConnectorSpec | None: + spec = match_template(description) + if spec is not None: + console.print("[green]Matched a built-in template.[/green]") + return spec + + +def _ai_path( + description: str, + registry: CapabilityRegistry, +) -> ConnectorSpec | None: + try: + from ..ai.planner import Planner, PlannerError + except Exception: + console.print( + "[yellow]AI planner unavailable. Falling back to template matching.[/yellow]" + ) + return _template_path(description, registry) + + try: + planner = Planner(registry) + except Exception: + console.print( + "[yellow]AI planner init failed. Falling back to template matching.[/yellow]" + ) + return _template_path(description, registry) + + try: + with console.status("[bold]Planning connector...[/bold]"): + spec = planner.plan(description) + except PlannerError as exc: + console.print(f"[yellow]{exc}[/yellow]") + console.print("[yellow]Falling back to template matching.[/yellow]") + return _template_path(description, registry) + + if spec.needs_clarification: + from ..ai.clarifier import render_clarification + + answers = render_clarification(spec) + if answers: + with console.status("[bold]Refining spec...[/bold]"): + spec = planner.refine(spec, answers) + + return spec + + +def _show_preview(spec: ConnectorSpec) -> None: + tree_text = preview_tree(spec) + + action_names = ", ".join(a.name for a in spec.actions) or "(none)" + trigger_names = ", ".join(t.name for t in spec.triggers) or "(none)" + + info = ( + f"[bold]{spec.app_name}[/bold] (type: {spec.app_type}, auth: {spec.auth.type})\n" + f"Actions: {action_names}\n" + f"Triggers: {trigger_names}\n\n" + f"[dim]{tree_text}[/dim]" + ) + console.print(Panel(info, title="[bold]Preview[/bold]", border_style="green")) diff --git a/src/workflows_cdk/registry/__init__.py b/src/workflows_cdk/registry/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/workflows_cdk/registry/capabilities/hubspot.yaml b/src/workflows_cdk/registry/capabilities/hubspot.yaml new file mode 100644 index 0000000..8e1f060 --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/hubspot.yaml @@ -0,0 +1,76 @@ +app: + name: HubSpot + slug: hubspot + description: "CRM, marketing, and sales automation platform" + auth: + type: oauth2 + scopes: + - "crm.objects.contacts.read" + - "crm.objects.contacts.write" + - "crm.objects.deals.read" + - "crm.objects.deals.write" + - "crm.objects.companies.read" + - "crm.objects.companies.write" + +actions: + - name: create_contact + category: action + description: "Create a new CRM contact" + required_fields: + - { name: email, type: string, description: "Contact email address" } + optional_fields: + - { name: firstname, type: string, description: "First name" } + - { name: lastname, type: string, description: "Last name" } + - { name: phone, type: string, description: "Phone number" } + - { name: company, type: string, description: "Company name" } + + - name: update_contact + category: action + description: "Update an existing CRM contact" + required_fields: + - { name: contact_id, type: string, description: "HubSpot contact ID" } + optional_fields: + - { name: email, type: string, description: "Email address" } + - { name: firstname, type: string, description: "First name" } + - { name: lastname, type: string, description: "Last name" } + - { name: phone, type: string, description: "Phone number" } + + - name: search_contacts + category: search + description: "Search for contacts by property values" + required_fields: + - { name: query, type: string, description: "Search query" } + + - name: create_deal + category: action + description: "Create a new deal in the pipeline" + required_fields: + - { name: dealname, type: string, description: "Deal name" } + - { name: amount, type: number, description: "Deal amount" } + optional_fields: + - { name: pipeline, type: string, description: "Pipeline ID" } + - { name: dealstage, type: string, description: "Stage ID" } + +triggers: + - name: contact_created + event: contact.created + description: "Fires when a new contact is created in HubSpot" + payload_fields: + - { name: id, type: string } + - { name: email, type: string } + - { name: firstname, type: string } + - { name: lastname, type: string } + + - name: deal_updated + event: deal.updated + description: "Fires when a deal is updated" + payload_fields: + - { name: id, type: string } + - { name: dealname, type: string } + - { name: dealstage, type: string } + - { name: amount, type: number } + +examples: + - "HubSpot connector to manage contacts and deals" + - "Create and update HubSpot contacts" + - "HubSpot CRM: create contact, search, update, manage deals" diff --git a/src/workflows_cdk/registry/capabilities/openai.yaml b/src/workflows_cdk/registry/capabilities/openai.yaml new file mode 100644 index 0000000..8b12489 --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/openai.yaml @@ -0,0 +1,51 @@ +app: + name: OpenAI + slug: openai + description: "AI models for text generation, embeddings, and classification" + auth: + type: api_key + scopes: [] + fields: + - { name: api_key, type: string, description: "OpenAI API key" } + +actions: + - name: chat_completion + category: action + description: "Generate a chat completion using GPT models" + required_fields: + - { name: prompt, type: string, description: "User message / prompt" } + optional_fields: + - { name: model, type: string, description: "Model name (default gpt-4o)" } + - { name: system_prompt, type: string, description: "System instruction" } + - { name: temperature, type: number, description: "Sampling temperature 0-2" } + - { name: max_tokens, type: number, description: "Max tokens in response" } + + - name: create_embedding + category: transform + description: "Create a vector embedding from text" + required_fields: + - { name: input, type: string, description: "Text to embed" } + optional_fields: + - { name: model, type: string, description: "Embedding model name" } + + - name: classify_text + category: transform + description: "Classify text into categories using a prompt" + required_fields: + - { name: text, type: string, description: "Text to classify" } + - { name: categories, type: array, description: "List of possible categories" } + + - name: summarize + category: transform + description: "Summarize a block of text" + required_fields: + - { name: text, type: string, description: "Text to summarize" } + optional_fields: + - { name: max_length, type: number, description: "Target summary length in words" } + +triggers: [] + +examples: + - "OpenAI connector for text generation" + - "AI enrichment: classify and summarize text" + - "OpenAI: generate completions, create embeddings, classify" diff --git a/src/workflows_cdk/registry/capabilities/postgres.yaml b/src/workflows_cdk/registry/capabilities/postgres.yaml new file mode 100644 index 0000000..52ddfd5 --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/postgres.yaml @@ -0,0 +1,60 @@ +app: + name: PostgreSQL + slug: postgres + description: "Relational database for structured data storage and queries" + auth: + type: basic + scopes: [] + fields: + - { name: host, type: string, description: "Database host" } + - { name: port, type: number, description: "Database port" } + - { name: database, type: string, description: "Database name" } + - { name: user, type: string, description: "Database user" } + - { name: password, type: string, description: "Database password" } + +actions: + - name: execute_query + category: action + description: "Execute a SQL query and return results" + required_fields: + - { name: query, type: string, description: "SQL query to execute" } + optional_fields: + - { name: params, type: array, description: "Query parameters for prepared statements" } + + - name: insert_row + category: action + description: "Insert a row into a table" + required_fields: + - { name: table, type: string, description: "Table name" } + - { name: data, type: object, description: "Column-value pairs to insert" } + + - name: update_rows + category: action + description: "Update rows matching a condition" + required_fields: + - { name: table, type: string, description: "Table name" } + - { name: data, type: object, description: "Column-value pairs to update" } + - { name: where, type: string, description: "WHERE clause (without the WHERE keyword)" } + + - name: search_rows + category: search + description: "Search rows in a table" + required_fields: + - { name: table, type: string, description: "Table name" } + optional_fields: + - { name: where, type: string, description: "WHERE clause" } + - { name: limit, type: number, description: "Max rows to return" } + - { name: order_by, type: string, description: "ORDER BY clause" } + +triggers: + - name: new_row + event: row.inserted + description: "Fires when a new row is inserted (polling-based)" + payload_fields: + - { name: table, type: string } + - { name: row, type: object } + +examples: + - "PostgreSQL connector for querying and inserting data" + - "Postgres: execute queries, insert rows, search tables" + - "Database connector to read and write Postgres rows" diff --git a/src/workflows_cdk/registry/capabilities/salesforce.yaml b/src/workflows_cdk/registry/capabilities/salesforce.yaml new file mode 100644 index 0000000..b89353a --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/salesforce.yaml @@ -0,0 +1,70 @@ +app: + name: Salesforce + slug: salesforce + description: "Enterprise CRM and sales management platform" + auth: + type: oauth2 + scopes: + - "api" + - "refresh_token" + +actions: + - name: create_lead + category: action + description: "Create a new lead in Salesforce" + required_fields: + - { name: LastName, type: string, description: "Lead last name" } + - { name: Company, type: string, description: "Company name" } + optional_fields: + - { name: FirstName, type: string, description: "First name" } + - { name: Email, type: string, description: "Email address" } + - { name: Phone, type: string, description: "Phone number" } + + - name: create_opportunity + category: action + description: "Create a new sales opportunity" + required_fields: + - { name: Name, type: string, description: "Opportunity name" } + - { name: StageName, type: string, description: "Current stage" } + - { name: CloseDate, type: string, description: "Expected close date (YYYY-MM-DD)" } + optional_fields: + - { name: Amount, type: number, description: "Deal amount" } + - { name: AccountId, type: string, description: "Associated account ID" } + + - name: search_records + category: search + description: "Search Salesforce records using SOQL" + required_fields: + - { name: query, type: string, description: "SOQL query string" } + + - name: update_record + category: action + description: "Update an existing Salesforce record" + required_fields: + - { name: object_type, type: string, description: "Salesforce object type (Lead, Contact, etc.)" } + - { name: record_id, type: string, description: "Record ID" } + - { name: fields, type: object, description: "Fields to update" } + +triggers: + - name: new_lead + event: lead.created + description: "Fires when a new lead is created" + payload_fields: + - { name: Id, type: string } + - { name: LastName, type: string } + - { name: Email, type: string } + - { name: Company, type: string } + + - name: opportunity_closed + event: opportunity.closed + description: "Fires when an opportunity is closed (won or lost)" + payload_fields: + - { name: Id, type: string } + - { name: Name, type: string } + - { name: StageName, type: string } + - { name: Amount, type: number } + +examples: + - "Salesforce connector to manage leads and opportunities" + - "Create Salesforce leads and search records" + - "Salesforce CRM: leads, opportunities, record updates" diff --git a/src/workflows_cdk/registry/capabilities/slack.yaml b/src/workflows_cdk/registry/capabilities/slack.yaml new file mode 100644 index 0000000..aedad13 --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/slack.yaml @@ -0,0 +1,66 @@ +app: + name: Slack + slug: slack + description: "Team messaging and collaboration platform" + auth: + type: oauth2 + scopes: + - "chat:write" + - "channels:read" + - "channels:history" + - "reactions:read" + - "reactions:write" + - "users:read" + +actions: + - name: send_message + category: action + description: "Send a message to a Slack channel or DM" + required_fields: + - { name: channel, type: string, description: "Channel name or ID" } + - { name: text, type: string, description: "Message body" } + optional_fields: + - { name: thread_ts, type: string, description: "Thread timestamp for replies" } + + - name: list_channels + category: search + description: "List all channels in the workspace" + required_fields: [] + + - name: add_reaction + category: action + description: "Add an emoji reaction to a message" + required_fields: + - { name: channel, type: string, description: "Channel containing the message" } + - { name: timestamp, type: string, description: "Message timestamp" } + - { name: emoji, type: string, description: "Emoji name without colons" } + + - name: list_users + category: search + description: "List all users in the workspace" + required_fields: [] + +triggers: + - name: new_message + event: message.created + description: "Fires when a new message is posted in a channel" + payload_fields: + - { name: channel, type: string } + - { name: text, type: string } + - { name: user, type: string } + - { name: ts, type: string } + + - name: new_reaction + event: reaction.added + description: "Fires when a reaction is added to a message" + payload_fields: + - { name: channel, type: string } + - { name: user, type: string } + - { name: reaction, type: string } + - { name: ts, type: string } + +examples: + - "Slack connector that sends messages to channels" + - "Listen for new Slack messages" + - "Slack: send messages, list channels, react to new messages" + - "Post notifications to Slack" diff --git a/src/workflows_cdk/registry/capabilities/stripe.yaml b/src/workflows_cdk/registry/capabilities/stripe.yaml new file mode 100644 index 0000000..8baa7f6 --- /dev/null +++ b/src/workflows_cdk/registry/capabilities/stripe.yaml @@ -0,0 +1,62 @@ +app: + name: Stripe + slug: stripe + description: "Online payment processing platform" + auth: + type: api_key + scopes: [] + fields: + - { name: api_key, type: string, description: "Stripe secret API key" } + +actions: + - name: create_customer + category: action + description: "Create a new Stripe customer" + required_fields: + - { name: email, type: string, description: "Customer email address" } + optional_fields: + - { name: name, type: string, description: "Customer full name" } + - { name: metadata, type: object, description: "Key-value metadata" } + + - name: create_charge + category: action + description: "Create a payment charge" + required_fields: + - { name: amount, type: number, description: "Amount in cents" } + - { name: currency, type: string, description: "Three-letter ISO currency code" } + - { name: customer, type: string, description: "Customer ID" } + + - name: list_customers + category: search + description: "List Stripe customers" + required_fields: [] + optional_fields: + - { name: limit, type: number, description: "Max results to return" } + + - name: get_balance + category: search + description: "Retrieve the current account balance" + required_fields: [] + +triggers: + - name: customer_created + event: customer.created + description: "Fires when a new customer is created" + payload_fields: + - { name: id, type: string } + - { name: email, type: string } + - { name: name, type: string } + + - name: payment_succeeded + event: payment_intent.succeeded + description: "Fires when a payment succeeds" + payload_fields: + - { name: id, type: string } + - { name: amount, type: number } + - { name: currency, type: string } + - { name: customer, type: string } + +examples: + - "Stripe connector to create customers and process payments" + - "Listen for new Stripe payments" + - "Stripe: create customer, charge, list customers" diff --git a/src/workflows_cdk/registry/manifest.py b/src/workflows_cdk/registry/manifest.py new file mode 100644 index 0000000..93416e7 --- /dev/null +++ b/src/workflows_cdk/registry/manifest.py @@ -0,0 +1,87 @@ +""" +Pydantic models that mirror the capability.yaml manifest format. + +Each YAML file in ``registry/capabilities/`` is loaded into an +``AppManifest`` instance so both the LLM planner and the deterministic +validator work against a single, typed source of truth. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Literal, Optional + +import yaml +from pydantic import BaseModel, Field + + +class ManifestField(BaseModel): + name: str + type: Literal["string", "number", "boolean", "object", "array"] = "string" + description: str = "" + + +class ManifestAuth(BaseModel): + type: Literal["oauth2", "api_key", "basic", "none"] = "oauth2" + scopes: list[str] = Field(default_factory=list) + fields: list[ManifestField] = Field(default_factory=list) + + +class ManifestAction(BaseModel): + name: str + category: Literal["action", "trigger", "search", "transform"] = "action" + description: str = "" + required_fields: list[ManifestField] = Field(default_factory=list) + optional_fields: list[ManifestField] = Field(default_factory=list) + + +class ManifestTrigger(BaseModel): + name: str + event: str = "" + description: str = "" + payload_fields: list[ManifestField] = Field(default_factory=list) + + +class ManifestApp(BaseModel): + name: str + slug: str + description: str = "" + auth: ManifestAuth = Field(default_factory=ManifestAuth) + + +class AppManifest(BaseModel): + """Top-level model for a single ``capability.yaml`` file.""" + + app: ManifestApp + actions: list[ManifestAction] = Field(default_factory=list) + triggers: list[ManifestTrigger] = Field(default_factory=list) + examples: list[str] = Field(default_factory=list) + + @classmethod + def from_yaml(cls, path: Path) -> "AppManifest": + with open(path, "r") as fh: + raw = yaml.safe_load(fh) or {} + return cls.model_validate(raw) + + def action_names(self) -> list[str]: + return [a.name for a in self.actions] + + def trigger_names(self) -> list[str]: + return [t.name for t in self.triggers] + + def summary_for_llm(self) -> dict: + """Compact JSON-serialisable summary injected into the LLM prompt.""" + return { + "app": self.app.slug, + "name": self.app.name, + "description": self.app.description, + "auth_type": self.app.auth.type, + "actions": [ + {"name": a.name, "category": a.category, "description": a.description} + for a in self.actions + ], + "triggers": [ + {"name": t.name, "event": t.event, "description": t.description} + for t in self.triggers + ], + } diff --git a/src/workflows_cdk/registry/registry.py b/src/workflows_cdk/registry/registry.py new file mode 100644 index 0000000..56e9517 --- /dev/null +++ b/src/workflows_cdk/registry/registry.py @@ -0,0 +1,77 @@ +""" +Capability registry -- loads all YAML manifests and provides lookup / search. + +Usage:: + + registry = CapabilityRegistry() # auto-loads built-in manifests + registry.load_directory(Path("./custom_capabilities")) # extend with user-provided + slack = registry.get("slack") # by slug + hits = registry.search("send message slack") # keyword search +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Optional + +from .manifest import AppManifest + +logger = logging.getLogger(__name__) + +_BUILTIN_DIR = Path(__file__).parent / "capabilities" + + +class CapabilityRegistry: + """In-memory index of every known app manifest.""" + + def __init__(self, *, load_builtins: bool = True) -> None: + self._manifests: dict[str, AppManifest] = {} + if load_builtins: + self.load_directory(_BUILTIN_DIR) + + def load_directory(self, directory: Path) -> None: + if not directory.is_dir(): + logger.warning("Capabilities directory not found: %s", directory) + return + for path in sorted(directory.glob("*.yaml")): + try: + manifest = AppManifest.from_yaml(path) + self._manifests[manifest.app.slug] = manifest + except Exception: + logger.warning("Failed to load manifest %s", path, exc_info=True) + + def get(self, slug: str) -> Optional[AppManifest]: + return self._manifests.get(slug) + + def all(self) -> list[AppManifest]: + return list(self._manifests.values()) + + def slugs(self) -> list[str]: + return sorted(self._manifests.keys()) + + def search(self, query: str) -> list[AppManifest]: + """Rank manifests by simple keyword overlap with *query*.""" + tokens = set(query.lower().split()) + scored: list[tuple[int, AppManifest]] = [] + + for manifest in self._manifests.values(): + corpus = " ".join([ + manifest.app.name.lower(), + manifest.app.slug.lower(), + manifest.app.description.lower(), + " ".join(manifest.examples).lower(), + " ".join(a.name.replace("_", " ") for a in manifest.actions), + " ".join(a.description.lower() for a in manifest.actions), + " ".join(t.name.replace("_", " ") for t in manifest.triggers), + ]) + score = sum(1 for t in tokens if t in corpus) + if score > 0: + scored.append((score, manifest)) + + scored.sort(key=lambda pair: pair[0], reverse=True) + return [m for _, m in scored] + + def summaries_for_llm(self) -> list[dict]: + """All manifests in the compact dict format consumed by the planner prompt.""" + return [m.summary_for_llm() for m in self._manifests.values()] diff --git a/src/workflows_cdk/spec/__init__.py b/src/workflows_cdk/spec/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/workflows_cdk/spec/compiler.py b/src/workflows_cdk/spec/compiler.py new file mode 100644 index 0000000..c688a3b --- /dev/null +++ b/src/workflows_cdk/spec/compiler.py @@ -0,0 +1,588 @@ +""" +Compiler: turns a ``ConnectorSpec`` into a scaffolded CDK project on disk. + +The generated project is immediately runnable with ``./run_dev.sh`` and +follows every convention documented in the Stacksync Workflows platform +(module schema v1.0.0, /execute + /content + /schema endpoints, Docker +deployment, file-based route discovery under ``src/modules/``). +""" + +from __future__ import annotations + +import json +import textwrap +from pathlib import Path +from typing import Optional + +import yaml + +from .connector_spec import ActionSpec, AuthSpec, ConnectorSpec, FieldSpec, TriggerSpec + +MODULES_DIR = "src/modules" +DEFAULT_PORT = 2003 + + +def compile_connector(spec: ConnectorSpec, output_dir: Path) -> Path: + """Write a full CDK project to *output_dir* / *spec.directory_name*. + + Returns the root directory of the generated project. + """ + project_dir = output_dir / spec.directory_name + project_dir.mkdir(parents=True, exist_ok=True) + + _write_main_py(spec, project_dir) + _write_app_config(spec, project_dir) + _write_requirements(project_dir) + _write_capability_yaml(spec, project_dir) + _write_deployment_files(spec, project_dir) + + for action in spec.actions: + _write_action_route(spec, action, project_dir) + + for trigger in spec.triggers: + _write_trigger_route(spec, trigger, project_dir) + + return project_dir + + +def preview_tree(spec: ConnectorSpec) -> str: + """Return a textual file-tree preview without writing anything.""" + lines = [f"{spec.directory_name}/"] + lines.append(" main.py") + lines.append(" app_config.yaml") + lines.append(" requirements.txt") + lines.append(" capability.yaml") + lines.append(" Dockerfile") + lines.append(" run_dev.sh") + lines.append(" config/") + lines.append(" Dockerfile.dev") + lines.append(" entrypoint.sh") + lines.append(" gunicorn_config.py") + lines.append(f" {MODULES_DIR}/") + + for action in spec.actions: + lines.append(f" {action.name}/") + lines.append(f" {spec.version}/") + lines.append(f" route.py") + lines.append(f" schema.json") + lines.append(f" module_config.yaml") + + for trigger in spec.triggers: + lines.append(f" {trigger.name}/") + lines.append(f" {spec.version}/") + lines.append(f" route.py") + lines.append(f" schema.json") + lines.append(f" module_config.yaml") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Static project files +# --------------------------------------------------------------------------- + +def _write_main_py(spec: ConnectorSpec, project_dir: Path) -> None: + slug = spec.app_type.replace("-", "_") + content = textwrap.dedent(f"""\ + from flask import Flask + from workflows_cdk import Router + + app = Flask("{slug}") + router = Router(app) + + if __name__ == "__main__": + router.run_app(app) + """) + (project_dir / "main.py").write_text(content) + + +def _write_app_config(spec: ConnectorSpec, project_dir: Path) -> None: + config = { + "app_settings": { + "app_type": spec.app_type, + "app_name": spec.app_name, + "app_icon_svg_url": "", + "app_description": f"Stacksync connector for {spec.app_name}", + "routes_directory": MODULES_DIR, + }, + "local_development_settings": { + "sentry_dsn": "", + "cors_origins": ["*"], + "routes_directory": MODULES_DIR, + "debug": True, + "host": "0.0.0.0", + "port": DEFAULT_PORT, + }, + } + (project_dir / "app_config.yaml").write_text( + yaml.dump(config, default_flow_style=False, sort_keys=False) + ) + + +def _write_requirements(project_dir: Path) -> None: + content = textwrap.dedent("""\ + # Flask + flask + werkzeug + requests + # Server + gunicorn==22.0.0 + # Monitoring + sentry-sdk[Flask] + # Core + pydantic>=2.0.0 + PyYAML>=6.0.1 + """) + (project_dir / "requirements.txt").write_text(content) + + +def _write_capability_yaml(spec: ConnectorSpec, project_dir: Path) -> None: + data: dict = { + "app": { + "name": spec.app_name.replace(" Connector", ""), + "slug": spec.app_type, + "description": f"Connector for {spec.app_name.replace(' Connector', '')}", + "auth": { + "type": spec.auth.type, + "scopes": spec.auth.scopes, + }, + }, + "actions": [ + { + "name": a.name, + "category": a.category, + "description": a.description, + "required_fields": [ + {"name": f.name, "type": f.type, "description": f.description} + for f in a.required_fields + ], + } + for a in spec.actions + ], + "triggers": [ + { + "name": t.name, + "event": t.event, + "description": t.description, + } + for t in spec.triggers + ], + } + (project_dir / "capability.yaml").write_text( + yaml.dump(data, default_flow_style=False, sort_keys=False) + ) + + +# --------------------------------------------------------------------------- +# Deployment files +# --------------------------------------------------------------------------- + +def _write_deployment_files(spec: ConnectorSpec, project_dir: Path) -> None: + config_dir = project_dir / "config" + config_dir.mkdir(parents=True, exist_ok=True) + + (project_dir / "Dockerfile").write_text(textwrap.dedent("""\ + FROM python:3.11-slim + ARG ENVIRONMENT + + WORKDIR /usr/src/app + + RUN pip install --upgrade pip + + RUN apt-get update && \\ + apt-get install -y git curl && \\ + rm -rf /var/lib/apt/lists/* + + RUN pip install git+https://github.com/stacksyncdata/workflows-cdk.git@prod + + COPY requirements.txt ./ + RUN pip3 install -r requirements.txt + + COPY . . + + EXPOSE 8080 + + RUN chmod +x ./entrypoint.sh + + ENTRYPOINT ["sh", "entrypoint.sh"] + """)) + + (config_dir / "Dockerfile.dev").write_text(textwrap.dedent("""\ + FROM python:3.11-slim + + WORKDIR /usr/src/app + + RUN apt-get update && \\ + apt-get install -y git curl && \\ + rm -rf /var/lib/apt/lists/* + + RUN pip install --upgrade pip + + RUN pip install git+https://github.com/stacksyncdata/workflows-cdk.git@prod + + COPY requirements.txt ./ + RUN pip3 install -r requirements.txt + + COPY . . + + RUN chmod +x ./config/entrypoint.sh + + EXPOSE 8080 + + ENTRYPOINT ["sh", "./config/entrypoint.sh"] + """)) + + (config_dir / "entrypoint.sh").write_text( + "exec gunicorn --config /usr/src/app/config/gunicorn_config.py main:app\n" + ) + + (config_dir / "gunicorn_config.py").write_text(textwrap.dedent("""\ + bind = "0.0.0.0:8080" + accesslog = "-" + errorlog = "-" + capture_output = True + enable_stdio_inheritance = True + + workers = 2 + threads = 1 + timeout = 360 + """)) + + port = DEFAULT_PORT + (project_dir / "run_dev.sh").write_text(textwrap.dedent(f"""\ + #!/bin/bash + + REBUILD=false + for arg in "$@"; do + if [ "$arg" == "--build" ]; then + REBUILD=true + fi + done + + echo "" + echo "Stacksync App Connector" + echo "Documentation: https://docs.stacksync.com/workflows/app-connector" + echo "" + + if [ ! -d "config" ]; then + mkdir -p config + fi + + PORT={port} + DOCKERFILE_PATH="config/Dockerfile.dev" + REPO_NAME=$(basename "$PWD") + APP_NAME="workflows-app-${{REPO_NAME}}" + + echo "Preparing ${{APP_NAME}}..." + + IMAGE_EXISTS=$(docker images -q ${{APP_NAME}} 2> /dev/null) + + if [ -z "$IMAGE_EXISTS" ] || [ "$REBUILD" == "true" ]; then + if [ "$REBUILD" == "true" ]; then + echo "Forcing rebuild of Docker image: ${{APP_NAME}}" + docker build --no-cache -t ${{APP_NAME}} -f ${{DOCKERFILE_PATH}} . + else + echo "Docker image not found. Building: ${{APP_NAME}}" + docker build -t ${{APP_NAME}} -f ${{DOCKERFILE_PATH}} . + fi + else + echo "Docker image ${{APP_NAME}} already exists. Skipping build." + echo "Use --build flag to force a rebuild." + fi + + if [ $? -ne 0 ]; then + echo "Docker build failed. Exiting..." + exit 1 + fi + + echo "Starting container on port ${{PORT}}..." + docker run --rm -p ${{PORT}}:8080 -it -e ENVIRONMENT=dev -e REGION=besg --name=${{APP_NAME}} -v $PWD:/usr/src/app/ ${{APP_NAME}} + """)) + + (project_dir / "run_dev.bat").write_text(textwrap.dedent(f"""\ + @echo off + setlocal enabledelayedexpansion + + set "REBUILD=false" + for %%a in (%*) do ( + if "%%a"=="--build" set "REBUILD=true" + ) + + echo. + echo Stacksync App Connector + echo Documentation: https://docs.stacksync.com/workflows/app-connector + echo. + + if not exist "config" mkdir config + + set "PORT={port}" + set "DOCKERFILE_PATH=config\\Dockerfile.dev" + + for %%%%I in ("%%CD%%") do set "DIRNAME=%%%%~nxI" + set "APP_NAME=workflows-app-!DIRNAME!" + + echo Preparing !APP_NAME!... + + set "IMAGE_EXISTS=" + for /f %%%%i in ('docker images -q !APP_NAME! 2^>nul') do set "IMAGE_EXISTS=%%%%i" + + if "!IMAGE_EXISTS!"=="" ( + echo Docker image not found. Building: !APP_NAME! + docker build -t !APP_NAME! -f !DOCKERFILE_PATH! . + ) else if "!REBUILD!"=="true" ( + echo Forcing rebuild: !APP_NAME! + docker build --no-cache -t !APP_NAME! -f !DOCKERFILE_PATH! . + ) else ( + echo Docker image !APP_NAME! already exists. Use --build to rebuild. + ) + + if errorlevel 1 ( + echo Docker build failed. Exiting... + exit /b 1 + ) + + echo Starting container on port !PORT!... + docker run --rm -p !PORT!:8080 -it -e ENVIRONMENT=dev -e REGION=besg --name=!APP_NAME! -v %%CD%%:/usr/src/app/ !APP_NAME! + """)) + + +# --------------------------------------------------------------------------- +# Module config +# --------------------------------------------------------------------------- + +def _write_module_config(name: str, description: str, route_dir: Path) -> None: + """Write module_config.yaml for a single module version.""" + human_name = name.replace("_", " ").title() + config = { + "module_settings": { + "module_name": human_name, + "module_description": description or f"{human_name} Module", + } + } + (route_dir / "module_config.yaml").write_text( + yaml.dump(config, default_flow_style=False, sort_keys=False) + ) + + +# --------------------------------------------------------------------------- +# Module Schema (Stacksync format) +# --------------------------------------------------------------------------- + +_WIDGET_MAP = { + "string": "input", + "number": "input", + "boolean": "checkbox", + "object": "SelectWidget", + "array": "input", +} + + +def _build_module_schema( + fields: list[FieldSpec], + auth: AuthSpec, + app_type: str, +) -> dict: + """Build a Stacksync Module Schema (v1.0.0) from field specs.""" + schema_fields: list[dict] = [] + + if auth.type in ("oauth2", "api_key"): + conn_field: dict = { + "type": "connection", + "id": "api_connection", + "label": "Connection", + "allowed_app_types": [app_type], + "allowed_connection_management_types": ["managed"], + "validation": {"required": True}, + } + schema_fields.append(conn_field) + + for f in fields: + field_def: dict = { + "id": f.name, + "type": f.type, + "label": f.name.replace("_", " ").title(), + "description": f.description, + } + if f.required: + field_def["validation"] = {"required": True} + + widget = _WIDGET_MAP.get(f.type, "input") + field_def["ui_options"] = {"ui_widget": widget} + + schema_fields.append(field_def) + + field_ids = [sf["id"] for sf in schema_fields] + + return { + "metadata": {"workflows_module_schema_version": "1.0.0"}, + "fields": schema_fields, + "ui_options": {"ui_order": field_ids}, + } + + +# --------------------------------------------------------------------------- +# Route generation +# --------------------------------------------------------------------------- + +def _extract_imports(code: str) -> tuple[list[str], str]: + """Separate import statements from implementation code. + + Returns (import_lines, remaining_code). + """ + import_lines: list[str] = [] + body_lines: list[str] = [] + for line in code.splitlines(): + stripped = line.strip() + if stripped.startswith("import ") or stripped.startswith("from "): + import_lines.append(stripped) + else: + body_lines.append(line) + remaining = "\n".join(body_lines).strip() + return import_lines, remaining + + +def _indent(code: str, level: int = 1) -> str: + """Indent each line of *code* by *level* * 4 spaces.""" + prefix = " " * level + return "\n".join(prefix + line if line.strip() else "" for line in code.splitlines()) + + +def _write_action_route( + spec: ConnectorSpec, + action: ActionSpec, + project_dir: Path, +) -> None: + route_dir = project_dir / MODULES_DIR / action.name / spec.version + route_dir.mkdir(parents=True, exist_ok=True) + + all_fields = list(action.required_fields) + list(action.optional_fields) + + extra_imports: list[str] = [] + if action.implementation.strip(): + extra_imports, clean_impl = _extract_imports(action.implementation) + execute_body = _indent(clean_impl) + else: + execute_body = _build_stub_body(spec, action.name, all_fields, action.category) + + route_code = _build_route_file(execute_body, extra_imports) + (route_dir / "route.py").write_text(route_code) + + schema = _build_module_schema(all_fields, spec.auth, spec.app_type) + (route_dir / "schema.json").write_text(json.dumps(schema, indent=2) + "\n") + + _write_module_config(action.name, action.description, route_dir) + + +def _write_trigger_route( + spec: ConnectorSpec, + trigger: TriggerSpec, + project_dir: Path, +) -> None: + route_dir = project_dir / MODULES_DIR / trigger.name / spec.version + route_dir.mkdir(parents=True, exist_ok=True) + + extra_imports: list[str] = [] + if trigger.implementation.strip(): + extra_imports, clean_impl = _extract_imports(trigger.implementation) + execute_body = _indent(clean_impl) + else: + execute_body = _build_trigger_stub_body(spec, trigger) + + route_code = _build_route_file(execute_body, extra_imports) + (route_dir / "route.py").write_text(route_code) + + schema = _build_module_schema(trigger.payload_fields, spec.auth, spec.app_type) + (route_dir / "schema.json").write_text(json.dumps(schema, indent=2) + "\n") + + _write_module_config(trigger.name, trigger.description, route_dir) + + +def _build_route_file(execute_body: str, extra_imports: list[str] | None = None) -> str: + """Assemble a complete route.py with /execute, /content, and /schema.""" + base_imports = [ + "import json", + "", + "import requests", + "from flask import request as flask_request", + "from workflows_cdk import Request, Response, ManagedError", + "from main import router", + ] + if extra_imports: + already = {line.strip() for line in base_imports if line.strip()} + for imp in extra_imports: + if imp not in already: + base_imports.insert(1, imp) + + import_block = "\n".join(base_imports) + + body = ( + import_block + + "\n\n\n" + + "@router.route(\"/execute\", methods=[\"POST\"])\n" + + "def execute():\n" + + " req = Request(flask_request)\n" + + " data = req.data\n" + + " credentials = req.credentials\n" + + "\n" + + execute_body + "\n" + + "\n\n" + + "@router.route(\"/content\", methods=[\"POST\"])\n" + + "def content():\n" + + " req = Request(flask_request)\n" + + " data = req.data\n" + + "\n" + + " if not data:\n" + + ' return Response(data={"message": "Missing request data"}, status_code=400)\n' + + "\n" + + ' form_data = data.get("form_data", {})\n' + + ' content_object_names = data.get("content_object_names", [])\n' + + "\n" + + " if isinstance(content_object_names, list) and content_object_names and isinstance(content_object_names[0], dict):\n" + + ' content_object_names = [obj.get("id") for obj in content_object_names if "id" in obj]\n' + + "\n" + + " content_objects = []\n" + + "\n" + + ' return Response(data={"content_objects": content_objects})\n' + + "\n\n" + + "@router.route(\"/schema\", methods=[\"POST\"])\n" + + "def schema():\n" + + " req = Request(flask_request)\n" + + ' schema_path = __file__.replace("route.py", "schema.json")\n' + + " with open(schema_path) as f:\n" + + " base_schema = json.load(f)\n" + + " return Response(data=base_schema)\n" + ) + return body + + +def _build_stub_body( + spec: ConnectorSpec, + action_name: str, + fields: list[FieldSpec], + category: str, +) -> str: + """Fallback stub when the LLM didn't generate implementation code.""" + lines: list[str] = [] + for f in fields: + lines.append(f' {f.name} = data.get("{f.name}")') + + lines.append("") + lines.append(f' # TODO: implement {spec.app_name} API call for "{action_name}"') + lines.append("") + + if category == "search": + lines.append(' return Response(data={"results": []})') + else: + lines.append(' return Response(data={"success": True})') + + return "\n".join(lines) + + +def _build_trigger_stub_body(spec: ConnectorSpec, trigger: TriggerSpec) -> str: + lines = [ + f' # TODO: implement polling / webhook logic for "{trigger.name}"', + f" # Event: {trigger.event}", + "", + ' return Response(data={"events": []})', + ] + return "\n".join(lines) diff --git a/src/workflows_cdk/spec/connector_spec.py b/src/workflows_cdk/spec/connector_spec.py new file mode 100644 index 0000000..ca5baec --- /dev/null +++ b/src/workflows_cdk/spec/connector_spec.py @@ -0,0 +1,78 @@ +""" +Intermediate connector specification format. + +The LLM planner produces a ConnectorSpec; the deterministic validator checks it; +the compiler turns it into a scaffolded CDK project on disk. No component +downstream of the planner ever sees raw LLM text -- only this typed model. +""" + +from __future__ import annotations + +from typing import Literal, Optional + +from pydantic import BaseModel, Field + + +class FieldSpec(BaseModel): + name: str + type: Literal["string", "number", "boolean", "object", "array"] = "string" + description: str = "" + required: bool = True + + +class AuthSpec(BaseModel): + type: Literal["oauth2", "api_key", "basic", "none"] = "oauth2" + scopes: list[str] = Field(default_factory=list) + fields: list[FieldSpec] = Field(default_factory=list) + + +class ActionSpec(BaseModel): + name: str + category: Literal["action", "trigger", "search", "transform"] = "action" + description: str = "" + required_fields: list[FieldSpec] = Field(default_factory=list) + optional_fields: list[FieldSpec] = Field(default_factory=list) + implementation: str = Field( + default="", + description="Python code for the /execute endpoint body (after data extraction)", + ) + + +class TriggerSpec(BaseModel): + name: str + event: str = "" + description: str = "" + payload_fields: list[FieldSpec] = Field(default_factory=list) + implementation: str = Field( + default="", + description="Python code for the /execute endpoint body (polling/webhook logic)", + ) + + +class AmbiguitySpec(BaseModel): + """A single point the planner could not resolve from the prompt alone.""" + + question: str + options: list[str] = Field(default_factory=list) + default: Optional[str] = None + + +class ConnectorSpec(BaseModel): + """Full specification for a connector to be scaffolded.""" + + app_type: str = Field(description="Slug identifier, e.g. 'slack'") + app_name: str = Field(description="Human-readable name, e.g. 'Slack Connector'") + version: str = "v1" + actions: list[ActionSpec] = Field(default_factory=list) + triggers: list[TriggerSpec] = Field(default_factory=list) + auth: AuthSpec = Field(default_factory=AuthSpec) + confidence: float = Field(default=1.0, ge=0.0, le=1.0) + ambiguities: list[AmbiguitySpec] = Field(default_factory=list) + + @property + def needs_clarification(self) -> bool: + return self.confidence < 0.85 and len(self.ambiguities) > 0 + + @property + def directory_name(self) -> str: + return f"{self.app_type}-connector" diff --git a/src/workflows_cdk/templates/__init__.py b/src/workflows_cdk/templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/workflows_cdk/templates/library/crm_sync.yaml b/src/workflows_cdk/templates/library/crm_sync.yaml new file mode 100644 index 0000000..77d28f9 --- /dev/null +++ b/src/workflows_cdk/templates/library/crm_sync.yaml @@ -0,0 +1,54 @@ +app_type: hubspot +app_name: HubSpot Connector +version: v1 + +keywords: + - hubspot + - crm + - contact + - lead + - deal + - sales + - customer + +auth: + type: oauth2 + scopes: + - "crm.objects.contacts.read" + - "crm.objects.contacts.write" + - "crm.objects.deals.read" + - "crm.objects.deals.write" + +actions: + - name: create_contact + category: action + description: "Create a new CRM contact" + required_fields: + - { name: email, type: string, description: "Contact email address" } + optional_fields: + - { name: firstname, type: string, description: "First name" } + - { name: lastname, type: string, description: "Last name" } + + - name: update_contact + category: action + description: "Update an existing contact" + required_fields: + - { name: contact_id, type: string, description: "HubSpot contact ID" } + optional_fields: + - { name: email, type: string, description: "Email address" } + - { name: firstname, type: string, description: "First name" } + - { name: lastname, type: string, description: "Last name" } + + - name: search_contacts + category: search + description: "Search for contacts" + required_fields: + - { name: query, type: string, description: "Search query" } + +triggers: + - name: contact_created + event: contact.created + description: "Fires when a new contact is created" + payload_fields: + - { name: id, type: string } + - { name: email, type: string } diff --git a/src/workflows_cdk/templates/library/data_enrichment.yaml b/src/workflows_cdk/templates/library/data_enrichment.yaml new file mode 100644 index 0000000..81bb594 --- /dev/null +++ b/src/workflows_cdk/templates/library/data_enrichment.yaml @@ -0,0 +1,43 @@ +app_type: openai +app_name: OpenAI Connector +version: v1 + +keywords: + - openai + - ai + - gpt + - enrich + - enrichment + - classify + - summarize + - embedding + - generate + +auth: + type: api_key + scopes: [] + +actions: + - name: chat_completion + category: action + description: "Generate a chat completion using GPT" + required_fields: + - { name: prompt, type: string, description: "User message / prompt" } + optional_fields: + - { name: model, type: string, description: "Model name (default gpt-4o)" } + - { name: system_prompt, type: string, description: "System instruction" } + + - name: classify_text + category: transform + description: "Classify text into categories" + required_fields: + - { name: text, type: string, description: "Text to classify" } + - { name: categories, type: array, description: "List of possible categories" } + + - name: summarize + category: transform + description: "Summarize a block of text" + required_fields: + - { name: text, type: string, description: "Text to summarize" } + +triggers: [] diff --git a/src/workflows_cdk/templates/library/payment_processor.yaml b/src/workflows_cdk/templates/library/payment_processor.yaml new file mode 100644 index 0000000..5db58c0 --- /dev/null +++ b/src/workflows_cdk/templates/library/payment_processor.yaml @@ -0,0 +1,47 @@ +app_type: stripe +app_name: Stripe Connector +version: v1 + +keywords: + - stripe + - payment + - charge + - customer + - billing + - invoice + - subscription + +auth: + type: api_key + scopes: [] + +actions: + - name: create_customer + category: action + description: "Create a new Stripe customer" + required_fields: + - { name: email, type: string, description: "Customer email address" } + optional_fields: + - { name: name, type: string, description: "Customer full name" } + + - name: create_charge + category: action + description: "Create a payment charge" + required_fields: + - { name: amount, type: number, description: "Amount in cents" } + - { name: currency, type: string, description: "Three-letter ISO currency code" } + - { name: customer, type: string, description: "Customer ID" } + + - name: list_customers + category: search + description: "List Stripe customers" + required_fields: [] + +triggers: + - name: payment_succeeded + event: payment_intent.succeeded + description: "Fires when a payment succeeds" + payload_fields: + - { name: id, type: string } + - { name: amount, type: number } + - { name: currency, type: string } diff --git a/src/workflows_cdk/templates/library/slack_notifier.yaml b/src/workflows_cdk/templates/library/slack_notifier.yaml new file mode 100644 index 0000000..7ddd279 --- /dev/null +++ b/src/workflows_cdk/templates/library/slack_notifier.yaml @@ -0,0 +1,40 @@ +app_type: slack +app_name: Slack Connector +version: v1 + +keywords: + - slack + - message + - channel + - notify + - notification + - chat + - alert + +auth: + type: oauth2 + scopes: + - "chat:write" + - "channels:read" + +actions: + - name: send_message + category: action + description: "Send a message to a Slack channel or DM" + required_fields: + - { name: channel, type: string, description: "Channel name or ID" } + - { name: text, type: string, description: "Message body" } + + - name: list_channels + category: search + description: "List all channels in the workspace" + required_fields: [] + +triggers: + - name: new_message + event: message.created + description: "Fires when a new message is posted" + payload_fields: + - { name: channel, type: string } + - { name: text, type: string } + - { name: user, type: string } diff --git a/src/workflows_cdk/templates/matcher.py b/src/workflows_cdk/templates/matcher.py new file mode 100644 index 0000000..b6000df --- /dev/null +++ b/src/workflows_cdk/templates/matcher.py @@ -0,0 +1,119 @@ +""" +Template matcher -- resolves a user description to a pre-built ConnectorSpec +without calling any LLM. Used as the ``--no-ai`` fast path and as a fallback +when the OpenAI key is not configured. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Optional + +import yaml + +from ..spec.connector_spec import ( + ActionSpec, + AuthSpec, + ConnectorSpec, + FieldSpec, + TriggerSpec, +) + +logger = logging.getLogger(__name__) + +_LIBRARY_DIR = Path(__file__).parent / "library" + + +class _Template: + """Internal wrapper around a single template YAML file.""" + + def __init__(self, data: dict, path: Path) -> None: + self.data = data + self.path = path + self.keywords: list[str] = [ + k.lower() for k in data.get("keywords", []) + ] + self.slug: str = data.get("app_type", path.stem) + + def score(self, query_tokens: set[str]) -> int: + corpus = set(self.keywords) | {self.slug} + return len(query_tokens & corpus) + + def to_connector_spec(self) -> ConnectorSpec: + d = self.data + auth_raw = d.get("auth", {}) + return ConnectorSpec( + app_type=d["app_type"], + app_name=d["app_name"], + version=d.get("version", "v1"), + actions=[_parse_action(a) for a in d.get("actions", [])], + triggers=[_parse_trigger(t) for t in d.get("triggers", [])], + auth=AuthSpec( + type=auth_raw.get("type", "oauth2"), + scopes=auth_raw.get("scopes", []), + ), + confidence=1.0, + ) + + +def _parse_field(raw: dict) -> FieldSpec: + return FieldSpec( + name=raw["name"], + type=raw.get("type", "string"), + description=raw.get("description", ""), + ) + + +def _parse_action(raw: dict) -> ActionSpec: + return ActionSpec( + name=raw["name"], + category=raw.get("category", "action"), + description=raw.get("description", ""), + required_fields=[_parse_field(f) for f in raw.get("required_fields", [])], + optional_fields=[_parse_field(f) for f in raw.get("optional_fields", [])], + ) + + +def _parse_trigger(raw: dict) -> TriggerSpec: + return TriggerSpec( + name=raw["name"], + event=raw.get("event", ""), + description=raw.get("description", ""), + payload_fields=[_parse_field(f) for f in raw.get("payload_fields", [])], + ) + + +def _load_templates() -> list[_Template]: + templates: list[_Template] = [] + if not _LIBRARY_DIR.is_dir(): + return templates + for path in sorted(_LIBRARY_DIR.glob("*.yaml")): + try: + with open(path) as fh: + data = yaml.safe_load(fh) or {} + templates.append(_Template(data, path)) + except Exception: + logger.warning("Failed to load template %s", path, exc_info=True) + return templates + + +def match_template(description: str) -> Optional[ConnectorSpec]: + """Return the best-matching template as a ``ConnectorSpec``, or *None*.""" + tokens = set(description.lower().split()) + templates = _load_templates() + if not templates: + return None + + best: Optional[_Template] = None + best_score = 0 + for tpl in templates: + s = tpl.score(tokens) + if s > best_score: + best_score = s + best = tpl + + if best is None or best_score == 0: + return None + + return best.to_connector_spec() From 93fb5663a6c62e913c167e2feb4493e1d360b9d4 Mon Sep 17 00:00:00 2001 From: Jason Benichou Date: Mon, 6 Apr 2026 13:55:46 +0200 Subject: [PATCH 2/2] feat(cli): AI-assisted connectors, guided UX, ngrok/Docker hardening - Interactive workflows menu (create, update, validate, run, expose, docs, setup) - Connector picker from nearby projects; clearer prompts; menu loops with pauses - Post-generation next steps: run run_dev, auto ngrok with optional install, open docs - Track CLI-started ngrok/background connector; prompt to stop on exit and after run - Planner: Haiku default, connection-error retry, prompts without registry jargon - Clarifier: one question at a time; validator/spec/template tweaks for modules - dynamic_routing: stable project root (env, Flask root_path, cwd fallback) - Compiler: template alignment (.gitignore, .env, README, entrypoint, no capability.yaml) - Docker dev: WORKFLOWS_PROJECT_ROOT, gunicorn chdir, entrypoint cd, run_dev docker rm - README: usage-focused quick start, menus, commands, ngrok/Studio tips Made-with: Cursor --- README.md | 357 +++--- src/workflows_cdk/ai/clarifier.py | 104 +- src/workflows_cdk/ai/planner.py | 61 +- src/workflows_cdk/ai/prompts.py | 30 +- src/workflows_cdk/ai/validator.py | 63 +- src/workflows_cdk/cli/main.py | 1276 ++++++++++++++++++++- src/workflows_cdk/core/dynamic_routing.py | 111 +- src/workflows_cdk/spec/compiler.py | 375 ++++-- src/workflows_cdk/spec/connector_spec.py | 4 + src/workflows_cdk/templates/matcher.py | 22 +- 10 files changed, 1887 insertions(+), 516 deletions(-) diff --git a/README.md b/README.md index 947f249..3a59e0a 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,34 @@ # Workflows CDK -A powerful CDK (Connector Development Kit) for building Stacksync Workflows Connectors with Python and Flask. +A CDK (Connector Development Kit) for building **Stacksync Workflows** connectors with **Python** and **Flask**, plus a **`workflows`** CLI for AI-assisted scaffolding and day-to-day tasks. ## Features -- 🚀 Automatic route discovery and registration file based (like in Next.js!) -- 🔒 Built-in error handling and Sentry integration -- 📦 Standardized request/response handling -- 🛠️ Error management with standardized error handling -- 🔄 Environment-aware configuration -- 🤖 AI-powered connector scaffolding via CLI (OpenAI or Anthropic) -- 📚 Built-in capability registry for Slack, Stripe, HubSpot, Salesforce, OpenAI, PostgreSQL +**Runtime (library)** + +- 🚀 File-based route discovery under `src/modules/` (like the [app connector template](https://github.com/stacksyncdata/workflows-app-connector-template)) +- 🔒 Built-in error handling and optional Sentry integration +- 📦 Standardized `Request` / `Response` handling +- 🔄 Environment-aware configuration via `app_config.yaml` + +**CLI (`workflows` command)** + +- 🤖 Generate connectors or modules from a short natural-language description (`workflows create`) +- 📋 Interactive **main menu** when you run `workflows` with no arguments +- ➕ **Update** an existing project with new modules (`create --module-only`, also from the menu) +- ✅ **Validate** a connector folder (`validate`, with path prompt in the menu) +- 🖥️ **Run locally** from the menu (`run_dev.sh` or `python main.py`) +- 🌐 **Expose with ngrok** from the menu (starts the app on the configured port when needed, then tunnel) +- 📍 **Region** from the project `.env` shown in the UI (keep Studio and workflows in the same region) +- 📖 **`workflows guide`** — short help for run, ngrok, register, test +- 🔎 **`workflows list`** / **`workflows inspect`** — browse built-in capability manifests + +## Prerequisites + +- **Python 3.10+** +- **Docker** — recommended for `./run_dev.sh` (same flow as the official template) +- **ngrok** — optional; install from [ngrok.com](https://ngrok.com/download) if you use menu option **Expose with ngrok** +- **Anthropic or OpenAI API key** — optional; required for AI generation (otherwise use `--no-ai` or template matching) ## Installation @@ -18,141 +36,155 @@ A powerful CDK (Connector Development Kit) for building Stacksync Workflows Conn pip install workflows-cdk ``` -## Configuration +## Quick start -On first run, the CLI will ask you to pick a provider and paste your API key: +### 1. Create a connector +```bash +workflows create "Klaviyo connector with API key" ``` -$ workflows create "Slack connector" -╭─ Welcome to Workflows CDK ──────────────────────────╮ -│ No API key found. Let's set one up. │ -│ You can reconfigure anytime with workflows setup. │ -╰──────────────────────────────────────────────────────╯ +Confirm at the preview, choose overwrite/version if prompted, then open the generated folder (name is derived from your description, e.g. `klaviyo-connector`). + +If you have no API key, the CLI can run **`workflows setup`** or fall back to template matching. -Which AI provider? [anthropic/openai] (anthropic): -Paste your ANTHROPIC_API_KEY (sk-ant-...): sk-ant-xxxxx +### 2. Run it locally -Done! ANTHROPIC_API_KEY saved to .env +```bash +cd klaviyo-connector # use your generated folder name +pip install -r requirements.txt +./run_dev.sh ``` -You can also configure manually: +Default URL: `http://localhost:2003` (change port in `app_config.yaml` if needed). -```bash -# Interactive setup / reconfigure -workflows setup +### 3. Expose and register in Stacksync -# Or set keys directly via .env -echo 'ANTHROPIC_API_KEY=sk-ant-your-key-here' > .env +- Use **Expose with ngrok** from the post-create menu, or run: `ngrok http 2003` (use your real port). +- Copy the **HTTPS** URL into **Stacksync Developer Studio**. +- Use the **same `REGION`** in Studio and in workflows as in the project `.env` (e.g. `REGION=besg`). -# Or export in your shell -export ANTHROPIC_API_KEY=sk-ant-your-key-here -``` +If Studio says the URL already exists, start a **new** ngrok session for a new URL, or remove/edit the existing private app. -Both `.env` and environment variables work. Environment variables take priority. +### 4. (Optional) Open the full menu anytime -| Variable | Required | Description | -|---|---|---| -| `ANTHROPIC_API_KEY` | One of these | Anthropic API key — uses Claude Sonnet 4.6 by default | -| `OPENAI_API_KEY` | One of these | OpenAI API key — uses GPT-5 Nano by default | -| `WORKFLOWS_AI_PROVIDER` | No | Force a provider when both keys are set: `anthropic` or `openai` | -| `WORKFLOWS_AI_MODEL` | No | Override the default model (default: `claude-sonnet-4-6` or `gpt-5-nano`) | -| `ENVIRONMENT` | No | Runtime environment: `dev`, `stage`, or `prod` | +```bash +workflows +``` -No API key? Use `--no-ai` to create connectors via built-in template matching instead. +Pick **1–8** at the prompt (enter a **path** only when the menu asks for it). -## Quick Start +--- -### AI-powered (recommended) +## Interactive main menu -Create a connector in one command: +Run: ```bash -workflows create "Slack connector: send messages, list channels" +workflows ``` -A complete project appears in `./slack-connector/` with working Flask routes, schemas, and config -- ready to run with `python main.py`. +| # | Option | What to do | +|---|--------|------------| +| **1** | Create a connector | Enter a description; same flow as `workflows create "…"` | +| **2** | Update a connector | Enter project path, then what to add → adds modules only | +| **3** | Validate a project | Enter connector root path | +| **4** | Run connector locally | Enter path → `run_dev.sh` or `python main.py` | +| **5** | Expose with ngrok | Enter path → app started if needed, then ngrok; copy the HTTPS URL | +| **6** | View documentation | Opens the custom connector guide in the browser | +| **7** | Setup AI provider | Configure Anthropic / OpenAI key (saved to `.env` in the current directory) | +| **8** | Exit | Leave the menu | -More examples: +After each action, press **Enter** when asked to return to the menu. -```bash -# Preview without writing files -workflows create "HubSpot CRM: create contacts, search, manage deals" --dry-run +--- -# Template matching only (no API key needed) -workflows create "Slack: send messages, list channels" --no-ai +## After `workflows create` — next steps menu -# Specify output directory -workflows create "PostgreSQL: query, insert, search rows" -o ./connectors +| # | Option | What to do | +|---|--------|------------| +| **1** | Run the connector | `run_dev.sh` or `python main.py` in the new project | +| **2** | Expose with ngrok | Same as main menu **5**, using the new project automatically | +| **3** | Open documentation | Opens Stacksync developer docs | +| **4** | Exit | Close this menu | -# List available capabilities -workflows list +The panel shows your **Stacksync region** from the generated `.env`. -# Inspect a specific app -workflows inspect slack -``` +--- -### Manual setup +## Command reference -1. Create a new project directory: +| Command | Use it to… | +|---------|------------| +| `workflows` | Open the interactive main menu | +| `workflows create ""` | Generate a new connector (or use `-o` for parent directory) | +| `workflows create --dry-run` | Preview without writing files | +| `workflows create --no-ai` | Template matching only (no LLM) | +| `workflows create --module-only` | Add modules into an existing connector directory (`-o` = that directory) | +| `workflows validate` | Validate the current directory | +| `workflows validate -p ` | Validate a specific connector root | +| `workflows setup` | Configure AI provider and API key | +| `workflows list` | List built-in app slugs in the registry | +| `workflows inspect ` | Show actions/triggers for one app | +| `workflows guide run` | Print how to run locally | +| `workflows guide ngrok` | Print how to expose with ngrok | +| `workflows guide register` | Print how to register in Developer Studio | +| `workflows guide test` | Print how to test in a workflow | -```bash -mkdir my-workflow-connector -cd my-workflow-connector -``` +--- + +## AI configuration -2. Install the required dependencies: +Set keys in the environment or in a `.env` file in the directory where you run the CLI: + +| Variable | Purpose | +|----------|---------| +| `ANTHROPIC_API_KEY` | Claude (default when set) | +| `OPENAI_API_KEY` | OpenAI | +| `WORKFLOWS_AI_PROVIDER` | `anthropic` or `openai` if both keys are set | +| `WORKFLOWS_AI_MODEL` | Override the default model | + +Or run: ```bash -pip install workflows-cdk flask pyyaml +workflows setup ``` -3. Create the basic project structure: +--- + +## Generated project layout + +Generated projects follow the [app connector template](https://github.com/stacksyncdata/workflows-app-connector-template) layout: ``` -my-workflow-connector/ +my-connector/ ├── main.py ├── app_config.yaml ├── requirements.txt -└── routes/ - └── hello/ +├── README.md +├── .env +├── .gitignore +├── Dockerfile +├── entrypoint.sh +├── run_dev.sh +├── run_dev.bat +├── config/ +│ ├── Dockerfile.dev +│ ├── entrypoint.sh +│ └── gunicorn_config.py +└── src/modules/ + └── / └── v1/ - └── route.py -``` - -4. Set up your `app_config.yaml`: - -```yaml -app_settings: - app_type: "example" - app_name: "My Workflow Connector" - app_description: "A simple workflow connector" - sentry_dsn: "your-sentry-dsn" # Optional - cors_origins: ["*"] - routes_directory: "routes" - debug: true - host: "0.0.0.0" - port: 2005 + ├── route.py + ├── schema.json + └── module_config.yaml ``` -5. Create your `main.py`: - -```python -from flask import Flask -from workflows_cdk import Router +--- -# Create Flask app -app = Flask("my-workflow-connector") +## Writing routes -# Initialize router with configuration -router = Router(app) - -# Run the app -if __name__ == "__main__": - router.run_app(app) -``` - -6. Create your first route in `routes/send_message/v1/route.py`: +Each module’s `route.py` uses the CDK helpers. Minimal pattern: ```python from workflows_cdk import Request, Response, ManagedError @@ -160,127 +192,26 @@ from main import router @router.route("/execute", methods=["POST"]) def execute(): - """Execute the send message action.""" - request = Request(flask_request) - data = request.data - - name = data.get("name", "World") - return Response.success(data={ - "message": f"Hello, {name}!" - }) + req = Request(flask_request) + data = req.data + credentials = req.credentials + # … call your API … + return Response(data={"result": "ok"}) ``` -## Core Components - -### Router - -The `Router` class is the heart of the CDK, providing: - -- Automatic route discovery based on file system structure -- Built-in error handling and Sentry integration -- CORS configuration -- Health check endpoints -- API documentation - -### Request - -The `Request` class wraps Flask's request object, providing: - -- Easy access to request data and credentials -- Automatic JSON parsing -- Type-safe access to common properties - -### Response - -The `Response` class provides standardized response formatting: - -- Success responses with optional metadata -- Error responses with appropriate status codes -- Environment-aware error details -- Sentry integration - -### ManagedError - -The `ManagedError` class provides structured error handling: +For validation, not-found, and auth errors, use `ManagedError` helpers as in the [package examples](https://github.com/stacksyncdata/workflows-cdk/blob/prod/README.md#error-handling). -- Type-safe error creation -- Automatic Sentry reporting -- Environment-aware error details -- Common error types (validation, not found, unauthorized, etc.) +--- -## Project Structure +## Documentation & resources -Recommended project structure for a workflow connector: +- [Build a custom connector](https://docs.stacksync.com/workflow-automation/developers/build-a-custom-connector) +- [Workflows app connector](https://docs.stacksync.com/workflows/app-connector) +- [Official connector template](https://github.com/stacksyncdata/workflows-app-connector-template) +- [Stacksync docs](https://docs.stacksync.com/) -``` -my-workflow-connector/ -├── main.py # Application entry point -├── app_config.yaml # Application configuration -├── requirements.txt # Python dependencies -├── README.md # Project documentation -├── Dockerfile # Container configuration -├── .env # Environment variables -└── routes/ # Route modules - └── action_name/ # Group routes by action - ├── v1/ # Version 1 of the action - │ ├── route.py # Route implementation - │ └── schema.json # JSON Schema for validation - └── v2/ # Version 2 of the action - ├── route.py - └── schema.json -``` - -## Error Handling - -The CDK provides comprehensive error handling: - -```python -from workflows_cdk import ManagedError - -# Validation error -raise ManagedError.validation_error( - error="Invalid input", - data={"field": "email"} -) - -# Not found error -raise ManagedError.not_found( - resource="User", - identifier="123" -) - -# Authorization error -raise ManagedError.unauthorized( - message="Invalid API key" -) - -# Server error -raise ManagedError.server_error( - error="Database connection failed" -) -``` - -## Response Formatting - -Standardized response formatting: - -```python -from workflows_cdk import Response - -# Success response -return Response.success( - data={"result": "ok"}, - message="Operation completed", - metadata={"timestamp": "2024-02-17"} -) - -# Error response -return Response.error( - error="Something went wrong", - status_code=400 -) -``` +--- ## License -This project is licensed under the Stacksync Connector License (SCL) v1.0. +This project is licensed under the **Stacksync Connector License (SCL) v1.0**. diff --git a/src/workflows_cdk/ai/clarifier.py b/src/workflows_cdk/ai/clarifier.py index d44374d..b3137fb 100644 --- a/src/workflows_cdk/ai/clarifier.py +++ b/src/workflows_cdk/ai/clarifier.py @@ -1,16 +1,15 @@ """ One-shot clarification engine. -Takes the ``ambiguities`` list from a ConnectorSpec, renders them as a single -compact terminal prompt, collects the user's answers, and returns a plain-text +Takes the ``ambiguities`` list from a ConnectorSpec, renders them as +individual prompts, collects the user's answers, and returns a plain-text string that can be fed back into the LLM refinement prompt. """ from __future__ import annotations from rich.console import Console -from rich.panel import Panel -from rich.text import Text +from rich.prompt import Prompt from ..spec.connector_spec import AmbiguitySpec, ConnectorSpec @@ -18,7 +17,7 @@ def render_clarification(spec: ConnectorSpec) -> str: - """Display ambiguities and collect answers interactively. + """Display ambiguities one at a time and collect answers interactively. Returns a plain-text summary of the user's choices, ready to be passed to ``Planner.refine()``. @@ -27,77 +26,38 @@ def render_clarification(spec: ConnectorSpec) -> str: if not ambiguities: return "" - action_count = len(spec.actions) - trigger_count = len(spec.triggers) - - parts = [] - if action_count: - parts.append(f"{action_count} action{'s' if action_count != 1 else ''}") - if trigger_count: - parts.append(f"{trigger_count} trigger{'s' if trigger_count != 1 else ''}") - summary = " + ".join(parts) if parts else "a connector" - - header = ( - f"I can build this {spec.app_name} with {summary}.\n" - f"I need {len(ambiguities)} detail{'s' if len(ambiguities) != 1 else ''} " - f"before scaffolding:" - ) - - body_lines: list[str] = [] - for idx, amb in enumerate(ambiguities, 1): - options_str = _format_options(amb) - body_lines.append(f" {idx}. {amb.question}: {options_str}") - - body = "\n".join(body_lines) - - console.print() - console.print(Panel( - f"{header}\n\n{body}", - title="[bold]Clarification needed[/bold]", - border_style="yellow", - )) + ambiguities = ambiguities[:3] + count = len(ambiguities) console.print( - "\n[dim]Press Enter to accept defaults (shown in brackets).[/dim]" + f"\n[bold]I need {count} detail{'s' if count != 1 else ''} " + f"before generating:[/bold]" ) - raw = console.input("[bold]Your answers:[/bold] ").strip() - - if not raw: - return _defaults_summary(ambiguities) - return _merge_answers(ambiguities, raw) - - -def _format_options(amb: AmbiguitySpec) -> str: - if not amb.options: - return f"[{amb.default or '?'}]" - - parts: list[str] = [] - for opt in amb.options: - if opt == amb.default: - parts.append(f"[{opt}]") + answers: list[str] = [] + for idx, amb in enumerate(ambiguities, 1): + console.print(f"\n[bold cyan]({idx}/{count})[/bold cyan] {amb.question}") + + options = amb.options or [] + default = amb.default or (options[0] if options else None) + + if options: + for i, opt in enumerate(options, 1): + marker = "[bold green]*[/bold green] " if opt == default else " " + console.print(f" {marker}{i}. {opt}") + console.print(f" [dim]Press Enter for default: {default}[/dim]") + + raw = console.input("[bold]> [/bold]").strip() + if not raw: + chosen = default + elif raw.isdigit() and 1 <= int(raw) <= len(options): + chosen = options[int(raw) - 1] + else: + chosen = raw else: - parts.append(opt) - return " / ".join(parts) + raw = console.input(f"[bold]> [/bold][dim]({default})[/dim] ").strip() + chosen = raw if raw else default + answers.append(f"{amb.question}: {chosen}") -def _defaults_summary(ambiguities: list[AmbiguitySpec]) -> str: - lines: list[str] = [] - for amb in ambiguities: - default = amb.default or (amb.options[0] if amb.options else "unspecified") - lines.append(f"{amb.question}: {default}") - return "\n".join(lines) - - -def _merge_answers(ambiguities: list[AmbiguitySpec], raw: str) -> str: - """Best-effort parse of comma-separated or numbered answers.""" - tokens = [t.strip() for t in raw.replace(";", ",").split(",")] - - lines: list[str] = [] - for idx, amb in enumerate(ambiguities): - if idx < len(tokens) and tokens[idx]: - lines.append(f"{amb.question}: {tokens[idx]}") - else: - default = amb.default or (amb.options[0] if amb.options else "unspecified") - lines.append(f"{amb.question}: {default}") - return "\n".join(lines) + return "\n".join(answers) diff --git a/src/workflows_cdk/ai/planner.py b/src/workflows_cdk/ai/planner.py index 9548af0..f400f98 100644 --- a/src/workflows_cdk/ai/planner.py +++ b/src/workflows_cdk/ai/planner.py @@ -23,6 +23,7 @@ import json import logging import os +import time from typing import Any, Literal from ..registry.registry import CapabilityRegistry @@ -127,7 +128,11 @@ class Planner: def __init__(self, registry: CapabilityRegistry) -> None: self.registry = registry - def plan(self, description: str) -> ConnectorSpec: + def build_prompt(self, description: str) -> tuple[str, str]: + """Phase 1: parse intent and build the system prompt (instant). + + Returns (system_prompt, user_message). + """ intent = parse_intent(description, self.registry) if intent.detected_slugs: @@ -143,10 +148,17 @@ def plan(self, description: str) -> ConnectorSpec: system = PLANNER_SYSTEM_PROMPT.format( capabilities_json=capabilities_json, ) + return system, description - raw = _call_llm(system=system, user=description) + def call_llm(self, system: str, user: str) -> ConnectorSpec: + """Phase 2: call the LLM and parse the spec (slow).""" + raw = _call_llm(system=system, user=user) return _parse_spec(raw) + def plan(self, description: str) -> ConnectorSpec: + system, user = self.build_prompt(description) + return self.call_llm(system, user) + def refine(self, draft: ConnectorSpec, user_answers: str) -> ConnectorSpec: system = REFINEMENT_PROMPT.format( draft_spec_json=draft.model_dump_json(indent=2), @@ -184,23 +196,29 @@ def _call_anthropic(*, system: str, user: str) -> str: if not api_key: raise PlannerError("ANTHROPIC_API_KEY environment variable is not set.") - client = anthropic.Anthropic(api_key=api_key) + client = anthropic.Anthropic(api_key=api_key, timeout=120.0) model = os.environ.get("WORKFLOWS_AI_MODEL", DEFAULT_ANTHROPIC_MODEL) - schema = _build_json_schema() - resp = client.messages.create( - model=model, - max_tokens=4096, - system=system, - messages=[{"role": "user", "content": user}], - output_config={ - "format": { - "type": "json_schema", - "schema": schema, - }, - }, - temperature=0.2, - ) + last_exc: Exception | None = None + for attempt in range(2): + try: + resp = client.messages.create( + model=model, + max_tokens=8192, + system=system, + messages=[{"role": "user", "content": user}], + temperature=0.2, + ) + break + except Exception as exc: + last_exc = exc + if attempt == 0 and "connection" in str(exc).lower(): + logger.debug("Anthropic connection error, retrying in 2s…") + time.sleep(2) + continue + raise PlannerError(f"Anthropic API error: {exc}") from exc + else: + raise PlannerError(f"Anthropic API error: {last_exc}") from last_exc text_blocks = [b.text for b in resp.content if b.type == "text"] if not text_blocks: @@ -247,10 +265,13 @@ def _call_openai(*, system: str, user: str) -> str: temperature=0.2, ) return resp.output_text - except Exception: - logger.debug("Responses API unavailable, falling back to chat completions") + except Exception as exc: + logger.debug("Responses API unavailable (%s), falling back to chat completions", exc) - return _chat_completions_fallback(client, model, system, user, schema) + try: + return _chat_completions_fallback(client, model, system, user, schema) + except Exception as exc: + raise PlannerError(f"OpenAI API error: {exc}") from exc def _chat_completions_fallback( diff --git a/src/workflows_cdk/ai/prompts.py b/src/workflows_cdk/ai/prompts.py index 58b7133..eedc614 100644 --- a/src/workflows_cdk/ai/prompts.py +++ b/src/workflows_cdk/ai/prompts.py @@ -35,8 +35,8 @@ "name": "string — snake_case action id", "category": "action | search | transform", "description": "string — one-sentence description", - "required_fields": [{{ "name": "string", "type": "string|number|boolean|object|array", "description": "string" }}], - "optional_fields": [{{ "name": "string", "type": "string|number|boolean|object|array", "description": "string" }}], + "required_fields": [{{ "name": "string", "type": "string|number|boolean|object|array", "description": "string", "widget": "string (optional)", "choices": [{{}}] (optional), "depends_on": "string (optional)", "dynamic_content": false }}], + "optional_fields": [{{ "name": "string", "type": "string|number|boolean|object|array", "description": "string", "widget": "string (optional)", "choices": [{{}}] (optional), "depends_on": "string (optional)", "dynamic_content": false }}], "implementation": "string — Python code (see )" }} ], @@ -84,6 +84,26 @@ the implementation code MUST exist in required_fields, optional_fields, or payload_fields. If the implementation reads a field, add it to the field list. Conversely, every field in the list should be used in the implementation. + +EXTENDED FIELD ATTRIBUTES (optional, use when appropriate): + - "widget": Override the default UI widget. Common values: "input", "textarea", + "checkbox", "SelectWidget", "password". Default is inferred from type. + - "choices": Array of {{id, label}} objects for static dropdown options. + Use when the field is type "object" and options are known at generation time. + Example: [{{"id": "twitter", "label": "Twitter"}}, {{"id": "linkedin", "label": "LinkedIn"}}] + - "dynamic_content": Set to true when the field's dropdown choices must be + loaded at runtime via the /content endpoint (e.g. fetched from an external API). + - "depends_on": The id of another field that this field's value depends on. + When set, the Stacksync UI will reload either /content or /schema when the + depended-on field changes. Only set this when there is a real dependency. + +ENDPOINT DECISION RULES: + - /execute is ALWAYS generated. + - /content is generated ONLY if at least one field has dynamic_content=true. + - /schema is generated ONLY if at least one field has depends_on set AND + dynamic_content is NOT true for that field (schema reload vs content reload). + - Do NOT set dynamic_content or depends_on unless the description clearly implies + runtime-dependent choices or form structure changes. @@ -137,11 +157,13 @@ - NEVER ask about things you can infer from context or from the capability manifest defaults. - If the user mentions an app that is NOT in , set - confidence to 0.5, include all actions you can reasonably infer, and add one - ambiguity noting the app is not in the built-in registry. + confidence to 0.5 and include all actions you can reasonably infer from + the app's public API. Do NOT mention the registry in clarification questions. +- Generate at most 3 actions + 1 trigger unless the user explicitly asks for + more. Pick the most useful operations for the requested app. - Map actions ONLY to capabilities listed in when the app is known. - Default auth type to what the capability manifest specifies. diff --git a/src/workflows_cdk/ai/validator.py b/src/workflows_cdk/ai/validator.py index 34f727b..504dcef 100644 --- a/src/workflows_cdk/ai/validator.py +++ b/src/workflows_cdk/ai/validator.py @@ -41,6 +41,7 @@ def validate_spec(spec: ConnectorSpec, registry: CapabilityRegistry) -> Validati _validate_actions(spec, registry, result) _validate_triggers(spec, registry, result) _validate_no_route_collisions(spec, result) + _validate_stacksync_contracts(spec, result) return result @@ -78,19 +79,7 @@ def _validate_actions( f"unknown type '{f.type}', defaulting to 'string'" ) - if manifest is not None: - known_actions = manifest.action_names() - if action.name not in known_actions: - result.warnings.append( - f"Action '{action.name}' is not in the {spec.app_type} " - f"capability manifest (known: {', '.join(known_actions)})" - ) - - if manifest is None and spec.app_type: - result.warnings.append( - f"App '{spec.app_type}' is not in the built-in registry. " - f"The generated connector will work but fields/auth are unverified." - ) + # Registry hints help the LLM but are not a gate for validation. def _validate_triggers( @@ -106,13 +95,7 @@ def _validate_triggers( result.errors.append(f"Duplicate trigger name: '{trigger.name}'") seen_names.add(trigger.name) - if manifest is not None: - known_triggers = manifest.trigger_names() - if trigger.name not in known_triggers: - result.warnings.append( - f"Trigger '{trigger.name}' is not in the {spec.app_type} " - f"capability manifest (known: {', '.join(known_triggers)})" - ) + # Registry hints help the LLM but are not a gate for validation. def _validate_no_route_collisions( @@ -131,3 +114,43 @@ def _validate_no_route_collisions( if path in paths: result.errors.append(f"Route collision: {path}") paths.add(path) + + +def _validate_stacksync_contracts( + spec: ConnectorSpec, + result: ValidationResult, +) -> None: + """Stacksync-specific checks that go beyond generic type/name validation.""" + if spec.auth.type in ("oauth2", "api_key") and not spec.app_type: + result.errors.append( + "Auth requires a connection but app_type is empty " + "(needed for allowed_app_types in schema)" + ) + + for action in spec.actions: + all_fields = list(action.required_fields) + list(action.optional_fields) + _check_field_contracts(action.name, all_fields, result) + + for trigger in spec.triggers: + _check_field_contracts(trigger.name, list(trigger.payload_fields), result) + + +def _check_field_contracts( + module_name: str, + fields: list, + result: ValidationResult, +) -> None: + all_ids = {f.name for f in fields} + + for f in fields: + if f.depends_on and f.depends_on not in all_ids: + result.errors.append( + f"Module '{module_name}', field '{f.name}': " + f"depends_on='{f.depends_on}' references non-existent field" + ) + + if f.type == "object" and not f.choices and not f.dynamic_content: + result.warnings.append( + f"Module '{module_name}', field '{f.name}': " + f"type 'object' typically needs choices or dynamic_content" + ) diff --git a/src/workflows_cdk/cli/main.py b/src/workflows_cdk/cli/main.py index c7360d2..927a989 100644 --- a/src/workflows_cdk/cli/main.py +++ b/src/workflows_cdk/cli/main.py @@ -1,5 +1,5 @@ """ -CLI entry point: ``workflows create "Slack connector: send messages, list channels"`` +CLI entry point for the Stacksync Workflows CDK. Registered as a console_script in setup.py so ``pip install workflows-cdk`` makes the ``workflows`` command available globally. @@ -7,27 +7,75 @@ from __future__ import annotations +import json import os +import re +import shlex +import shutil +import signal +import socket +import subprocess +import sys +import time +import urllib.error +import urllib.request +import webbrowser from pathlib import Path +from typing import Any, Optional import click from dotenv import load_dotenv from rich.console import Console from rich.panel import Panel -from rich.prompt import Prompt +from rich.prompt import Confirm, Prompt from rich.tree import Tree -load_dotenv() - from ..ai.validator import validate_spec from ..registry.registry import CapabilityRegistry -from ..spec.compiler import compile_connector, preview_tree +from ..spec.compiler import ( + MODULES_DIR, + compile_connector, + detect_port, + needs_content_endpoint, + needs_schema_endpoint, +) from ..spec.connector_spec import ConnectorSpec from ..templates.matcher import match_template console = Console() -ENV_FILE = Path.cwd() / ".env" +_AUTH_DISPLAY = { + "oauth2": "OAuth2 via managed connection", + "api_key": "API key via connection field", + "basic": "Basic auth", + "none": "No auth", +} + +_BANNER = ( + "[bold cyan]" + " ____ _ _\n" + "/ ___|| |_ __ _ ___| | _____ _ _ _ __ ___\n" + "\\___ \\| __/ _` |/ __| |/ / __| | | | '_ \\ / __|\n" + " ___) | || (_| | (__| <\\__ \\ |_| | | | | (__\n" + "|____/ \\__\\__,_|\\___|_|\\_\\___/\\__, |_| |_|\\___|\n" + " |___/" + "[/bold cyan]" +) + + +def _print_banner() -> None: + console.print(_BANNER) + console.print("[bold green]Workflows CDK[/bold green]") + console.print("[dim]https://docs.stacksync.com/workflows/app-connector[/dim]") + console.print() + + +def _env_file() -> Path: + return Path.cwd() / ".env" + + +def _ensure_dotenv() -> None: + load_dotenv(_env_file()) def _has_api_key() -> bool: @@ -68,21 +116,21 @@ def _run_setup() -> bool: return False os.environ[key_name] = api_key - _save_to_env(key_name, api_key) - console.print(f"\n[green bold]Done![/green bold] {key_name} saved to [cyan]{ENV_FILE}[/cyan]") + console.print(f"\n[green bold]Done![/green bold] {key_name} saved to [cyan]{_env_file()}[/cyan]") console.print("[dim]You can also export it in your shell or edit .env directly.[/dim]\n") return True def _save_to_env(key_name: str, value: str) -> None: """Append or update a key in the .env file.""" + env = _env_file() lines: list[str] = [] found = False - if ENV_FILE.exists(): - for line in ENV_FILE.read_text().splitlines(): + if env.exists(): + for line in env.read_text().splitlines(): stripped = line.lstrip("# ").split("=", 1)[0].strip() if stripped == key_name: lines.append(f"{key_name}={value}") @@ -93,17 +141,705 @@ def _save_to_env(key_name: str, value: str) -> None: if not found: lines.append(f"{key_name}={value}") - ENV_FILE.write_text("\n".join(lines) + "\n") + env.write_text("\n".join(lines) + "\n") # --------------------------------------------------------------------------- # CLI group # --------------------------------------------------------------------------- -@click.group() +_DOCS_URL = "https://docs.stacksync.com/workflow-automation/developers/build-a-custom-connector" +_NGROK_DOWNLOAD_URL = "https://ngrok.com/download" + +# Processes started by this CLI session (Expose with ngrok / background connector). +_managed_ngrok_proc: subprocess.Popen | None = None +_managed_connector_launcher: subprocess.Popen | None = None +_managed_connector_docker_script: bool = False +_managed_connector_project_dir: Path | None = None + + +def _docker_app_name(project_dir: Path) -> str: + """Container name used by generated run_dev.sh (workflows-app-).""" + return f"workflows-app-{project_dir.resolve().name}" + + +def _terminate_process_group(proc: subprocess.Popen, *, timeout: float = 8.0) -> None: + """Stop a process started with start_new_session=True (best-effort).""" + if proc.poll() is not None: + return + if sys.platform == "win32": + proc.terminate() + else: + try: + os.killpg(os.getpgid(proc.pid), signal.SIGTERM) + except (ProcessLookupError, OSError): + proc.terminate() + try: + proc.wait(timeout=timeout) + except subprocess.TimeoutExpired: + proc.kill() + + +def _dispose_managed_ngrok_silent() -> None: + """Stop ngrok started by this CLI, if still running.""" + global _managed_ngrok_proc + if _managed_ngrok_proc is not None: + _terminate_process_group(_managed_ngrok_proc) + _managed_ngrok_proc = None + + +def _dispose_managed_connector_silent() -> None: + """Stop background connector launcher and Docker container if we started them.""" + global _managed_connector_launcher, _managed_connector_docker_script + global _managed_connector_project_dir + + if _managed_connector_launcher is not None: + _terminate_process_group(_managed_connector_launcher) + _managed_connector_launcher = None + + if _managed_connector_docker_script and _managed_connector_project_dir is not None: + name = _docker_app_name(_managed_connector_project_dir) + try: + subprocess.run( + ["docker", "rm", "-f", name], + capture_output=True, + timeout=30, + check=False, + ) + except (OSError, subprocess.TimeoutExpired): + pass + + _managed_connector_docker_script = False + _managed_connector_project_dir = None + + +def _managed_ngrok_still_running() -> bool: + global _managed_ngrok_proc + if _managed_ngrok_proc is None: + return False + if _managed_ngrok_proc.poll() is not None: + _managed_ngrok_proc = None + return False + return True + + +def _managed_connector_still_running() -> bool: + global _managed_connector_launcher + if _managed_connector_launcher is None: + return False + if _managed_connector_launcher.poll() is not None: + _managed_connector_launcher = None + return False + return True + + +def _prompt_stop_managed_services_if_any() -> None: + """If this CLI started ngrok or a background connector, offer to stop them.""" + ngrok_live = _managed_ngrok_still_running() + conn_live = _managed_connector_still_running() + + if not ngrok_live and not conn_live: + return + + console.print() + console.rule("[bold cyan]Background services[/bold cyan]", style="cyan") + if ngrok_live: + if Confirm.ask( + "Stop the [bold]ngrok tunnel[/bold] that this CLI started?", + default=True, + ): + _dispose_managed_ngrok_silent() + console.print("[dim]ngrok stopped.[/dim]") + else: + console.print("[dim]Left ngrok running.[/dim]") + if conn_live: + if Confirm.ask( + "Stop the [bold]connector[/bold] this CLI started in the background " + "(Docker or python)?", + default=True, + ): + _dispose_managed_connector_silent() + console.print("[dim]Connector stopped.[/dim]") + else: + console.print("[dim]Left the connector running.[/dim]") + console.print() + + +def _detect_region(project_dir: Path) -> str: + """Read REGION from the connector's .env (matches run_dev.sh / template).""" + env_path = project_dir / ".env" + if env_path.exists(): + for raw in env_path.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + if line.startswith("REGION="): + val = line.split("=", 1)[1].strip().strip('"').strip("'") + return val if val else "besg" + return "besg" + + +def _ngrok_local_api_tunnels() -> dict[str, Any] | None: + """Return ngrok agent local API JSON, or None if unreachable.""" + try: + req = urllib.request.Request( + "http://127.0.0.1:4040/api/tunnels", + headers={"Accept": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=1.5) as resp: + return json.loads(resp.read().decode()) + except (urllib.error.URLError, OSError, json.JSONDecodeError, ValueError): + return None + + +def _port_from_upstream_addr(addr: str) -> int | None: + """Parse local port from ngrok config.addr (e.g. http://127.0.0.1:2003).""" + s = addr.strip() + if not s or s.lower() in ("undefined", "none", "null"): + return None + m = re.search(r":(\d{1,5})\s*$", s) + if m: + p = int(m.group(1)) + return p if 1 <= p <= 65535 else None + if s.isdigit(): + p = int(s) + return p if 1 <= p <= 65535 else None + return None + + +def _tunnel_local_port(tunnel: dict[str, Any]) -> int | None: + """Return the upstream port this tunnel forwards to, or None if unknown / invalid.""" + conf = tunnel.get("config") or {} + addr = conf.get("addr") + if isinstance(addr, dict): + p = addr.get("port") + if p is not None: + try: + pi = int(p) + return pi if 1 <= pi <= 65535 else None + except (TypeError, ValueError): + pass + url = addr.get("URL") or addr.get("url") + if isinstance(url, str): + return _port_from_upstream_addr(url) + return None + if isinstance(addr, str): + return _port_from_upstream_addr(addr) + return None + + +def _tunnel_targets_port(tunnel: dict[str, Any], port: int) -> bool: + return _tunnel_local_port(tunnel) == port + + +def _pick_https_public_url_for_port(tunnels_data: dict[str, Any], port: int) -> str | None: + """Return the public HTTPS URL only for a tunnel whose upstream port matches *port*. + + Does not fall back to unrelated tunnels (avoids false positives when another + broken tunnel exists with undefined upstream). + """ + tunnels = tunnels_data.get("tunnels") or [] + for t in tunnels: + if t.get("proto") == "https" and _tunnel_targets_port(t, port): + url = t.get("public_url") + if url: + return str(url) + for t in tunnels: + if _tunnel_targets_port(t, port): + url = t.get("public_url") + if url: + return str(url) + return None + + +def _port_is_listening(port: int, host: str = "127.0.0.1") -> bool: + """True if something accepts TCP connections on host:port.""" + try: + with socket.create_connection((host, port), timeout=0.75): + return True + except OSError: + return False + + +def _wait_for_port(port: int, *, timeout: float, host: str = "127.0.0.1") -> bool: + """Poll until the port accepts connections or *timeout* seconds elapse.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if _port_is_listening(port, host=host): + return True + time.sleep(0.4) + return False + + +def _start_main_py_background(project_dir: Path, main_py: Path) -> subprocess.Popen | None: + try: + return subprocess.Popen( + [sys.executable, str(main_py)], + cwd=str(project_dir), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + except OSError as exc: + console.print(f"[red]Could not start python main.py: {exc}[/red]") + return None + + +def _start_connector_background( + project_dir: Path, port: int +) -> tuple[bool, subprocess.Popen | None, bool]: + """Start ./run_dev.sh (with a pseudo-TTY when needed) or python main.py in the background. + + Returns: + (success, launcher_popen, docker_via_script) — *docker_via_script* is True when + the launcher is ``script`` wrapping ``run_dev.sh`` (Docker), for cleanup via ``docker rm``. + """ + run_dev = project_dir / "run_dev.sh" + main_py = project_dir / "main.py" + + if run_dev.exists(): + script_bin = shutil.which("script") + if script_bin and sys.platform != "win32": + try: + console.print( + "[dim]Launching [cyan]./run_dev.sh[/cyan] in the background " + "(Docker; first run may build for several minutes)…[/dim]" + ) + if sys.platform == "darwin": + script_argv = [ + script_bin, "-q", "/dev/null", "bash", str(run_dev), + ] + else: + # util-linux `script`: -c runs a command with a pty + inner = f"bash {shlex.quote(str(run_dev.name))}" + script_argv = [ + script_bin, "-q", "-c", inner, "/dev/null", + ] + proc = subprocess.Popen( + script_argv, + cwd=str(project_dir), + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + return True, proc, True + except OSError as exc: + console.print(f"[yellow]Could not start run_dev.sh via script: {exc}[/yellow]") + + if main_py.exists(): + console.print( + "[yellow]Docker [bold]run_dev.sh[/bold] needs a pseudo-TTY for [bold]-it[/bold]. " + "Starting [cyan]python main.py[/cyan] in the background instead.[/yellow]\n" + "[dim]For full Docker dev, run [cyan]./run_dev.sh[/cyan] in a separate terminal.[/dim]" + ) + py = _start_main_py_background(project_dir, main_py) + return py is not None, py, False + + console.print( + "[red]Cannot start the connector: [cyan]script[/cyan] is not available for headless " + "[cyan]./run_dev.sh[/cyan], and [cyan]main.py[/cyan] is missing.[/red]" + ) + return False, None, False + + if main_py.exists(): + console.print("[dim]Starting [cyan]python main.py[/cyan] in the background…[/dim]") + py = _start_main_py_background(project_dir, main_py) + return py is not None, py, False + + console.print( + f"[red]No [cyan]run_dev.sh[/cyan] or [cyan]main.py[/cyan] in {project_dir}[/red]" + ) + return False, None, False + + +def _ensure_connector_running_for_ngrok(project_dir: Path, port: int) -> bool: + """Ensure localhost:*port* is serving the connector before exposing with ngrok.""" + global _managed_connector_launcher, _managed_connector_docker_script + global _managed_connector_project_dir + + if _port_is_listening(port): + console.print( + f"[dim]Connector already listening on [cyan]localhost:{port}[/cyan].[/dim]" + ) + return True + + console.print( + "\n[bold]Starting the connector locally first[/bold] " + "[dim](ngrok needs your app on this port).[/dim]" + ) + ok, launcher, docker_script = _start_connector_background(project_dir, port) + if not ok or launcher is None: + return False + + _managed_connector_launcher = launcher + _managed_connector_docker_script = docker_script + _managed_connector_project_dir = project_dir + + console.print( + f"[dim]Waiting for [cyan]localhost:{port}[/cyan] " + f"(up to ~7 min on first Docker build)…[/dim]" + ) + if not _wait_for_port(port, timeout=420.0): + _dispose_managed_connector_silent() + console.print( + Panel( + f"[red]Nothing accepted connections on port {port} in time.[/red]\n\n" + "Start the connector manually:\n" + f" [cyan]cd {project_dir}[/cyan]\n" + " [cyan]./run_dev.sh[/cyan] [dim]or[/dim] [cyan]python main.py[/cyan]\n\n" + "Then run **Expose with ngrok** again.", + title="[red]Connector did not become ready[/red]", + border_style="red", + ) + ) + return False + + console.print(f"[green]Connector is reachable on port {port}.[/green]") + return True + + +def _expose_with_ngrok(project_dir: Path, port: int) -> None: + """Ensure ngrok is available; start a tunnel or reuse an existing one.""" + ngrok_bin = shutil.which("ngrok") + if not ngrok_bin: + console.print(Panel( + "[bold]ngrok is not installed or not on your PATH.[/bold]\n\n" + " [bold]macOS (Homebrew):[/bold]\n" + " brew install ngrok/ngrok/ngrok\n\n" + " Or download the agent from the official site.\n" + " After installing, run [cyan]ngrok config add-authtoken [/cyan] once.", + title="[yellow]ngrok not found[/yellow]", + border_style="yellow", + )) + if Confirm.ask("Open the ngrok download page?", default=True): + webbrowser.open(_NGROK_DOWNLOAD_URL) + return + + if not _ensure_connector_running_for_ngrok(project_dir, port): + return + + region = _detect_region(project_dir) + region_hint = ( + f"Use Stacksync region [bold]{region}[/bold] in Developer Studio and workflows." + ) + + data = _ngrok_local_api_tunnels() + if data is not None: + existing = _pick_https_public_url_for_port(data, port) + if existing: + console.print( + f"\n[green]A tunnel to localhost:{port} is already running.[/green]\n" + f" Public URL: [bold cyan]{existing}[/bold cyan]\n" + ) + console.print(f"[dim]{region_hint}[/dim]") + return + + global _managed_ngrok_proc + _dispose_managed_ngrok_silent() + + console.print(f"\n[bold]Starting ngrok[/bold] [dim](forwarding to localhost:{port})…[/dim]") + try: + proc = subprocess.Popen( + [ngrok_bin, "http", str(port)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + start_new_session=True, + ) + except OSError as exc: + console.print(f"[red]Could not start ngrok: {exc}[/red]") + return + + deadline = time.monotonic() + 12.0 + public_url: str | None = None + while time.monotonic() < deadline: + if proc.poll() is not None: + console.print( + Panel( + "[red]ngrok exited immediately.[/red]\n\n" + "Common causes: missing authtoken or port already in use.\n" + " [cyan]ngrok config add-authtoken YOUR_TOKEN[/cyan]\n" + f"Or run [cyan]ngrok http {port}[/cyan] in a terminal to see the full error.", + title="[red]ngrok failed[/red]", + border_style="red", + ) + ) + return + tunnels = _ngrok_local_api_tunnels() + if tunnels is not None: + public_url = _pick_https_public_url_for_port(tunnels, port) + if public_url: + break + time.sleep(0.35) + + if public_url: + _managed_ngrok_proc = proc + console.print( + f"\n[green bold]Tunnel is up.[/green bold]\n" + f" Public URL: [bold cyan]{public_url}[/bold cyan]\n" + ) + console.print( + f"[dim]{region_hint} " + f"Paste the URL into Developer Studio. " + f"Inspect tunnels at http://127.0.0.1:4040[/dim]" + ) + else: + _managed_ngrok_proc = proc + console.print( + f"\n[yellow]ngrok was started in the background but the public URL could not " + f"be read automatically.[/yellow]\n" + f" Open [cyan]http://127.0.0.1:4040[/cyan] in your browser to copy the HTTPS URL.\n" + ) + + +@click.group(invoke_without_command=True) @click.version_option(package_name="workflows_cdk") -def cli() -> None: - """Stacksync Workflows CLI -- create connectors in one command.""" +@click.pass_context +def cli(ctx: click.Context) -> None: + """Stacksync Workflows CLI -- AI-assisted connector and module generation.""" + ctx.ensure_object(dict) + if ctx.invoked_subcommand is None: + _start_menu(ctx) + + +def _banner_once(ctx: click.Context) -> None: + """Print the banner only once per CLI invocation.""" + state = ctx.ensure_object(dict) + if not state.get("banner_shown"): + _print_banner() + state["banner_shown"] = True + + +def _pause_return_to_menu(*, label: str) -> None: + """Wait for Enter so the user can read output before the menu redraws.""" + console.print() + console.rule(f"[dim]Press Enter to return to the {label}[/dim]", style="dim") + try: + console.input() + except (KeyboardInterrupt, EOFError): + _prompt_stop_managed_services_if_any() + console.print() + + +def _hint_path_instead_of_menu_number(choice: str, *, menu_range: str = "1–8") -> None: + """If the user typed a path at the menu prompt, explain the flow.""" + s = choice.strip() + if len(s) < 2: + return + if s.isdigit() and len(s) == 1: + return + pathish = ( + s.startswith((".", "/", "~")) + or "/" in s + or "\\" in s + or s.endswith(("-connector", "_connector")) + ) + if pathish: + console.print( + "[yellow]That looks like a path, not a menu number.[/yellow]\n" + f"[dim]Pick an option from the menu ([cyan]{menu_range}[/cyan]) first.[/dim]" + ) + + +def _is_connector_root(path: Path) -> bool: + """True if *path* looks like a connector root (has app_config.yaml).""" + return (path / "app_config.yaml").is_file() + + +def _discover_connectors_near_cwd(*, max_results: int = 30) -> list[Path]: + """List connector roots: current directory, then each immediate subfolder with app_config.yaml.""" + cwd = Path.cwd().resolve() + ordered: list[Path] = [] + seen: set[Path] = set() + + def add(candidate: Path) -> None: + rp = candidate.resolve() + if rp in seen or not _is_connector_root(rp): + return + seen.add(rp) + if len(ordered) < max_results: + ordered.append(rp) + + add(cwd) + try: + for child in sorted(cwd.iterdir(), key=lambda p: p.name.lower()): + if child.is_dir() and not child.name.startswith("."): + add(child) + except OSError: + pass + return ordered + + +def _connector_choice_label(path: Path) -> str: + try: + rel = path.resolve().relative_to(Path.cwd().resolve()) + s = str(rel) + return "." if s in (".", "") else s + except ValueError: + return str(path) + + +def _read_custom_connector_path() -> Path | None: + raw = console.input( + "[bold]Path to connector project[/bold] [dim](default: .)[/dim]\n> " + ).strip() or "." + project = Path(raw).expanduser().resolve() + if not project.is_dir(): + console.print(f"[red]Not a directory:[/red] {project}") + return None + return project + + +def _prompt_connector_project_dir() -> Path | None: + """Pick a connector from discovered folders, paste a path, or type a custom path.""" + discovered = _discover_connectors_near_cwd() + + if not discovered: + console.print( + "[dim]No app_config.yaml in . or immediate subfolders — enter path manually.[/dim]" + ) + return _read_custom_connector_path() + + console.print() + console.print( + "[bold]Connector projects[/bold] " + "[dim](current dir or subfolders with app_config.yaml)[/dim]" + ) + for i, p in enumerate(discovered, start=1): + console.print(f" [cyan][{i}][/cyan] {_connector_choice_label(p)}") + console.print(f" [cyan][c][/cyan] Enter a custom path") + + if len(discovered) == 1: + hint = "[bold]Select [cyan]1[/cyan] or [cyan]c[/cyan] [dim](Enter = 1)[/dim]:[/bold] " + else: + hint = ( + f"[bold]Select a number [cyan](1–{len(discovered)})[/cyan] " + f"or [cyan]c[/cyan] [dim](Enter = 1)[/dim]:[/bold] " + ) + sel = console.input(f"\n{hint}").strip() + + if not sel: + return discovered[0] + low = sel.lower() + if low in ("c", "custom"): + return _read_custom_connector_path() + if sel.isdigit(): + n = int(sel) + if 1 <= n <= len(discovered): + return discovered[n - 1] + console.print("[yellow]Invalid number.[/yellow]") + return None + + project = Path(sel).expanduser().resolve() + if project.is_dir(): + return project + console.print(f"[red]Not a directory:[/red] {project}") + return None + + +def _start_menu(ctx: click.Context) -> None: + """Branded interactive menu when no subcommand is given.""" + _banner_once(ctx) + while True: + console.print(Panel( + " [bold cyan][1][/bold cyan] Create a connector " + "[dim]Generate modules from a description[/dim]\n" + " [bold cyan][2][/bold cyan] Update a connector " + "[dim]Add modules to an existing project[/dim]\n" + " [bold cyan][3][/bold cyan] Validate a project " + "[dim]Pick connector or custom path[/dim]\n" + " [bold cyan][4][/bold cyan] Run connector locally " + "[dim]Pick project, then run_dev.sh[/dim]\n" + " [bold cyan][5][/bold cyan] Expose with ngrok " + "[dim]Pick project, then tunnel[/dim]\n" + " [bold cyan][6][/bold cyan] View documentation " + "[dim]Open the Stacksync developer docs[/dim]\n" + " [bold cyan][7][/bold cyan] Setup AI provider " + "[dim]Configure your Anthropic / OpenAI key[/dim]\n" + " [bold cyan][8][/bold cyan] Exit", + title="[bold]What would you like to do?[/bold]", + border_style="cyan", + padding=(1, 2), + )) + try: + choice = console.input("[bold]Select [cyan][1-8][/cyan]:[/bold] ").strip() + except (KeyboardInterrupt, EOFError): + _prompt_stop_managed_services_if_any() + return + + if choice == "8" or choice == "": + _prompt_stop_managed_services_if_any() + return + + try: + if choice == "1": + console.rule("[bold cyan]Create a connector[/bold cyan]", style="cyan") + console.print() + desc = console.input( + "[bold]Describe your connector:[/bold] " + "[dim](e.g. \"Klaviyo connector with API key\")[/dim]\n> " + ).strip() + if desc: + ctx.invoke(create, description=desc) + elif choice == "2": + console.rule("[bold cyan]Update a connector[/bold cyan]", style="cyan") + console.print() + proj_path = _prompt_connector_project_dir() + if proj_path is None: + continue + desc = console.input( + "[bold]What to add:[/bold] " + "[dim](e.g. \"add a delete_contact action\")[/dim]\n> " + ).strip() + if desc: + ctx.invoke( + create, + description=desc, + output=str(proj_path), + module_only=True, + ) + else: + console.print("[dim]No description provided.[/dim]") + elif choice == "3": + console.rule("[bold cyan]Validate a project[/bold cyan]", style="cyan") + console.print() + project = _prompt_connector_project_dir() + if project is not None: + ctx.invoke(validate, path=str(project)) + elif choice == "4": + console.rule("[bold cyan]Run connector locally[/bold cyan]", style="cyan") + console.print() + project = _prompt_connector_project_dir() + if project is not None: + _run_connector(project) + elif choice == "5": + console.rule("[bold cyan]Expose with ngrok[/bold cyan]", style="cyan") + console.print() + project = _prompt_connector_project_dir() + if project is not None: + port = detect_port(project) + _expose_with_ngrok(project, port) + elif choice == "6": + console.rule("[bold cyan]Documentation[/bold cyan]", style="cyan") + console.print(f"[dim]Opening {_DOCS_URL} in your browser…[/dim]") + webbrowser.open(_DOCS_URL) + console.print("[green]Done.[/green]") + elif choice == "7": + console.rule("[bold cyan]Setup AI provider[/bold cyan]", style="cyan") + console.print() + ctx.invoke(setup) + else: + console.print( + "[yellow]Invalid choice.[/yellow] " + "[dim]Enter a number [cyan]1[/cyan]–[cyan]8[/cyan] " + "(see the list above).[/dim]" + ) + _hint_path_instead_of_menu_number(choice, menu_range="1–8") + except SystemExit: + pass + + _pause_return_to_menu(label="main menu") # --------------------------------------------------------------------------- @@ -111,8 +847,11 @@ def cli() -> None: # --------------------------------------------------------------------------- @cli.command() -def setup() -> None: +@click.pass_context +def setup(ctx: click.Context) -> None: """Configure your AI provider and API key.""" + _banner_once(ctx) + _ensure_dotenv() _run_setup() @@ -134,8 +873,23 @@ def setup() -> None: is_flag=True, help="Use template matching only (no LLM call).", ) -def create(description: str, output: str, dry_run: bool, no_ai: bool) -> None: - """Create a connector from a natural-language description (max ~30 words).""" +@click.option( + "--module-only", + is_flag=True, + help="Generate only module files into an existing connector project.", +) +@click.pass_context +def create( + ctx: click.Context, + description: str, + output: str, + dry_run: bool, + no_ai: bool, + module_only: bool, +) -> None: + """Generate a Stacksync module or connector project from a natural-language description.""" + _banner_once(ctx) + _ensure_dotenv() registry = CapabilityRegistry() spec: ConnectorSpec | None = None @@ -154,9 +908,9 @@ def create(description: str, output: str, dry_run: bool, no_ai: bool) -> None: if spec is None: console.print( - "[red]Could not generate a connector spec from that description.[/red]\n" + "[red]Could not generate a module spec from that description.[/red]\n" "Try being more specific, e.g.:\n" - ' workflows create "Slack connector: send messages, list channels"' + ' workflows create "Get LinkedIn posts using an API key"' ) raise SystemExit(1) @@ -170,18 +924,118 @@ def create(description: str, output: str, dry_run: bool, no_ai: bool) -> None: console.print("\n[red]Spec has blocking errors. Aborting.[/red]") raise SystemExit(1) - _show_preview(spec) + _show_preview(spec, module_only=module_only) if dry_run: console.print("\n[dim]--dry-run: no files written.[/dim]") return + if not click.confirm("\nContinue?", default=True): + console.print("[dim]Aborted.[/dim]") + return + output_dir = Path(output).resolve() - project_dir = compile_connector(spec, output_dir) - console.print(f"\n[green bold]Connector created at:[/green bold] {project_dir}") - console.print("[dim]Run it with:[/dim] cd {0} && pip install -r requirements.txt && python main.py".format( - project_dir.name - )) + + version = _handle_overwrite(spec, output_dir, module_only) + if version is None: + return + if version != spec.version: + spec.version = version + + project_dir, _rationale = compile_connector(spec, output_dir, module_only=module_only) + + port = detect_port(project_dir) + _show_post_gen(spec, project_dir, port, module_only=module_only) + _interactive_menu(project_dir, port) + + +# --------------------------------------------------------------------------- +# workflows validate +# --------------------------------------------------------------------------- + +@cli.command() +@click.option( + "-p", "--path", + default=".", + type=click.Path(exists=True, file_okay=False), + help="Path to a connector project directory.", +) +@click.pass_context +def validate(ctx: click.Context, path: str) -> None: + """Validate a generated connector/module for Stacksync compatibility.""" + _banner_once(ctx) + project = Path(path).resolve() + console.print(f"\nValidating connector at [bold]{project}[/bold]...\n") + + ok = True + app_cfg = project / "app_config.yaml" + if app_cfg.exists(): + port = detect_port(project) + console.print(f" app_config.yaml: [green]OK[/green] (port: {port})") + else: + console.print(" app_config.yaml: [red]MISSING[/red]") + ok = False + + modules_root = project / MODULES_DIR + if not modules_root.is_dir(): + console.print(f" {MODULES_DIR}/: [red]MISSING[/red]") + ok = False + return + + for module_dir in sorted(modules_root.iterdir()): + if not module_dir.is_dir(): + continue + for version_dir in sorted(module_dir.iterdir()): + if not version_dir.is_dir(): + continue + rel = version_dir.relative_to(project) + console.print(f" {rel}/:") + + cfg = version_dir / "module_config.yaml" + if cfg.exists(): + console.print(" module_config.yaml: [green]OK[/green]") + else: + console.print(" module_config.yaml: [red]MISSING[/red]") + ok = False + + schema_path = version_dir / "schema.json" + if schema_path.exists(): + try: + schema = json.loads(schema_path.read_text()) + ver = schema.get("metadata", {}).get("workflows_module_schema_version", "?") + n_fields = len(schema.get("fields", [])) + console.print(f" schema.json: [green]OK[/green] ({n_fields} fields, Module Schema v{ver})") + except json.JSONDecodeError: + console.print(" schema.json: [red]INVALID JSON[/red]") + ok = False + else: + console.print(" schema.json: [red]MISSING[/red]") + ok = False + + route = version_dir / "route.py" + if route.exists(): + code = route.read_text() + has_execute = bool(re.search(r'"/execute"', code)) + has_content = bool(re.search(r'"/content"', code)) + has_schema = bool(re.search(r'"/schema"', code)) + parts = ["/execute" if has_execute else "[red]/execute MISSING[/red]"] + if has_content: + parts.append("/content") + if has_schema: + parts.append("/schema") + console.print(f" route.py: [green]OK[/green] ({', '.join(parts)})") + if not has_execute: + ok = False + else: + console.print(" route.py: [red]MISSING[/red]") + ok = False + + console.print() + if ok: + console.print("[green bold]All checks passed.[/green bold]\n") + else: + console.print("[red bold]Some checks failed.[/red bold]\n") + raise SystemExit(1) # --------------------------------------------------------------------------- @@ -249,17 +1103,103 @@ def inspect(app_slug: str) -> None: # --------------------------------------------------------------------------- -# Internal paths +# workflows guide +# --------------------------------------------------------------------------- + +@cli.group() +def guide() -> None: + """Step-by-step guidance for the Stacksync connector workflow.""" + + +@guide.command("run") +def guide_run() -> None: + """How to start the connector locally.""" + port = _detect_port_cwd() + console.print(Panel( + f"Starting your connector locally:\n\n" + f" 1. cd \n" + f" 2. pip install -r requirements.txt\n" + f" 3. ./run_dev.sh\n\n" + f" Or without Docker:\n" + f" python main.py\n\n" + f" Your connector will start on port {port} " + f"(configurable in app_config.yaml).", + title="[bold]Run locally[/bold]", + border_style="blue", + )) + + +@guide.command("ngrok") +def guide_ngrok() -> None: + """How to expose your connector with ngrok.""" + port = _detect_port_cwd() + console.print(Panel( + f"Exposing your connector with ngrok:\n\n" + f" Detected connector port: {port}\n\n" + f" 1. Open a new terminal\n" + f" 2. Run:\n" + f" ngrok http {port}\n" + f" 3. Copy the public HTTPS URL\n" + f" 4. Paste it in Developer Studio (see: workflows guide register)", + title="[bold]ngrok setup[/bold]", + border_style="blue", + )) + + +@guide.command("register") +def guide_register() -> None: + """How to register the connector in Developer Studio.""" + console.print(Panel( + "Registering your connector in Stacksync Developer Studio:\n\n" + " 1. Open Developer Studio in your browser\n" + " 2. Create a new Custom Connector\n" + " 3. Paste your ngrok URL as the connector base URL\n" + " 4. Configure auth settings to match your connector\n" + " 5. Save and activate the connector\n\n" + " Important: Your connector and workflows must be in the same region.", + title="[bold]Register connector[/bold]", + border_style="blue", + )) + + +@guide.command("test") +def guide_test() -> None: + """How to test your module in a Stacksync workflow.""" + console.print(Panel( + "Testing your module in a Stacksync workflow:\n\n" + " 1. Create a new workflow in Stacksync (same region as your connector)\n" + " 2. Add a step using your custom connector\n" + " 3. Select the action/module you generated\n" + " 4. Fill in the form fields\n" + " 5. Run the workflow and check the response\n\n" + " Tip: Check the connector logs in your terminal for debugging.", + title="[bold]Test in workflow[/bold]", + border_style="blue", + )) + + +# --------------------------------------------------------------------------- +# Internal helpers # --------------------------------------------------------------------------- +def _detect_port_cwd() -> int: + """Try to read port from app_config.yaml in the current directory.""" + return detect_port(Path.cwd()) + + def _template_path( description: str, registry: CapabilityRegistry, ) -> ConnectorSpec | None: - spec = match_template(description) - if spec is not None: - console.print("[green]Matched a built-in template.[/green]") - return spec + result = match_template(description) + if result is not None: + kw = ", ".join(result.matched_keywords) + console.print( + f"[green]Matched template '{result.template_name}' " + f"(keywords: {kw})[/green]" + ) + return result.spec + return None def _ai_path( @@ -268,49 +1208,287 @@ def _ai_path( ) -> ConnectorSpec | None: try: from ..ai.planner import Planner, PlannerError - except Exception: + except ImportError: console.print( - "[yellow]AI planner unavailable. Falling back to template matching.[/yellow]" + "[yellow]AI planner unavailable (missing dependency). " + "Falling back to template matching.[/yellow]" ) return _template_path(description, registry) try: planner = Planner(registry) + except (ImportError, PlannerError) as exc: + console.print(f"[yellow]AI planner init failed: {exc}[/yellow]") + console.print("[yellow]Falling back to template matching.[/yellow]") + return _template_path(description, registry) except Exception: - console.print( - "[yellow]AI planner init failed. Falling back to template matching.[/yellow]" - ) + console.print_exception(max_frames=4) + console.print("[yellow]Unexpected error initializing planner. Falling back to template matching.[/yellow]") + return _template_path(description, registry) + + # Phase 1: parse intent and build prompt (instant) + console.print(" [bold][1/3][/bold] Parsing intent…", end=" ") + try: + system, user_msg = planner.build_prompt(description) + except Exception as exc: + console.print("[red]failed[/red]") + console.print(f"[yellow]{exc}[/yellow]") return _template_path(description, registry) + console.print("[green]done[/green]") + # Phase 2: call LLM (slow) + t0 = time.monotonic() try: - with console.status("[bold]Planning connector...[/bold]"): - spec = planner.plan(description) + with console.status(" [bold][2/3][/bold] Calling AI provider…"): + spec = planner.call_llm(system, user_msg) except PlannerError as exc: + console.print(f" [bold][2/3][/bold] Calling AI provider… [red]failed[/red]") console.print(f"[yellow]{exc}[/yellow]") console.print("[yellow]Falling back to template matching.[/yellow]") return _template_path(description, registry) + except KeyboardInterrupt: + console.print("\n[dim]Cancelled.[/dim]") + return None + except Exception: + console.print(f" [bold][2/3][/bold] Calling AI provider… [red]failed[/red]") + console.print_exception(max_frames=6) + console.print("[yellow]AI generation failed. Falling back to template matching.[/yellow]") + return _template_path(description, registry) + elapsed = time.monotonic() - t0 + console.print(f" [bold][2/3][/bold] Calling AI provider… [green]done[/green] [dim]({elapsed:.0f}s)[/dim]") + + # Phase 3: validate (instant) + console.print(" [bold][3/3][/bold] Validating spec…", end=" ") + console.print("[green]done[/green]") if spec.needs_clarification: from ..ai.clarifier import render_clarification answers = render_clarification(spec) if answers: - with console.status("[bold]Refining spec...[/bold]"): - spec = planner.refine(spec, answers) + try: + with console.status(" Refining with your answers…"): + spec = planner.refine(spec, answers) + except PlannerError as exc: + console.print(f"[yellow]Refinement failed: {exc}[/yellow]") + except Exception: + console.print_exception(max_frames=6) + console.print("[yellow]Refinement failed, using initial spec.[/yellow]") return spec -def _show_preview(spec: ConnectorSpec) -> None: - tree_text = preview_tree(spec) +def _show_preview(spec: ConnectorSpec, *, module_only: bool = False) -> None: + """Compact preview: modules, fields summary, auth, endpoints.""" + lines: list[str] = [] + + action_names = [a.name for a in spec.actions] + trigger_names = [t.name for t in spec.triggers] + module_summary = ", ".join(action_names + trigger_names) + n_a, n_t = len(action_names), len(trigger_names) + counts = [] + if n_a: + counts.append(f"{n_a} action{'s' if n_a != 1 else ''}") + if n_t: + counts.append(f"{n_t} trigger{'s' if n_t != 1 else ''}") + lines.append(f"[bold]Modules:[/bold] {module_summary} ({', '.join(counts)})") + + all_fields = [] + for a in spec.actions: + all_fields.extend(a.required_fields) + all_fields.extend(a.optional_fields) + for t in spec.triggers: + all_fields.extend(t.payload_fields) + + max_show = 8 + if all_fields: + shown = all_fields[:max_show] + field_names = ", ".join(f.name for f in shown) + extra = f" (+{len(all_fields) - max_show} more)" if len(all_fields) > max_show else "" + lines.append(f"[bold]Fields:[/bold] {field_names}{extra}") + + auth_display = _AUTH_DISPLAY.get(spec.auth.type, spec.auth.type) + lines.append(f"[bold]Auth:[/bold] {auth_display}") + + endpoints = ["/execute"] + if needs_content_endpoint(all_fields): + endpoints.append("/content") + if needs_schema_endpoint(all_fields): + endpoints.append("/schema") + lines.append(f"[bold]Endpoints:[/bold] {', '.join(endpoints)}") + + console.print(Panel( + "\n".join(lines), + title="[bold]Preview[/bold]", + border_style="green", + )) - action_names = ", ".join(a.name for a in spec.actions) or "(none)" - trigger_names = ", ".join(t.name for t in spec.triggers) or "(none)" - info = ( - f"[bold]{spec.app_name}[/bold] (type: {spec.app_type}, auth: {spec.auth.type})\n" - f"Actions: {action_names}\n" - f"Triggers: {trigger_names}\n\n" - f"[dim]{tree_text}[/dim]" +def _handle_overwrite( + spec: ConnectorSpec, + output_dir: Path, + module_only: bool, +) -> Optional[str]: + """Check for existing module paths and handle overwrite/versioning. + + Returns the version string to use, or None to abort. + """ + project_dir = output_dir if module_only else (output_dir / spec.directory_name) + for a in spec.actions: + target = project_dir / MODULES_DIR / a.name / spec.version + if target.exists(): + return _prompt_overwrite(target, spec.version) + for t in spec.triggers: + target = project_dir / MODULES_DIR / t.name / spec.version + if target.exists(): + return _prompt_overwrite(target, spec.version) + return spec.version + + +def _prompt_overwrite(path: Path, current_version: str) -> Optional[str]: + console.print(f"\n[yellow]A module already exists at {path}[/yellow]") + choice = Prompt.ask( + "Choose an option", + choices=["abort", "overwrite", "new-version"], + default="abort", ) - console.print(Panel(info, title="[bold]Preview[/bold]", border_style="green")) + if choice == "abort": + console.print("[dim]Aborted.[/dim]") + return None + if choice == "overwrite": + return current_version + num = int(current_version.lstrip("v")) + 1 if current_version.lstrip("v").isdigit() else 2 + return f"v{num}" + + +def _show_post_gen( + spec: ConnectorSpec, + project_dir: Path, + port: int, + *, + module_only: bool = False, +) -> None: + """Display post-generation summary and next steps.""" + files: list[str] = [] + for a in spec.actions: + base = f"{MODULES_DIR}/{a.name}/{spec.version}" + files.extend([f"{base}/module_config.yaml", f"{base}/schema.json", f"{base}/route.py"]) + for t in spec.triggers: + base = f"{MODULES_DIR}/{t.name}/{spec.version}" + files.extend([f"{base}/module_config.yaml", f"{base}/schema.json", f"{base}/route.py"]) + + file_list = "\n".join(f" {f}" for f in files) + + if module_only: + region = _detect_region(project_dir) + body = ( + f"[green bold]Module generated successfully[/green bold]\n\n" + f"Files created:\n{file_list}\n\n" + f"Next: restart your connector and test the new module " + f"(Stacksync region: [bold]{region}[/bold])." + ) + else: + region = _detect_region(project_dir) + body = ( + f"[green bold]Module generated successfully[/green bold]\n\n" + f"Files created:\n{file_list}\n\n" + f"[bold]Next steps:[/bold]\n" + f" 1. Start the connector locally:\n" + f" ./run_dev.sh\n" + f" 2. Expose your local backend:\n" + f" ngrok http {port}\n" + f" 3. Copy the ngrok URL into Stacksync Developer Studio " + f"(region: [bold]{region}[/bold])\n" + f" 4. Create a workflow in the same region ([bold]{region}[/bold])\n" + f" 5. Add your new action and test it" + ) + + console.print(Panel(body, border_style="green")) + + +def _interactive_menu(project_dir: Path, port: int) -> None: + """Offer guided next-step actions after generation.""" + region = _detect_region(project_dir) + while True: + console.print(Panel( + f" [bold cyan][1][/bold cyan] Run the connector " + f"[dim]Start locally via run_dev.sh[/dim]\n" + f" [bold cyan][2][/bold cyan] Expose with ngrok " + f"[dim]run_dev.sh then ngrok http {port}[/dim]\n" + f" [bold cyan][3][/bold cyan] Open documentation " + f"[dim]Stacksync developer docs[/dim]\n" + f" [bold cyan][4][/bold cyan] Exit\n\n" + f" [dim]Stacksync region:[/dim] [bold]{region}[/bold]", + title="[bold]What would you like to do next?[/bold]", + border_style="cyan", + padding=(1, 2), + )) + try: + choice = console.input("[bold]Select [cyan][1-4][/cyan]:[/bold] ").strip() + except (KeyboardInterrupt, EOFError): + _prompt_stop_managed_services_if_any() + return + + if choice == "4" or choice == "": + _prompt_stop_managed_services_if_any() + return + + if choice == "1": + console.rule("[bold cyan]Run the connector[/bold cyan]", style="cyan") + console.print() + _run_connector(project_dir) + elif choice == "2": + console.rule("[bold cyan]Expose with ngrok[/bold cyan]", style="cyan") + console.print() + _expose_with_ngrok(project_dir, port) + elif choice == "3": + console.rule("[bold cyan]Documentation[/bold cyan]", style="cyan") + console.print(f"[dim]Opening {_DOCS_URL} in your browser…[/dim]") + webbrowser.open(_DOCS_URL) + console.print("[green]Done.[/green]") + else: + console.print( + "[yellow]Invalid choice.[/yellow] " + "[dim]Enter [cyan]1[/cyan]–[cyan]4[/cyan].[/dim]" + ) + _hint_path_instead_of_menu_number(choice, menu_range="1–4") + + _pause_return_to_menu(label="next steps menu") + + +def _run_connector(project_dir: Path) -> None: + """Try to start the connector from *project_dir*.""" + run_dev = project_dir / "run_dev.sh" + main_py = project_dir / "main.py" + + if run_dev.exists(): + console.print(f"\n[bold]Starting connector via run_dev.sh…[/bold]") + console.print( + "[dim]Press Ctrl+C to stop the connector. " + "If this CLI started ngrok or a background connector earlier, " + "you will be asked whether to stop those next.[/dim]\n" + ) + try: + subprocess.run(["bash", str(run_dev)], cwd=str(project_dir)) + except KeyboardInterrupt: + console.print("\n[dim]Interrupted.[/dim]") + _prompt_stop_managed_services_if_any() + elif main_py.exists(): + console.print(f"\n[bold]Starting connector via python main.py…[/bold]") + console.print( + "[dim]Press Ctrl+C to stop the connector. " + "If this CLI started ngrok or a background connector earlier, " + "you will be asked whether to stop those next.[/dim]\n" + ) + try: + subprocess.run(["python", str(main_py)], cwd=str(project_dir)) + except KeyboardInterrupt: + console.print("\n[dim]Interrupted.[/dim]") + _prompt_stop_managed_services_if_any() + else: + console.print( + f"\n[yellow]No run_dev.sh or main.py found in {project_dir}.[/yellow]\n" + f" cd {project_dir}\n" + f" pip install -r requirements.txt\n" + f" python main.py" + ) diff --git a/src/workflows_cdk/core/dynamic_routing.py b/src/workflows_cdk/core/dynamic_routing.py index 41174f6..36e7376 100644 --- a/src/workflows_cdk/core/dynamic_routing.py +++ b/src/workflows_cdk/core/dynamic_routing.py @@ -31,6 +31,14 @@ +def _safe_getcwd() -> Optional[str]: + """Return cwd if valid; None if the process cwd was deleted (raises FileNotFoundError).""" + try: + return os.path.abspath(os.getcwd()) + except (FileNotFoundError, OSError): + return None + + def load_app_config(app_dir: str) -> Dict[str, Any]: """Load app config with proper file handle cleanup.""" config_path = os.path.join(app_dir, "app_config.yaml") @@ -204,23 +212,59 @@ def __init__(self, app: Optional[Flask] = None, *, self.routes: List[Dict[str, Any]] = [] # List to store module metadata self.modules_list: List[Dict[str, Any]] = [] - # Flask application instance - self.app: Optional[Flask] = None + # Flask app set early so ``_project_root()`` can use ``app.root_path`` (stable vs getcwd). + self.app: Optional[Flask] = app self._router_instance = self + self._project_root_cache: Optional[str] = None self.environment = get_environment() - - # Initial app config loading - self.app_config = load_app_config(os.getcwd()) + + # Initial app config loading (avoids bare os.getcwd() — can raise FileNotFoundError) + self.app_config = load_app_config(self._project_root()) self.config = config or {} self.sentry_dsn = sentry_dsn self.cors_origins = cors_origins or ["*"] - + # Apply configuration settings self.refresh_app_config_variables(self.app_config) if app is not None: self.init_app(app) + def _project_root(self) -> str: + """Directory containing ``app_config.yaml`` and the routes tree. + + ``os.getcwd()`` can raise ``FileNotFoundError`` if the process cwd was deleted. Prefer + ``WORKFLOWS_PROJECT_ROOT`` / ``STACKSYNC_CONNECTOR_ROOT``, then Flask ``root_path`` + (use ``Flask(__name__)`` in ``main.py``), then cwd, then ``/usr/src/app`` in Docker. + """ + if self._project_root_cache is not None: + return self._project_root_cache + for key in ("WORKFLOWS_PROJECT_ROOT", "STACKSYNC_CONNECTOR_ROOT"): + raw = os.environ.get(key) + if raw: + p = os.path.abspath(raw) + if os.path.isdir(p): + self._project_root_cache = p + return p + flask_app = self.app + if flask_app is not None: + rp = getattr(flask_app, "root_path", None) + if rp: + p = os.path.abspath(rp) + if os.path.isdir(p): + self._project_root_cache = p + return p + cwd = _safe_getcwd() + if cwd and os.path.isdir(cwd): + self._project_root_cache = cwd + return cwd + docker_default = "/usr/src/app" + if os.path.isdir(docker_default): + self._project_root_cache = docker_default + return docker_default + self._project_root_cache = os.path.abspath(".") + return self._project_root_cache + def refresh_app_config_variables(self, app_config: Dict[str, Any]) -> None: """Apply configuration settings to the router instance. @@ -301,7 +345,7 @@ def _create_route_info(self, function: Callable, rule: Optional[str] = None, opt raise ValueError("Could not determine the module path for the function") - routes_directory = Path(os.path.join(os.getcwd(), self.routes_directory)) + routes_directory = Path(os.path.join(self._project_root(), self.routes_directory)) module_file_path = Path(function_module.__file__).resolve() # Check if the module is in the routes directory and generate metadata @@ -369,13 +413,13 @@ def _scan_routes_directory(self) -> list: Returns: list: List of route file paths (Path objects) """ - current_working_directory = os.getcwd() - routes_directory = Path(os.path.join(current_working_directory, self.routes_directory)) - + project_root = self._project_root() + routes_directory = Path(os.path.join(project_root, self.routes_directory)) + if not routes_directory.exists(): print_error(f"Routes directory not found at: {routes_directory}") return [] - + # Find all Python files in routes directory and its subdirectories # Only include .py files that are directly inside a version directory (v1, v2, vX, etc.) version_dir_pattern = re.compile(r"^v[\w\d]+$") @@ -402,9 +446,9 @@ def _collect_route_information(self) -> tuple: Returns: tuple: (routes_info, modules_list) containing metadata """ - current_working_directory = os.getcwd() - routes_directory = Path(os.path.join(current_working_directory, self.routes_directory)) - + project_root = self._project_root() + routes_directory = Path(os.path.join(project_root, self.routes_directory)) + # Lists to store collected data routes_info = [] modules_list = [] @@ -495,22 +539,22 @@ def discover_routes(self) -> None: Key issue: Each discovery loads Python modules that stay in memory forever. In serverless, this grows with each container reuse. """ - current_working_directory = os.getcwd() - + project_root = self._project_root() + # Collect route info without any imports first routes_info, module_list = self._collect_route_information() - + # Add module metadata (but clear old references first) for module in module_list: if not any(m.get("module_id") == module["module_id"] for m in self.modules_list): self.modules_list.append(module) - + if not routes_info: return - # Add working directory to path for imports - if current_working_directory not in sys.path: - sys.path.insert(0, current_working_directory) + # Add project root to path for imports + if project_root not in sys.path: + sys.path.insert(0, project_root) # Use clean import context to prevent module accumulation with clean_module_import(): @@ -518,7 +562,7 @@ def discover_routes(self) -> None: original_router_module = sys.modules.get('main', None) temporary_main_module = types.ModuleType('main') setattr(temporary_main_module, 'router', self) - setattr(temporary_main_module, '__file__', os.path.join(current_working_directory, 'main.py')) + setattr(temporary_main_module, '__file__', os.path.join(project_root, 'main.py')) sys.modules['main'] = temporary_main_module try: @@ -568,8 +612,8 @@ def discover_routes(self) -> None: sys.modules.pop('main', None) # Remove the project root from sys.path - if current_working_directory in sys.path: - sys.path.remove(current_working_directory) + if project_root in sys.path: + sys.path.remove(project_root) def register_error_handlers(self, app: Flask) -> None: """Register error handlers for the application.""" @@ -616,7 +660,7 @@ def health_check(): def app_config(): try: # Reload app config to get fresh settings - fresh_app_config = load_app_config(os.getcwd()) + fresh_app_config = load_app_config(self._project_root()) # Apply the fresh configuration self.refresh_app_config_variables(fresh_app_config) @@ -681,7 +725,7 @@ def routes(): def register_schema_routes(self, app: Flask) -> None: """Register schema routes without loading schema content into memory.""" - routes_path = os.path.join(os.getcwd(), self.routes_directory) + routes_path = os.path.join(self._project_root(), self.routes_directory) # Only proceed if auto-registration is enabled if self.app_settings.get("automatically_register_schema_routes", True): @@ -705,8 +749,8 @@ def _handle_dynamic_schema_request(self, dynamic_path: str) -> FlaskResponse: """ try: # Construct the full path to the schema file - current_working_directory = os.getcwd() - routes_directory = os.path.join(current_working_directory, self.routes_directory) + project_root = self._project_root() + routes_directory = os.path.join(project_root, self.routes_directory) schema_file_path = os.path.join(routes_directory, dynamic_path, 'schema.json') # Check if the schema file exists @@ -757,8 +801,8 @@ def _create_dynamic_schema_handler(self, route_path: str) -> Callable[[], FlaskR def dynamic_schema_handler() -> FlaskResponse: try: # Get absolute path to schema file - current_working_directory = os.getcwd() - routes_directory = os.path.join(current_working_directory, self.routes_directory) + project_root = self._project_root() + routes_directory = os.path.join(project_root, self.routes_directory) schema_file_path = os.path.join(routes_directory, route_path.lstrip('/'), 'schema.json') # Check if file exists @@ -825,10 +869,11 @@ def _register_schema_route(self, route_path: str, app: Optional[Flask] = None) - def init_app(self, app: Flask) -> None: """Initialize the router with a Flask app and register all discovered routes.""" self.app = app - + self._project_root_cache = None + self.clear_accumulated_data() - - self.app_config = load_app_config(os.getcwd()) + + self.app_config = load_app_config(self._project_root()) self.refresh_app_config_variables(self.app_config) # Update Flask configuration diff --git a/src/workflows_cdk/spec/compiler.py b/src/workflows_cdk/spec/compiler.py index c688a3b..da4ffba 100644 --- a/src/workflows_cdk/spec/compiler.py +++ b/src/workflows_cdk/spec/compiler.py @@ -22,27 +22,64 @@ DEFAULT_PORT = 2003 -def compile_connector(spec: ConnectorSpec, output_dir: Path) -> Path: - """Write a full CDK project to *output_dir* / *spec.directory_name*. +def needs_content_endpoint(fields: list[FieldSpec]) -> bool: + """True when any field requires dynamic dropdown choices via /content.""" + return any(f.dynamic_content for f in fields) - Returns the root directory of the generated project. + +def needs_schema_endpoint(fields: list[FieldSpec]) -> bool: + """True when any field declares a depends_on relationship requiring /schema reload.""" + return any(f.depends_on for f in fields) + + +def detect_port(project_dir: Path) -> int: + """Read the port from an existing app_config.yaml, falling back to DEFAULT_PORT.""" + cfg_path = project_dir / "app_config.yaml" + if cfg_path.exists(): + try: + cfg = yaml.safe_load(cfg_path.read_text()) or {} + return int( + cfg.get("local_development_settings", {}).get("port", DEFAULT_PORT) + ) + except Exception: + pass + return DEFAULT_PORT + + +def compile_connector( + spec: ConnectorSpec, + output_dir: Path, + *, + module_only: bool = False, +) -> tuple[Path, list[str]]: + """Write a CDK project (or just module files) to disk. + + Returns ``(project_dir, rationale_lines)`` where *rationale_lines* explains + what was generated and why, suitable for terminal display. """ - project_dir = output_dir / spec.directory_name + # module_only: *output_dir* is the existing connector root (do not append directory_name). + project_dir = output_dir if module_only else (output_dir / spec.directory_name) project_dir.mkdir(parents=True, exist_ok=True) + rationale: list[str] = [] - _write_main_py(spec, project_dir) - _write_app_config(spec, project_dir) - _write_requirements(project_dir) - _write_capability_yaml(spec, project_dir) - _write_deployment_files(spec, project_dir) + if not module_only: + _write_main_py(spec, project_dir) + _write_app_config(spec, project_dir) + _write_requirements(project_dir) + _write_deployment_files(spec, project_dir) + _write_gitignore(project_dir) + _write_dot_env(project_dir) + _write_readme(spec, project_dir) for action in spec.actions: - _write_action_route(spec, action, project_dir) + r = _write_action_route(spec, action, project_dir) + rationale.extend(r) for trigger in spec.triggers: - _write_trigger_route(spec, trigger, project_dir) + r = _write_trigger_route(spec, trigger, project_dir) + rationale.extend(r) - return project_dir + return project_dir, rationale def preview_tree(spec: ConnectorSpec) -> str: @@ -51,7 +88,9 @@ def preview_tree(spec: ConnectorSpec) -> str: lines.append(" main.py") lines.append(" app_config.yaml") lines.append(" requirements.txt") - lines.append(" capability.yaml") + lines.append(" .env") + lines.append(" .gitignore") + lines.append(" README.md") lines.append(" Dockerfile") lines.append(" run_dev.sh") lines.append(" config/") @@ -82,12 +121,12 @@ def preview_tree(spec: ConnectorSpec) -> str: # --------------------------------------------------------------------------- def _write_main_py(spec: ConnectorSpec, project_dir: Path) -> None: - slug = spec.app_type.replace("-", "_") - content = textwrap.dedent(f"""\ + """Use ``Flask(__name__)`` so ``app.root_path`` is the project dir (stable under Docker/gunicorn).""" + content = textwrap.dedent("""\ from flask import Flask from workflows_cdk import Router - app = Flask("{slug}") + app = Flask(__name__) router = Router(app) if __name__ == "__main__": @@ -136,41 +175,73 @@ def _write_requirements(project_dir: Path) -> None: (project_dir / "requirements.txt").write_text(content) -def _write_capability_yaml(spec: ConnectorSpec, project_dir: Path) -> None: - data: dict = { - "app": { - "name": spec.app_name.replace(" Connector", ""), - "slug": spec.app_type, - "description": f"Connector for {spec.app_name.replace(' Connector', '')}", - "auth": { - "type": spec.auth.type, - "scopes": spec.auth.scopes, - }, - }, - "actions": [ - { - "name": a.name, - "category": a.category, - "description": a.description, - "required_fields": [ - {"name": f.name, "type": f.type, "description": f.description} - for f in a.required_fields - ], - } - for a in spec.actions - ], - "triggers": [ - { - "name": t.name, - "event": t.event, - "description": t.description, - } - for t in spec.triggers - ], - } - (project_dir / "capability.yaml").write_text( - yaml.dump(data, default_flow_style=False, sort_keys=False) - ) +def _write_gitignore(project_dir: Path) -> None: + content = textwrap.dedent("""\ + __pycache__/ + *.pyc + *.pyo + .env + .venv/ + venv/ + *.egg-info/ + dist/ + build/ + .pytest_cache/ + """) + (project_dir / ".gitignore").write_text(content) + + +def _write_dot_env(project_dir: Path) -> None: + content = textwrap.dedent("""\ + ENVIRONMENT=dev + REGION=besg + """) + path = project_dir / ".env" + if not path.exists(): + path.write_text(content) + + +def _write_readme(spec: ConnectorSpec, project_dir: Path) -> None: + name = spec.app_name + actions = ", ".join(a.name for a in spec.actions) or "(none)" + triggers = ", ".join(t.name for t in spec.triggers) or "(none)" + content = textwrap.dedent(f"""\ + # {name} + + Stacksync connector generated with the Workflows CDK. + + ## Quick start + + ```bash + # Start with Docker + ./run_dev.sh + + # Or without Docker + pip install -r requirements.txt + python main.py + ``` + + The connector runs on port {DEFAULT_PORT} by default (see `app_config.yaml`). + + ## Modules + + - **Actions:** {actions} + - **Triggers:** {triggers} + + ## Project structure + + ``` + app_config.yaml Connector configuration + main.py Flask entry point + src/modules/ Module folders (route.py, schema.json, module_config.yaml) + config/ Docker & Gunicorn configs + ``` + + ## Documentation + + https://docs.stacksync.com/workflow-automation/developers/build-a-custom-connector + """) + (project_dir / "README.md").write_text(content) # --------------------------------------------------------------------------- @@ -187,6 +258,8 @@ def _write_deployment_files(spec: ConnectorSpec, project_dir: Path) -> None: WORKDIR /usr/src/app + ENV WORKFLOWS_PROJECT_ROOT=/usr/src/app + RUN pip install --upgrade pip RUN apt-get update && \\ @@ -212,6 +285,8 @@ def _write_deployment_files(spec: ConnectorSpec, project_dir: Path) -> None: WORKDIR /usr/src/app + ENV WORKFLOWS_PROJECT_ROOT=/usr/src/app + RUN apt-get update && \\ apt-get install -y git curl && \\ rm -rf /var/lib/apt/lists/* @@ -233,11 +308,16 @@ def _write_deployment_files(spec: ConnectorSpec, project_dir: Path) -> None: """)) (config_dir / "entrypoint.sh").write_text( - "exec gunicorn --config /usr/src/app/config/gunicorn_config.py main:app\n" + "cd /usr/src/app\nexec gunicorn --config /usr/src/app/config/gunicorn_config.py main:app\n" + ) + + (project_dir / "entrypoint.sh").write_text( + "cd /usr/src/app\nexec gunicorn --config /usr/src/app/config/gunicorn_config.py main:app\n" ) (config_dir / "gunicorn_config.py").write_text(textwrap.dedent("""\ bind = "0.0.0.0:8080" + chdir = "/usr/src/app" accesslog = "-" errorlog = "-" capture_output = True @@ -296,6 +376,8 @@ def _write_deployment_files(spec: ConnectorSpec, project_dir: Path) -> None: fi echo "Starting container on port ${{PORT}}..." + # Remove any previous container with the same name (avoids "name already in use") + docker rm -f "${{APP_NAME}}" 2>/dev/null || true docker run --rm -p ${{PORT}}:8080 -it -e ENVIRONMENT=dev -e REGION=besg --name=${{APP_NAME}} -v $PWD:/usr/src/app/ ${{APP_NAME}} """)) @@ -318,13 +400,13 @@ def _write_deployment_files(spec: ConnectorSpec, project_dir: Path) -> None: set "PORT={port}" set "DOCKERFILE_PATH=config\\Dockerfile.dev" - for %%%%I in ("%%CD%%") do set "DIRNAME=%%%%~nxI" + for %%I in ("!CD!") do set "DIRNAME=%%~nxI" set "APP_NAME=workflows-app-!DIRNAME!" echo Preparing !APP_NAME!... set "IMAGE_EXISTS=" - for /f %%%%i in ('docker images -q !APP_NAME! 2^>nul') do set "IMAGE_EXISTS=%%%%i" + for /f %%i in ('docker images -q !APP_NAME! 2^>nul') do set "IMAGE_EXISTS=%%i" if "!IMAGE_EXISTS!"=="" ( echo Docker image not found. Building: !APP_NAME! @@ -342,6 +424,7 @@ def _write_deployment_files(spec: ConnectorSpec, project_dir: Path) -> None: ) echo Starting container on port !PORT!... + docker rm -f !APP_NAME! 2>nul docker run --rm -p !PORT!:8080 -it -e ENVIRONMENT=dev -e REGION=besg --name=!APP_NAME! -v %%CD%%:/usr/src/app/ !APP_NAME! """)) @@ -376,6 +459,11 @@ def _write_module_config(name: str, description: str, route_dir: Path) -> None: "array": "input", } +_CONN_MGMT_MAP = { + "oauth2": ["managed"], + "api_key": ["user_managed"], +} + def _build_module_schema( fields: list[FieldSpec], @@ -386,28 +474,44 @@ def _build_module_schema( schema_fields: list[dict] = [] if auth.type in ("oauth2", "api_key"): + mgmt_types = _CONN_MGMT_MAP.get(auth.type, ["managed"]) conn_field: dict = { "type": "connection", "id": "api_connection", "label": "Connection", "allowed_app_types": [app_type], - "allowed_connection_management_types": ["managed"], + "allowed_connection_management_types": mgmt_types, "validation": {"required": True}, } schema_fields.append(conn_field) for f in fields: + widget = f.widget or _WIDGET_MAP.get(f.type, "input") + field_def: dict = { "id": f.name, "type": f.type, "label": f.name.replace("_", " ").title(), "description": f.description, + "ui_options": {"ui_widget": widget}, } + if f.required: field_def["validation"] = {"required": True} - widget = _WIDGET_MAP.get(f.type, "input") - field_def["ui_options"] = {"ui_widget": widget} + if f.description: + field_def["ui_options"]["helper_text"] = f.description + + if f.choices: + field_def["choices"] = f.choices + + if f.dynamic_content: + field_def.setdefault("on_action", {})["load_content"] = True + if f.depends_on: + field_def["content_object_depends_on_fields"] = [f.depends_on] + + if f.depends_on and not f.dynamic_content: + field_def.setdefault("on_action", {})["load_schema"] = True schema_fields.append(field_def) @@ -451,11 +555,16 @@ def _write_action_route( spec: ConnectorSpec, action: ActionSpec, project_dir: Path, -) -> None: +) -> list[str]: + """Write route.py, schema.json, module_config.yaml and return rationale lines.""" route_dir = project_dir / MODULES_DIR / action.name / spec.version route_dir.mkdir(parents=True, exist_ok=True) all_fields = list(action.required_fields) + list(action.optional_fields) + want_content = needs_content_endpoint(all_fields) + want_schema = needs_schema_endpoint(all_fields) + + rationale = _build_rationale(action.name, all_fields, want_content, want_schema) extra_imports: list[str] = [] if action.implementation.strip(): @@ -464,23 +573,35 @@ def _write_action_route( else: execute_body = _build_stub_body(spec, action.name, all_fields, action.category) - route_code = _build_route_file(execute_body, extra_imports) + route_code = _build_route_file( + execute_body, extra_imports, + include_content=want_content, + include_schema=want_schema, + ) (route_dir / "route.py").write_text(route_code) schema = _build_module_schema(all_fields, spec.auth, spec.app_type) (route_dir / "schema.json").write_text(json.dumps(schema, indent=2) + "\n") _write_module_config(action.name, action.description, route_dir) + return rationale def _write_trigger_route( spec: ConnectorSpec, trigger: TriggerSpec, project_dir: Path, -) -> None: +) -> list[str]: + """Write trigger route files and return rationale lines.""" route_dir = project_dir / MODULES_DIR / trigger.name / spec.version route_dir.mkdir(parents=True, exist_ok=True) + fields = list(trigger.payload_fields) + want_content = needs_content_endpoint(fields) + want_schema = needs_schema_endpoint(fields) + + rationale = _build_rationale(trigger.name, fields, want_content, want_schema) + extra_imports: list[str] = [] if trigger.implementation.strip(): extra_imports, clean_impl = _extract_imports(trigger.implementation) @@ -488,17 +609,58 @@ def _write_trigger_route( else: execute_body = _build_trigger_stub_body(spec, trigger) - route_code = _build_route_file(execute_body, extra_imports) + route_code = _build_route_file( + execute_body, extra_imports, + include_content=want_content, + include_schema=want_schema, + ) (route_dir / "route.py").write_text(route_code) - schema = _build_module_schema(trigger.payload_fields, spec.auth, spec.app_type) + schema = _build_module_schema(fields, spec.auth, spec.app_type) (route_dir / "schema.json").write_text(json.dumps(schema, indent=2) + "\n") _write_module_config(trigger.name, trigger.description, route_dir) + return rationale -def _build_route_file(execute_body: str, extra_imports: list[str] | None = None) -> str: - """Assemble a complete route.py with /execute, /content, and /schema.""" +def _build_rationale( + name: str, + fields: list[FieldSpec], + want_content: bool, + want_schema: bool, +) -> list[str]: + lines = [f" Adding /execute for '{name}' (every runnable module needs it)"] + + dynamic_names = [f.name for f in fields if f.dynamic_content] + if want_content: + lines.append( + f" Adding /content (dynamic dropdown data for field" + f"{'s' if len(dynamic_names) > 1 else ''} " + f"'{', '.join(dynamic_names)}')" + ) + else: + lines.append(" Skipping /content (no dynamic fields were requested)") + + dep_names = [f.name for f in fields if f.depends_on] + if want_schema: + lines.append( + f" Adding /schema (schema reload needed when " + f"'{', '.join(dep_names)}' changes)" + ) + else: + lines.append(" Skipping /schema (no schema reload needed)") + + return lines + + +def _build_route_file( + execute_body: str, + extra_imports: list[str] | None = None, + *, + include_content: bool = False, + include_schema: bool = False, +) -> str: + """Assemble a route.py with /execute and conditionally /content and /schema.""" base_imports = [ "import json", "", @@ -515,44 +677,53 @@ def _build_route_file(execute_body: str, extra_imports: list[str] | None = None) import_block = "\n".join(base_imports) - body = ( - import_block - + "\n\n\n" - + "@router.route(\"/execute\", methods=[\"POST\"])\n" - + "def execute():\n" - + " req = Request(flask_request)\n" - + " data = req.data\n" - + " credentials = req.credentials\n" - + "\n" - + execute_body + "\n" - + "\n\n" - + "@router.route(\"/content\", methods=[\"POST\"])\n" - + "def content():\n" - + " req = Request(flask_request)\n" - + " data = req.data\n" - + "\n" - + " if not data:\n" - + ' return Response(data={"message": "Missing request data"}, status_code=400)\n' - + "\n" - + ' form_data = data.get("form_data", {})\n' - + ' content_object_names = data.get("content_object_names", [])\n' - + "\n" - + " if isinstance(content_object_names, list) and content_object_names and isinstance(content_object_names[0], dict):\n" - + ' content_object_names = [obj.get("id") for obj in content_object_names if "id" in obj]\n' - + "\n" - + " content_objects = []\n" - + "\n" - + ' return Response(data={"content_objects": content_objects})\n' - + "\n\n" - + "@router.route(\"/schema\", methods=[\"POST\"])\n" - + "def schema():\n" - + " req = Request(flask_request)\n" - + ' schema_path = __file__.replace("route.py", "schema.json")\n' - + " with open(schema_path) as f:\n" - + " base_schema = json.load(f)\n" - + " return Response(data=base_schema)\n" - ) - return body + parts = [ + import_block, + "\n\n", + "@router.route(\"/execute\", methods=[\"POST\"])\n", + "def execute():\n", + " req = Request(flask_request)\n", + " data = req.data\n", + " credentials = req.credentials\n", + "\n", + execute_body + "\n", + ] + + if include_content: + parts.extend([ + "\n\n", + "@router.route(\"/content\", methods=[\"POST\"])\n", + "def content():\n", + " req = Request(flask_request)\n", + " data = req.data\n", + "\n", + " if not data:\n", + ' return Response(data={"message": "Missing request data"}, status_code=400)\n', + "\n", + ' form_data = data.get("form_data", {})\n', + ' content_object_names = data.get("content_object_names", [])\n', + "\n", + " if isinstance(content_object_names, list) and content_object_names and isinstance(content_object_names[0], dict):\n", + ' content_object_names = [obj.get("id") for obj in content_object_names if "id" in obj]\n', + "\n", + " content_objects = []\n", + "\n", + ' return Response(data={"content_objects": content_objects})\n', + ]) + + if include_schema: + parts.extend([ + "\n\n", + "@router.route(\"/schema\", methods=[\"POST\"])\n", + "def schema():\n", + " req = Request(flask_request)\n", + ' schema_path = __file__.replace("route.py", "schema.json")\n', + " with open(schema_path) as f:\n", + " base_schema = json.load(f)\n", + " return Response(data=base_schema)\n", + ]) + + return "".join(parts) def _build_stub_body( diff --git a/src/workflows_cdk/spec/connector_spec.py b/src/workflows_cdk/spec/connector_spec.py index ca5baec..a54c869 100644 --- a/src/workflows_cdk/spec/connector_spec.py +++ b/src/workflows_cdk/spec/connector_spec.py @@ -18,6 +18,10 @@ class FieldSpec(BaseModel): type: Literal["string", "number", "boolean", "object", "array"] = "string" description: str = "" required: bool = True + widget: str = "" + choices: list[dict] = Field(default_factory=list) + depends_on: str = "" + dynamic_content: bool = False class AuthSpec(BaseModel): diff --git a/src/workflows_cdk/templates/matcher.py b/src/workflows_cdk/templates/matcher.py index b6000df..7e0f83c 100644 --- a/src/workflows_cdk/templates/matcher.py +++ b/src/workflows_cdk/templates/matcher.py @@ -98,8 +98,17 @@ def _load_templates() -> list[_Template]: return templates -def match_template(description: str) -> Optional[ConnectorSpec]: - """Return the best-matching template as a ``ConnectorSpec``, or *None*.""" +class MatchResult: + """Wraps a template match with explanation metadata.""" + + def __init__(self, spec: ConnectorSpec, template_name: str, matched_keywords: list[str]) -> None: + self.spec = spec + self.template_name = template_name + self.matched_keywords = matched_keywords + + +def match_template(description: str) -> Optional[MatchResult]: + """Return the best-matching template as a ``MatchResult``, or *None*.""" tokens = set(description.lower().split()) templates = _load_templates() if not templates: @@ -107,13 +116,20 @@ def match_template(description: str) -> Optional[ConnectorSpec]: best: Optional[_Template] = None best_score = 0 + best_overlap: list[str] = [] for tpl in templates: s = tpl.score(tokens) if s > best_score: best_score = s best = tpl + corpus = set(tpl.keywords) | {tpl.slug} + best_overlap = sorted(tokens & corpus) if best is None or best_score == 0: return None - return best.to_connector_spec() + return MatchResult( + spec=best.to_connector_spec(), + template_name=best.slug, + matched_keywords=best_overlap, + )