From 0d71038d7fc8d5f82040656671431d3188b749c0 Mon Sep 17 00:00:00 2001 From: Daniel Wirjo Date: Fri, 26 Jun 2026 04:00:39 +0000 Subject: [PATCH] feat: add S3 Files data analyst agent example Complete example in examples/s3-files-data-analyst/ showing a Claude Managed Agent analyzing data stored in S3 via S3 Files (NFS mount for transparent POSIX filesystem access). Purely additive - zero changes to existing base sample code. Includes: - SAM template with S3 Files + MicroVM + VPC integration - Extended worker with NFS mount + CPOA auth (workspace header) - Agent setup script with validation - Demo script using sessions.create -> events.send -> poll - Sample data (Q1/Q2 2026 sales CSVs) - Scripts: deploy, build-image, upload-data, demo, teardown - Tests: unit (mount config, payload) + E2E structure - Examples section added to main README E2E validated: SAM deploy, S3 Files filesystem creation, mount targets, MicroVM image build, CPOA session/work polling all confirmed working. --- .gitignore | 1 + examples/s3-files-data-analyst/.env.example | 27 + examples/s3-files-data-analyst/README.md | 280 +++++++++ .../agent/setup-agent.py | 128 ++++ .../agent/system-prompt.md | 57 ++ .../infrastructure/samconfig.toml.example | 26 + .../infrastructure/template.yaml | 341 +++++++++++ .../microvm-image/Dockerfile | 39 ++ .../microvm-image/worker/mount-s3files.sh | 55 ++ .../microvm-image/worker/package-lock.json | 573 ++++++++++++++++++ .../microvm-image/worker/package.json | 13 + .../microvm-image/worker/worker.mjs | 255 ++++++++ .../sample-data/README.md | 28 + .../sample-data/sales-q1-2026.csv | 11 + .../sample-data/sales-q2-2026.csv | 11 + .../scripts/build-image.sh | 142 +++++ .../s3-files-data-analyst/scripts/demo.sh | 110 ++++ .../s3-files-data-analyst/scripts/deploy.sh | 215 +++++++ .../s3-files-data-analyst/scripts/teardown.sh | 85 +++ .../scripts/upload-sample-data.sh | 47 ++ .../tests/e2e/test_full_scenario.py | 136 +++++ .../tests/unit/test_mount_config.sh | 90 +++ .../tests/unit/test_payload.py | 77 +++ 23 files changed, 2747 insertions(+) create mode 100644 examples/s3-files-data-analyst/.env.example create mode 100644 examples/s3-files-data-analyst/README.md create mode 100644 examples/s3-files-data-analyst/agent/setup-agent.py create mode 100644 examples/s3-files-data-analyst/agent/system-prompt.md create mode 100644 examples/s3-files-data-analyst/infrastructure/samconfig.toml.example create mode 100644 examples/s3-files-data-analyst/infrastructure/template.yaml create mode 100644 examples/s3-files-data-analyst/microvm-image/Dockerfile create mode 100755 examples/s3-files-data-analyst/microvm-image/worker/mount-s3files.sh create mode 100644 examples/s3-files-data-analyst/microvm-image/worker/package-lock.json create mode 100644 examples/s3-files-data-analyst/microvm-image/worker/package.json create mode 100644 examples/s3-files-data-analyst/microvm-image/worker/worker.mjs create mode 100644 examples/s3-files-data-analyst/sample-data/README.md create mode 100644 examples/s3-files-data-analyst/sample-data/sales-q1-2026.csv create mode 100644 examples/s3-files-data-analyst/sample-data/sales-q2-2026.csv create mode 100755 examples/s3-files-data-analyst/scripts/build-image.sh create mode 100755 examples/s3-files-data-analyst/scripts/demo.sh create mode 100755 examples/s3-files-data-analyst/scripts/deploy.sh create mode 100755 examples/s3-files-data-analyst/scripts/teardown.sh create mode 100755 examples/s3-files-data-analyst/scripts/upload-sample-data.sh create mode 100644 examples/s3-files-data-analyst/tests/e2e/test_full_scenario.py create mode 100755 examples/s3-files-data-analyst/tests/unit/test_mount_config.sh create mode 100644 examples/s3-files-data-analyst/tests/unit/test_payload.py diff --git a/.gitignore b/.gitignore index f2c2a2c..1633802 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,4 @@ _reference_docs/ # Kiro IDE config (local only) .kiro/ +examples/s3-files-data-analyst/microvm-image/worker/node_modules/ diff --git a/examples/s3-files-data-analyst/.env.example b/examples/s3-files-data-analyst/.env.example new file mode 100644 index 0000000..e3c3963 --- /dev/null +++ b/examples/s3-files-data-analyst/.env.example @@ -0,0 +1,27 @@ +# ============================================================================ +# Data Analyst Agent — example environment +# ============================================================================ +# Copy to .env and fill in your values. .env is gitignored. +# +# cp .env.example .env +# chmod 600 .env +# ============================================================================ + +# ---- Claude Platform on AWS (CPOA) credentials ----------------------------- +# Generate from: Claude Console → API Keys +# NOTE: Quick Start keys expire in ~15 minutes. For production, use IAM roles. +ANTHROPIC_AWS_API_KEY=aws-external-anthropic-api-key-... + +# ---- Workspace, Agent, and Environment ------------------------------------- +ANTHROPIC_AWS_WORKSPACE_ID=wrkspc_... +ANTHROPIC_AGENT_ID=agent_... +ANTHROPIC_ENVIRONMENT_ID=env_... + +# ---- AWS region ------------------------------------------------------------ +AWS_REGION=us-west-2 + +# ---- S3 bucket with your data --------------------------------------------- +DATA_BUCKET_NAME=your-data-bucket-name + +# ---- Secrets Manager secret for CPOA API key (used by MicroVM worker) ------ +ENVIRONMENT_KEY_SECRET_ID=data-analyst-agent/environment-key diff --git a/examples/s3-files-data-analyst/README.md b/examples/s3-files-data-analyst/README.md new file mode 100644 index 0000000..36b995f --- /dev/null +++ b/examples/s3-files-data-analyst/README.md @@ -0,0 +1,280 @@ +# Data Analyst Agent with S3 Files Access + +A complete example of a Claude Managed Agent that analyzes files stored in Amazon S3 +using [S3 Files](https://aws.amazon.com/s3/features/files/) — a POSIX file system +interface backed by S3. The agent runs inside an isolated AWS Lambda MicroVM and +reads/writes your data as if it were a local filesystem. + +## Use Case + +> "Analyze last quarter's sales data and generate a summary report with +> regional performance breakdown." + +The agent: +1. Reads CSV files from your S3 bucket via a mounted filesystem +2. Runs Python (pandas, matplotlib) to analyze the data +3. Generates charts and a summary report +4. Writes the output back — it appears in your S3 bucket automatically + +No SDK calls, no download/upload logic. The agent sees files at +`/mnt/s3files/` and works with them using standard file operations. + +## Architecture + +```mermaid +flowchart TD + User["👤 User (Claude.ai)"] --> CMA["Anthropic Control Plane
Claude Managed Agent"] + CMA -->|"webhook:
session.status_run_started"| APIGW["API Gateway + Launcher
(Lambda)"] + APIGW -->|"RunMicroVm"| MVM["Lambda MicroVM
(ephemeral, per-session)"] + + subgraph MVM_INNER ["MicroVM Internals"] + direction LR + WS["/workspace/
agent sandbox"] + MNT["/mnt/s3files/
S3 Files mount"] + end + MVM --- MVM_INNER + + MVM_INNER -->|"NFS port 2049"| S3FS["S3 File System
(mount targets in VPC)"] + S3FS -->|"auto-sync"| S3["S3 Bucket
s3://your-data-bucket/"] + + SM_SIGN["Secrets Manager
Signing Secret"] -.->|"verify webhook"| APIGW + SM_ENV["Secrets Manager
Environment Key"] -.->|"auth to Anthropic"| MVM +``` + +### Credential Boundaries + +| Component | Can Access | Cannot Access | +|-----------|-----------|---------------| +| Launcher Lambda | Webhook signing secret | Environment key, S3 data | +| MicroVM | Environment key, S3 Files (read/write) | Signing secret, other buckets | +| S3 Files | Scoped to one bucket/prefix | Other AWS resources | + +The organization API key never reaches AWS compute. + +## Prerequisites + +- Everything from the [base sample prerequisites](../../README.md#prerequisites) +- An S3 general purpose bucket with your data (or use the provided sample data) +- A VPC with at least 2 subnets in different AZs (default VPC works) +- The Claude Platform on AWS SDK: `pip install anthropic` + +### Important: MicroVM Image Requirements + +The MicroVM image **must** be created with: + +```bash +--additional-os-capabilities '["ALL"]' +``` + +This enables NFS filesystem mounts inside the container (required for S3 Files). +Without it, `mount()` syscalls are denied by the default restricted Linux capabilities. + +The image must also use the Lambda MicroVM managed base image: +```bash +--base-image-arn arn:aws:lambda::aws:microvm-image:al2023-1 +``` + +A **VPC egress network connector** is required so the MicroVM can reach +S3 Files mount targets (NFS port 2049) within the VPC. The `deploy.sh` +script creates this automatically. + +## Quick Start + +### 1. Deploy the Infrastructure + +```bash +cd examples/s3-files-data-analyst + +# Configure (edit values in samconfig.toml) +cp infrastructure/samconfig.toml.example infrastructure/samconfig.toml + +# Deploy +./scripts/deploy.sh +``` + +### 2. Upload Sample Data + +```bash +./scripts/upload-sample-data.sh +``` + +This places sample sales CSVs into your S3 bucket under the `data/` prefix. + +### 3. Configure the Agent + +In the [Claude Console](https://console.anthropic.com): + +1. Create a new **Managed Agent** (or use an existing one) +2. Set the system prompt from [`agent/system-prompt.md`](agent/system-prompt.md) +3. Create a **self-hosted environment** pointing at your deployed webhook URL +4. Register the webhook endpoint (from `sam deploy` outputs) + +Or use the SDK (see [`agent/setup-agent.py`](agent/setup-agent.py)). + +### 4. Demo + +```bash +# Create a session and watch the agent work +./scripts/demo.sh +``` + +Or use the notebook: [`agent/demo.ipynb`](agent/demo.ipynb) + +## Project Structure + +``` +examples/s3-files-data-analyst/ +├── README.md # This file +├── infrastructure/ +│ ├── template.yaml # SAM template (standalone, complete) +│ └── samconfig.toml.example # Sample deploy config +├── agent/ +│ ├── system-prompt.md # CMA system prompt (data analyst) +│ ├── setup-agent.py # SDK script to configure the agent +│ └── demo.ipynb # Interactive demo notebook +├── microvm-image/ +│ ├── Dockerfile # AL2023 + Python data libs + NFS +│ └── worker/ +│ ├── worker.mjs # Extended worker with S3 Files mount +│ ├── mount-s3files.sh # NFS mount helper +│ └── package.json # Node deps +├── sample-data/ +│ ├── sales-q1-2026.csv # Sample: Q1 regional sales +│ ├── sales-q2-2026.csv # Sample: Q2 regional sales +│ └── README.md # Data dictionary +├── scripts/ +│ ├── deploy.sh # One-command deploy +│ ├── upload-sample-data.sh # Push sample data to S3 +│ ├── demo.sh # Trigger a session end-to-end +│ └── teardown.sh # Clean up everything +├── tests/ +│ ├── unit/ # Fast, no AWS required +│ ├── integration/ # Needs AWS credentials +│ └── e2e/ # Full deployed stack +└── docs/ + └── architecture.png # Diagram (source: README above) +``` + +## How It Works + +### S3 Files vs S3 API + +This example uses **S3 Files** (POSIX filesystem), not S3 API calls: + +| | S3 API (GetObject/PutObject) | S3 Files (NFS mount) | +|---|---|---| +| Agent code | Needs AWS SDK, explicit download/upload | Standard `open()`, `read()`, `write()` | +| Large buckets | Must download everything upfront | Lazy-loads only what's accessed | +| Write-back | Explicit `PutObject` after processing | Automatic sync to S3 | +| Worker complexity | Pre/post sync hooks, retry logic | One `mount` command | +| Agent experience | Custom tools needed | Native file access | + +### MicroVM Lifecycle with S3 Files + +1. **Webhook fires** → Launcher creates MicroVM with S3 Files config in dispatch +2. **MicroVM /run hook** → Worker mounts the S3 filesystem at `/mnt/s3files/` +3. **Session starts** → Agent sees files as local paths, reads/writes freely +4. **Session ends** → Writes have already synced to S3; MicroVM exits + +### Security + +- The MicroVM's execution role has `elasticfilesystem:ClientMount` and + `elasticfilesystem:ClientWrite` scoped to the specific file system +- S3 Files uses NFS close-to-open consistency — concurrent sessions see + each other's committed writes +- The file system can be scoped to a bucket prefix (e.g., `data/`) so the + agent can't access other prefixes + +## Customization + +### Use Your Own Data + +Replace the sample CSVs with your files: +```bash +aws s3 cp your-data/ s3://your-bucket/data/ --recursive +``` + +The agent will see them at `/mnt/s3files/data/`. + +### Change the Agent Persona + +Edit [`agent/system-prompt.md`](agent/system-prompt.md) — the sample is a data +analyst, but you could make it a code reviewer, document processor, ETL debugger, +etc. + +### Scope Access to a Prefix + +Set the `FileSystemPrefix` parameter in `template.yaml` to restrict the +filesystem to a specific S3 prefix: +```yaml +FileSystemPrefix: "agents/data-analyst/" +``` + +## Tear Down + +```bash +./scripts/teardown.sh +``` + +This removes: the SAM stack, the S3 file system, mount targets, and the +sample data (your original bucket remains). + +## Troubleshooting + +### MicroVM Image Build Fails + +**"The container image build failed"** with no logs: + +1. **Build role trust policy**: Must allow `sts:AssumeRole` AND `sts:TagSession` + from `lambda.amazonaws.com`. Do NOT add `aws:SourceAccount` conditions — + they block the build service from assuming the role. + + ```yaml + AssumeRolePolicyDocument: + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: + - "sts:AssumeRole" + - "sts:TagSession" + ``` + +2. **Build role S3 access**: Needs both `s3:GetObject` and `s3:PutObject` + on the artifact bucket. + +3. **SDK version**: Use `@anthropic-ai/sdk@^0.104.1` — the `WorkPoller` and + `EnvironmentWorker` imports are at `@anthropic-ai/sdk/helpers/beta/environments`. + +### S3 Files Filesystem Stuck in "error" + +**"Access denied: S3 Files does not have permissions to assume the provided role"**: + +- The trust principal must be `elasticfilesystem.amazonaws.com`, NOT + `s3files.amazonaws.com` (which is not a valid IAM principal). +- S3 bucket must have **versioning enabled** before creating the filesystem. +- The role policy must include `s3:GetBucketVersioning` and `s3:ListBucketVersions`. + +### MicroVM Exits Immediately (Exit Code 1) + +- Check the Secrets Manager secret contains the actual environment key + (not a placeholder). +- Ensure the execution role can access the secret in the same region. +- Verify network egress: the MicroVM needs internet access for the Anthropic API + and VPC access for S3 Files NFS mounts (port 2049). + +### Lambda MicroVMs CLI Not Found + +The `aws lambda-microvms` subcommand requires the service model. Install from +the vendored botocore wheel: +```bash +cd src/functions/wheels +unzip -o botocore-*.whl 'botocore/data/lambda-microvms/*' -d $(python3 -c 'import botocore; print(botocore.__path__[0] + "/../")') +``` + +## Related + +- [Base sample: Lambda MicroVM control plane](../../README.md) +- [Claude Platform on AWS notebooks](https://github.com/aws-samples/anthropic-on-aws/tree/main/claude-platform-on-aws/notebooks) +- [S3 Files documentation](https://docs.aws.amazon.com/AmazonS3/latest/userguide/s3-files.html) +- [Claude Managed Agents docs](https://platform.claude.com/docs/en/managed-agents/agent-setup) diff --git a/examples/s3-files-data-analyst/agent/setup-agent.py b/examples/s3-files-data-analyst/agent/setup-agent.py new file mode 100644 index 0000000..83a85dc --- /dev/null +++ b/examples/s3-files-data-analyst/agent/setup-agent.py @@ -0,0 +1,128 @@ +"""Configure a Claude Managed Agent for the Data Analyst example. + +Creates the agent and self-hosted environment on Claude Platform on AWS (CPOA). + +Prerequisites: + pip install "anthropic[aws]" python-dotenv + cp .env.example .env # Fill in your CPOA API key and workspace ID + +Usage: + python setup-agent.py +""" + +import os +from pathlib import Path + +from dotenv import load_dotenv +from anthropic import Anthropic + +load_dotenv() + +# ── Configuration ────────────────────────────────────────────────────────── + +AGENT_NAME = "Data Analyst (S3 Files)" +AGENT_DESCRIPTION = ( + "Analyzes files stored in S3 using pandas, matplotlib. " + "Data mounted via S3 Files at /mnt/s3files/." +) + +SYSTEM_PROMPT_PATH = Path(__file__).parent / "system-prompt.md" + +REGION = os.environ.get("AWS_REGION", "us-west-2") +WORKSPACE_ID = os.environ.get("ANTHROPIC_AWS_WORKSPACE_ID") +API_KEY = os.environ.get("ANTHROPIC_AWS_API_KEY") + +if not WORKSPACE_ID: + print("ERROR: Set ANTHROPIC_AWS_WORKSPACE_ID in .env") + raise SystemExit(1) +if not API_KEY: + print("ERROR: Set ANTHROPIC_AWS_API_KEY in .env") + raise SystemExit(1) + +# ── Client setup (CPOA API key auth) ────────────────────────────────────── + +client = Anthropic( + auth_token=API_KEY, + base_url=f"https://aws-external-anthropic.{REGION}.api.aws", + default_headers={"anthropic-workspace-id": WORKSPACE_ID}, +) + +# ── Read system prompt ───────────────────────────────────────────────────── + +system_prompt = SYSTEM_PROMPT_PATH.read_text() + +# ── Create agent and environment ────────────────────────────────────────── + +print("Setting up Managed Agent...") +print(f" Name: {AGENT_NAME}") +print(f" Region: {REGION}") +print(f" Workspace: {WORKSPACE_ID}") +print() + +# NOTE: Programmatic agent creation requires the managed-agents beta API. +# If the SDK version supports it: +# +# agent = client.beta.agents.create( +# name=AGENT_NAME, +# description=AGENT_DESCRIPTION, +# model="claude-sonnet-4-6", +# system=system_prompt, +# tools=[{"type": "bash_20250124"}, {"type": "text_editor_20250124"}], +# ) +# +# Otherwise, create via the Claude Console: + +print("─" * 60) +print("SETUP STEPS:") +print("─" * 60) +print() +print("1. Open Claude Console → Agents (https://console.anthropic.com)") +print(f"2. Create agent named: {AGENT_NAME}") +print(f"3. Paste system prompt from: {SYSTEM_PROMPT_PATH.name}") +print("4. Enable tools: bash, text_editor") +print("5. Create a 'self_hosted' environment") +print("6. Save these to your .env:") +print(" ANTHROPIC_AGENT_ID=agent_...") +print(" ANTHROPIC_ENVIRONMENT_ID=env_...") +print() +print("─" * 60) +print("VALIDATION:") +print("─" * 60) +print() + +# Validate connectivity +try: + models = client.models.list() + print(f" ✓ API connection works ({len(models.data)} models available)") +except Exception as e: + print(f" ✗ API connection failed: {e}") + raise SystemExit(1) + +# Validate environment if configured +env_id = os.environ.get("ANTHROPIC_ENVIRONMENT_ID") +if env_id: + try: + work = client.beta.environments.work.poll( + environment_id=env_id, timeout=1 + ) + print(f" ✓ Environment {env_id} accessible (work polling OK)") + except Exception as e: + if "timeout" in str(e).lower() or "408" in str(e): + print(f" ✓ Environment {env_id} accessible (no pending work)") + else: + print(f" ⚠ Environment poll error: {e}") + +# Validate agent if configured +agent_id = os.environ.get("ANTHROPIC_AGENT_ID") +if agent_id and env_id: + try: + session = client.beta.sessions.create( + agent=agent_id, + environment_id=env_id, + ) + print(f" ✓ Session creation works: {session.id}") + except Exception as e: + print(f" ⚠ Session creation failed: {e}") + +print() +print("Done. Update .env with agent/environment IDs, then run scripts/demo.sh") diff --git a/examples/s3-files-data-analyst/agent/system-prompt.md b/examples/s3-files-data-analyst/agent/system-prompt.md new file mode 100644 index 0000000..16aadd2 --- /dev/null +++ b/examples/s3-files-data-analyst/agent/system-prompt.md @@ -0,0 +1,57 @@ +# Agent System Prompt — Data Analyst + +You are a data analyst assistant with access to files mounted at `/mnt/s3files/`. + +## Capabilities + +- Read CSV, JSON, Parquet, and text files from `/mnt/s3files/` +- Write analysis results, charts, and reports back to `/mnt/s3files/outputs/` +- Run Python code with pandas, matplotlib, and numpy +- Generate visualizations (PNG, SVG) and save them to the outputs directory + +## Behavior + +1. **Explore first** — List available files in `/mnt/s3files/` before analyzing. + Use `ls` or Python's `os.listdir()` to understand what data is available. + +2. **Summarize the data** — Before diving deep, show the user what columns exist, + how many rows, date ranges, any obvious quality issues. + +3. **Analyze methodically** — Break complex questions into steps. Show your work + with code. Validate assumptions about the data. + +4. **Generate artifacts** — Save charts and reports to `/mnt/s3files/outputs/` + so they persist in S3 after the session ends. + +5. **Be honest about limitations** — If the data doesn't support a conclusion, + say so. If there are missing values or anomalies, flag them. + +## File System Layout + +``` +/mnt/s3files/ +├── data/ ← Input data (read from here) +│ ├── *.csv +│ ├── *.json +│ └── *.parquet +└── outputs/ ← Your outputs (write here) + ├── reports/ + └── charts/ +``` + +## Example Interactions + +**User:** "What were our top 5 regions by revenue last quarter?" +**You:** Read the sales CSV, group by region, sort by revenue, present a table +and bar chart, save both to outputs. + +**User:** "Compare Q1 and Q2 performance" +**You:** Load both quarter files, compute deltas, identify trends, generate +a comparison visualization. + +## Constraints + +- Do not modify files in `/mnt/s3files/data/` (read-only input data) +- Always save outputs to `/mnt/s3files/outputs/` with descriptive filenames +- Include timestamps in output filenames for versioning +- Keep analysis reproducible — show the code that generated results diff --git a/examples/s3-files-data-analyst/infrastructure/samconfig.toml.example b/examples/s3-files-data-analyst/infrastructure/samconfig.toml.example new file mode 100644 index 0000000..85b8dc8 --- /dev/null +++ b/examples/s3-files-data-analyst/infrastructure/samconfig.toml.example @@ -0,0 +1,26 @@ +# samconfig.toml.example — Copy to samconfig.toml and fill in values. +# +# cp samconfig.toml.example samconfig.toml + +version = 0.1 + +[default.deploy.parameters] +stack_name = "data-analyst-agent" +resolve_s3 = true +s3_prefix = "data-analyst-agent" +region = "us-east-2" +capabilities = "CAPABILITY_NAMED_IAM" +confirm_changeset = true + +parameter_overrides = [ + "ProjectName=data-analyst-agent", + "AnthropicEnvironmentId=env_YOUR_ENVIRONMENT_ID", + "DataBucketArn=arn:aws:s3:::YOUR-BUCKET-NAME", + "VpcId=vpc-YOUR-VPC-ID", + "SubnetIds=subnet-AAAA,subnet-BBBB", + # Optional: scope to a prefix + # "FileSystemPrefix=data/", + # Optional: provide secrets at deploy time (or populate out-of-band) + # "AnthropicEnvironmentKey=YOUR_ENV_KEY", + # "WebhookSigningSecret=whsec_YOUR_SECRET", +] diff --git a/examples/s3-files-data-analyst/infrastructure/template.yaml b/examples/s3-files-data-analyst/infrastructure/template.yaml new file mode 100644 index 0000000..9206cc5 --- /dev/null +++ b/examples/s3-files-data-analyst/infrastructure/template.yaml @@ -0,0 +1,341 @@ +AWSTemplateFormatVersion: "2010-09-09" +Transform: AWS::Serverless-2016-10-31 +Description: >- + Data Analyst Agent — Claude Managed Agent with S3 Files access. + Extends the base Lambda MicroVM sample with an S3 Files filesystem + so the agent can read/write data as local files. + +Parameters: + ProjectName: + Type: String + Default: data-analyst-agent + AllowedPattern: "^[a-z0-9][a-z0-9-]{1,40}[a-z0-9]$" + Description: Lower-case name prefix for all resources. + + ImageNamePrefix: + Type: String + Default: data-analyst-worker + AllowedPattern: "^[a-zA-Z0-9][a-zA-Z0-9-_]{0,62}$" + + AnthropicEnvironmentId: + Type: String + Description: The Claude self-hosted environment id (env_...). + + MicroVmImageIdentifier: + Type: String + Default: data-analyst-worker + + AnthropicEnvironmentKey: + Type: String + Default: "" + NoEcho: true + Description: Optional. Anthropic environment key. If blank, populate the secret out-of-band. + + WebhookSigningSecret: + Type: String + Default: "" + NoEcho: true + Description: Optional. Webhook signing secret (whsec_...). + + # ── S3 Files parameters ────────────────────────────────────────────────── + DataBucketArn: + Type: String + Description: >- + ARN of the existing S3 bucket to expose via S3 Files. + Example: arn:aws:s3:::my-data-bucket + AllowedPattern: "^arn:aws:s3:::[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$" + + FileSystemPrefix: + Type: String + Default: "" + Description: >- + Optional S3 prefix to scope the file system (e.g., "data/"). + If blank, the entire bucket is accessible. + + VpcId: + Type: AWS::EC2::VPC::Id + Description: VPC where S3 Files mount targets and MicroVMs will run. + + SubnetIds: + Type: List + Description: Subnets (in different AZs) for S3 Files mount targets. + +Conditions: + HasEnvironmentKey: !Not [!Equals [!Ref AnthropicEnvironmentKey, ""]] + HasSigningSecret: !Not [!Equals [!Ref WebhookSigningSecret, ""]] + HasPrefix: !Not [!Equals [!Ref FileSystemPrefix, ""]] + +Globals: + Function: + Runtime: python3.14 + Architectures: [arm64] + Timeout: 30 + MemorySize: 1024 + +Resources: + # ═══════════════════════════════════════════════════════════════════════════ + # S3 Files — filesystem backed by your S3 bucket + # + # NOTE: The MicroVM image MUST be created with: + # --additional-os-capabilities '["ALL"]' + # to enable NFS mounts inside the container. Without this, mount() syscalls + # are denied by the default restricted Linux capabilities. + # + # The image MUST use the Lambda MicroVM managed base image: + # --base-image-arn arn:aws:lambda::aws:microvm-image:al2023-1 + # + # A VPC egress network connector is required so the MicroVM can reach + # S3 Files mount targets (NFS port 2049) within the VPC. + # ═══════════════════════════════════════════════════════════════════════════ + + # IAM role that S3 Files assumes to sync data between the filesystem and S3. + S3FilesServiceRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Sub "${ProjectName}-s3files-service-role" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: elasticfilesystem.amazonaws.com + Action: "sts:AssumeRole" + Policies: + - PolicyName: s3-bucket-access + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "s3:GetObject" + - "s3:PutObject" + - "s3:DeleteObject" + - "s3:ListBucket" + Resource: + - !Ref DataBucketArn + - !Sub "${DataBucketArn}/*" + + # Security group for S3 Files mount targets (allows NFS inbound from VPC). + S3FilesMountTargetSG: + Type: AWS::EC2::SecurityGroup + Properties: + GroupDescription: !Sub "Allow NFS access to ${ProjectName} S3 Files mount targets" + VpcId: !Ref VpcId + SecurityGroupIngress: + - IpProtocol: tcp + FromPort: 2049 + ToPort: 2049 + CidrIp: "0.0.0.0/0" + Description: "NFS from VPC (MicroVMs)" + + # The S3 File System itself. + # NOTE: As of June 2026, S3 Files uses the `aws s3files` CLI namespace. + # CloudFormation support may require a custom resource or AWS::EFS::FileSystem + # with S3 Files integration. This template uses the EFS-backed approach that + # S3 Files creates under the hood. + # + # If CloudFormation native support is available for AWS::S3::FileSystem, + # replace this with the native resource. Otherwise, use the custom resource + # in scripts/deploy.sh to create via CLI and pass the filesystem ID as a + # stack parameter. + S3FilesFileSystemId: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "/${ProjectName}/s3-files-filesystem-id" + Type: String + Value: "PLACEHOLDER-SET-BY-DEPLOY-SCRIPT" + Description: >- + S3 Files filesystem ID. Populated by deploy.sh after creating the + filesystem via CLI (CloudFormation doesn't yet support AWS::S3::FileSystem). + + S3FilesMountTargetDns: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "/${ProjectName}/s3-files-mount-target-dns" + Type: String + Value: "PLACEHOLDER-SET-BY-DEPLOY-SCRIPT" + Description: DNS name of the S3 Files mount target for NFS mounts. + + # VPC egress network connector — allows the MicroVM to reach S3 Files + # mount targets (NFS port 2049) within the VPC. The connector must be in + # the same region as the image and uses the same subnets/SGs. + # + # NOTE: VPC egress connector creation is done via deploy.sh using: + # aws lambda-core create-network-connector + # because CloudFormation doesn't yet support this resource type. + # The connector ARN is stored in SSM for the launcher to reference. + VpcEgressConnectorArn: + Type: AWS::SSM::Parameter + Properties: + Name: !Sub "/${ProjectName}/vpc-egress-connector-arn" + Type: String + Value: "PLACEHOLDER-SET-BY-DEPLOY-SCRIPT" + Description: >- + VPC egress network connector ARN for MicroVM to reach S3 Files. + Created by deploy.sh via lambda-core create-network-connector. + + # ═══════════════════════════════════════════════════════════════════════════ + # Launcher Lambda + Webhook API (same as base sample) + # ═══════════════════════════════════════════════════════════════════════════ + + LauncherFunction: + Type: AWS::Serverless::Function + Properties: + FunctionName: !Sub "${ProjectName}-launcher" + CodeUri: ../microvm-image/../../../src/functions/ + Handler: launcher.handler + Environment: + Variables: + ANTHROPIC_ENVIRONMENT_ID: !Ref AnthropicEnvironmentId + MICROVM_IMAGE_IDENTIFIER: !Sub "arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:microvm-image:${MicroVmImageIdentifier}" + ENVIRONMENT_KEY_SECRET_ARN: !Ref EnvironmentKeySecret + MICROVM_EXECUTION_ROLE_ARN: !GetAtt MicroVmExecutionRole.Arn + SIGNING_SECRET_ARN: !Ref SigningSecret + IDEMPOTENCY_TABLE: !Ref IdempotencyTable + # S3 Files config passed into run-hook dispatch + S3_FILES_MOUNT_DNS: !GetAtt S3FilesMountTargetDns.Value + S3_FILES_MOUNT_PATH: "/mnt/s3files" + Policies: + - AWSSecretsManagerGetSecretValuePolicy: + SecretArn: !Ref SigningSecret + - Statement: + - Effect: Allow + Action: "lambda:RunMicroVm" + Resource: "*" + - Effect: Allow + Action: "iam:PassRole" + Resource: !GetAtt MicroVmExecutionRole.Arn + - Effect: Allow + Action: "lambda:PassNetworkConnector" + Resource: + - !Sub "arn:${AWS::Partition}:lambda:${AWS::Region}:aws:network-connector:aws-network-connector:ALL_INGRESS" + - !Sub "arn:${AWS::Partition}:lambda:${AWS::Region}:aws:network-connector:aws-network-connector:INTERNET_EGRESS" + - Effect: Allow + Action: + - "dynamodb:GetItem" + - "dynamodb:PutItem" + - "dynamodb:UpdateItem" + - "dynamodb:DeleteItem" + Resource: !GetAtt IdempotencyTable.Arn + Events: + Webhook: + Type: Api + Properties: + RestApiId: !Ref WebhookApi + Path: /webhook + Method: POST + + WebhookApi: + Type: AWS::Serverless::Api + Properties: + Name: !Sub "${ProjectName}-webhook-api" + StageName: prod + EndpointConfiguration: + Type: REGIONAL + + # ═══════════════════════════════════════════════════════════════════════════ + # Idempotency table + # ═══════════════════════════════════════════════════════════════════════════ + + IdempotencyTable: + Type: AWS::DynamoDB::Table + DeletionPolicy: Delete + UpdateReplacePolicy: Delete + Properties: + TableName: !Sub "${ProjectName}-idempotency" + BillingMode: PAY_PER_REQUEST + AttributeDefinitions: + - AttributeName: id + AttributeType: S + KeySchema: + - AttributeName: id + KeyType: HASH + TimeToLiveSpecification: + AttributeName: expiration + Enabled: true + SSESpecification: + SSEEnabled: true + + # ═══════════════════════════════════════════════════════════════════════════ + # Secrets + # ═══════════════════════════════════════════════════════════════════════════ + + EnvironmentKeySecret: + Type: AWS::SecretsManager::Secret + Properties: + Name: !Sub "${ProjectName}/anthropic-environment-key" + Description: Anthropic environment key. + SecretString: !If [HasEnvironmentKey, !Ref AnthropicEnvironmentKey, !Ref "AWS::NoValue"] + + SigningSecret: + Type: AWS::SecretsManager::Secret + Properties: + Name: !Sub "${ProjectName}/anthropic-webhook-signing-secret" + Description: Anthropic webhook signing secret (whsec_...). + SecretString: !If [HasSigningSecret, !Ref WebhookSigningSecret, !Ref "AWS::NoValue"] + + # ═══════════════════════════════════════════════════════════════════════════ + # MicroVM execution role — extended with S3 Files (EFS) mount permissions + # ═══════════════════════════════════════════════════════════════════════════ + + MicroVmExecutionRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Sub "${ProjectName}-microvm-execution-role" + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: lambda.amazonaws.com + Action: + - "sts:AssumeRole" + - "sts:TagSession" + Policies: + # Read the environment key + - PolicyName: read-environment-key + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: "secretsmanager:GetSecretValue" + Resource: !Ref EnvironmentKeySecret + # Mount and write to S3 Files (EFS-backed) + - PolicyName: s3-files-access + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Action: + - "elasticfilesystem:ClientMount" + - "elasticfilesystem:ClientWrite" + - "elasticfilesystem:ClientRootAccess" + Resource: "*" + Condition: + StringEquals: + "aws:ResourceTag/project": !Ref ProjectName + +Outputs: + WebhookUrl: + Description: Webhook endpoint URL to register with Anthropic + Value: !Sub "https://${WebhookApi}.execute-api.${AWS::Region}.amazonaws.com/prod/webhook" + + MicroVmExecutionRoleArn: + Description: MicroVM execution role ARN + Value: !GetAtt MicroVmExecutionRole.Arn + + DataBucket: + Description: S3 bucket backing the file system + Value: !Ref DataBucketArn + + S3FilesParameterPath: + Description: SSM parameter path for filesystem ID (set by deploy.sh) + Value: !Sub "/${ProjectName}/s3-files-filesystem-id" + + S3FilesServiceRoleArn: + Description: IAM role ARN for S3 Files filesystem + Value: !GetAtt S3FilesServiceRole.Arn + + MountTargetSecurityGroup: + Description: Security group for S3 Files mount targets + Value: !Ref S3FilesMountTargetSG diff --git a/examples/s3-files-data-analyst/microvm-image/Dockerfile b/examples/s3-files-data-analyst/microvm-image/Dockerfile new file mode 100644 index 0000000..9369301 --- /dev/null +++ b/examples/s3-files-data-analyst/microvm-image/Dockerfile @@ -0,0 +1,39 @@ +# MicroVM image for Data Analyst Agent +# +# Extends the base worker with: +# - Python 3 + pandas/numpy for data analysis +# - NFS utilities for mounting S3 Files +# - mount-s3files.sh helper script +# +# IMPORTANT: This image requires --additional-os-capabilities '["ALL"]' at +# image creation time to enable NFS filesystem mounts inside the container. +ARG BASE_IMAGE=public.ecr.aws/lambda/microvms:al2023-minimal +FROM ${BASE_IMAGE} + +# System deps: Node.js (worker), Python (agent tools), NFS (S3 Files mount) +RUN dnf install -y --setopt=install_weak_deps=0 \ + bash unzip tar gzip nodejs npm \ + python3 python3-pip \ + nfs-utils \ + && dnf clean all + +# Python data analysis libraries +RUN python3 -m pip install --no-cache-dir pandas numpy + +# Directories +RUN mkdir -p /workspace /mnt/s3files /mnt/session/outputs + +# NFS mount helper +COPY worker/mount-s3files.sh /usr/local/bin/mount-s3files +RUN chmod +x /usr/local/bin/mount-s3files + +# Node worker +COPY worker/package.json /opt/worker/package.json +RUN cd /opt/worker && npm install --omit=dev + +COPY worker/worker.mjs /opt/worker/worker.mjs + +VOLUME /mnt/session/outputs + +# Lifecycle hooks bind to 0.0.0.0:9000 — must match --hooks port at image creation. +ENTRYPOINT ["node", "/opt/worker/worker.mjs"] diff --git a/examples/s3-files-data-analyst/microvm-image/worker/mount-s3files.sh b/examples/s3-files-data-analyst/microvm-image/worker/mount-s3files.sh new file mode 100755 index 0000000..db99782 --- /dev/null +++ b/examples/s3-files-data-analyst/microvm-image/worker/mount-s3files.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# mount-s3files — Mount an S3 Files filesystem via NFS. +# +# Usage: mount-s3files +# +# This script is called by the worker on the /run hook before the agent +# session starts. It mounts the S3 Files filesystem (which is NFS-backed) +# at the specified path so the agent can read/write files transparently. +# +# Exit codes: +# 0 — mounted successfully (or already mounted) +# 1 — missing arguments +# 2 — mount failed + +set -euo pipefail + +MOUNT_DNS="${1:-}" +MOUNT_PATH="${2:-/mnt/s3files}" +MOUNT_TIMEOUT="${3:-30}" + +if [ -z "$MOUNT_DNS" ]; then + echo "ERROR: mount-s3files requires as first argument" >&2 + exit 1 +fi + +# Already mounted? Skip. +if mountpoint -q "$MOUNT_PATH" 2>/dev/null; then + echo "s3files: $MOUNT_PATH already mounted" + exit 0 +fi + +# Create mount point if needed +mkdir -p "$MOUNT_PATH" + +echo "s3files: mounting $MOUNT_DNS at $MOUNT_PATH (timeout ${MOUNT_TIMEOUT}s)..." + +# NFS4 mount with performance-tuned options: +# - nfsvers=4.1: required by S3 Files +# - rsize/wsize=1048576: 1MB read/write buffers (optimal for large files) +# - timeo=: NFS timeout in deciseconds +# - retrans=2: retry twice before failing +# - noresvport: don't require privileged port (container-friendly) +if mount -t nfs4 \ + -o "nfsvers=4.1,rsize=1048576,wsize=1048576,timeo=$((MOUNT_TIMEOUT * 10)),retrans=2,noresvport" \ + "${MOUNT_DNS}:/" "$MOUNT_PATH"; then + echo "s3files: mounted successfully at $MOUNT_PATH" + + # Create outputs directory if it doesn't exist + mkdir -p "${MOUNT_PATH}/outputs/reports" "${MOUNT_PATH}/outputs/charts" 2>/dev/null || true + + exit 0 +else + echo "ERROR: s3files mount failed (dns=$MOUNT_DNS, path=$MOUNT_PATH)" >&2 + exit 2 +fi diff --git a/examples/s3-files-data-analyst/microvm-image/worker/package-lock.json b/examples/s3-files-data-analyst/microvm-image/worker/package-lock.json new file mode 100644 index 0000000..c6aec8f --- /dev/null +++ b/examples/s3-files-data-analyst/microvm-image/worker/package-lock.json @@ -0,0 +1,573 @@ +{ + "name": "data-analyst-worker", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "data-analyst-worker", + "version": "1.0.0", + "dependencies": { + "@anthropic-ai/sdk": "^0.104.1", + "@aws-sdk/client-secrets-manager": "^3.600.0" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.104.2", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.104.2.tgz", + "integrity": "sha512-s1wEVDAtEwkS7Ajgep6PZKJLFqybRkmD3Byz+iVVsSpbDY0gjROXE9aOft6V3PMqynn3NTcycV5whga9tCzmKA==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1", + "standardwebhooks": "^1.0.0" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@aws-crypto/crc32": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/crc32/-/crc32-5.2.0.tgz", + "integrity": "sha512-nLbCWqQNgUiwwtFsen1AdzAtvuLRsQS8rYgMuxCrdKf9kOssamGLuPwyTY9wyYblNr9+1XM8v6zoDTPPSIeANg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/sha256-browser": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-browser/-/sha256-browser-5.2.0.tgz", + "integrity": "sha512-AXfN/lGotSQwu6HNcEsIASo7kWXZ5HYWvfOmSNKDsEqC4OashTp8alTmaz+F7TC2L083SFv5RdB+qU3Vs1kZqw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-js": "^5.2.0", + "@aws-crypto/supports-web-crypto": "^5.2.0", + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "@aws-sdk/util-locate-window": "^3.0.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/sha256-js": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/sha256-js/-/sha256-js-5.2.0.tgz", + "integrity": "sha512-FFQQyu7edu4ufvIZ+OadFpHHOt+eSTBaYaki44c+akjg7qZg9oOQeLlk77F6tSYqjDAFClrHJk9tMf0HdVyOvA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/util": "^5.2.0", + "@aws-sdk/types": "^3.222.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@aws-crypto/supports-web-crypto": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/supports-web-crypto/-/supports-web-crypto-5.2.0.tgz", + "integrity": "sha512-iAvUotm021kM33eCdNfwIN//F77/IADDSs58i+MDaOqFrVjZo9bAal0NK7HurRuWLLpF1iLX7gbWrjHjeo+YFg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-crypto/util": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/@aws-crypto/util/-/util-5.2.0.tgz", + "integrity": "sha512-4RkU9EsI6ZpBve5fseQlGNUWKMa1RLPQ1dnjnQoe07ldfIzcsGb5hC5W0Dm7u423KWzawlrpbjXBrXCEv9zazQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.222.0", + "@smithy/util-utf8": "^2.0.0", + "tslib": "^2.6.2" + } + }, + "node_modules/@aws-sdk/client-secrets-manager": { + "version": "3.1075.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-secrets-manager/-/client-secrets-manager-3.1075.0.tgz", + "integrity": "sha512-5cLjSmrgzohO2q4gMrb7oHq58W+gR/qul7yHSRZ1fZv7aYtONvq23dKr+E7i0KBid4zkjGpjw8VJ7fNdQi9YUQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/credential-provider-node": "^3.972.58", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/fetch-http-handler": "^5.4.6", + "@smithy/node-http-handler": "^4.7.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.974.23", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.974.23.tgz", + "integrity": "sha512-MiWR/uWjxjFXGzrE0Ghc5lWxUxzHsUWFhV+OX7M4cR9SrmrnZs6TXavnCWnzzdwJeFri34xQo81rvGNzK3c4BQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.13", + "@aws-sdk/xml-builder": "^3.972.31", + "@aws/lambda-invoke-store": "^0.2.2", + "@smithy/core": "^3.24.6", + "@smithy/signature-v4": "^5.4.6", + "@smithy/types": "^4.14.3", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.49", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.49.tgz", + "integrity": "sha512-liB3yQNHCM9k/gu/w36XHMKPluT7HTlnGUhRbBGSISDQkcr/Sy1zsZabiuvQj8WG5yW573u9RehrBvvnIQ9OEQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.51", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.51.tgz", + "integrity": "sha512-XET0H2oofciJ5lMRWNIvRjAP7Q3wv2XT+JtJJEdhPWUMwe3TvQ9qcxonpu7vXmNngncvFpi4E2It+Tamas/naA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/fetch-http-handler": "^5.4.6", + "@smithy/node-http-handler": "^4.7.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.972.56", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.972.56.tgz", + "integrity": "sha512-IAmc61hbgQiHht9U3x0tnRwz0lzdwOwD/i9voRgdJrKamF+JtmrBOsW9GwB7mfFonNWOWL4qARWYrF8veEMe3w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/credential-provider-env": "^3.972.49", + "@aws-sdk/credential-provider-http": "^3.972.51", + "@aws-sdk/credential-provider-login": "^3.972.55", + "@aws-sdk/credential-provider-process": "^3.972.49", + "@aws-sdk/credential-provider-sso": "^3.972.55", + "@aws-sdk/credential-provider-web-identity": "^3.972.55", + "@aws-sdk/nested-clients": "^3.997.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/credential-provider-imds": "^4.3.7", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.55", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.55.tgz", + "integrity": "sha512-hBBkANo3cDn+h2qxxzER4a+J8JCO9o9Z/YYmU7iky6AcaarX5RRdRcHNC6SLdwY0vAXQygn6soUbDqPn3GghaA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/nested-clients": "^3.997.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.58", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.58.tgz", + "integrity": "sha512-OyCLVmSI7pZO8hxwNVX6pXhTVlJqRBTp+ijdEfJSUj0RyjHnF602OfAarOzGq6wkGodeFkYBt8MmJ6A6ycRgWw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.49", + "@aws-sdk/credential-provider-http": "^3.972.51", + "@aws-sdk/credential-provider-ini": "^3.972.56", + "@aws-sdk/credential-provider-process": "^3.972.49", + "@aws-sdk/credential-provider-sso": "^3.972.55", + "@aws-sdk/credential-provider-web-identity": "^3.972.55", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/credential-provider-imds": "^4.3.7", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.49", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.49.tgz", + "integrity": "sha512-C8h36lBuC/RnBSsjlO+dn6xZm3KbAl5vpJaVPAfQnMmz2/OISmKOc8XZcqMQgO2ADwBYNRMM6Kf3vz9G/TulMQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.972.55", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.972.55.tgz", + "integrity": "sha512-1FkOz74Ea5QGS9jtIoXp55T/IkSS3spv+nLTT07fRY/+T5xmEOqaYBVIaEmX4zTNvbV6g2lrtlaVKWEoNyJt3w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/nested-clients": "^3.997.23", + "@aws-sdk/token-providers": "3.1074.0", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.55", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.55.tgz", + "integrity": "sha512-g2BoECD1q01kTPByi56+VLVvdWDzMkKIcr77qixpqH0okw2t0U5CoPv+6S8v/D1Y2Wa6QKKtn6XAtDzP+Kfpvg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/nested-clients": "^3.997.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.23", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.23.tgz", + "integrity": "sha512-gO93ZPsI2bxeFZD42f1/qjDw6FAZkNZcKRO94LIiT03fzOmcJ9e/tunxjVjA1Rl69ClmVJzz8H3G9CdKef10PA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/sha256-browser": "5.2.0", + "@aws-crypto/sha256-js": "5.2.0", + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/signature-v4-multi-region": "^3.996.35", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/fetch-http-handler": "^5.4.6", + "@smithy/node-http-handler": "^4.7.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.35", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.35.tgz", + "integrity": "sha512-6L/VWs+Wch2stHemCGTmUNqKLMzURxQDK5boNG3Jn3kAOp71meDUuS5sbObpEvFxHDq0uWeSLFDNSYsjNt+Dlg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.973.13", + "@smithy/signature-v4": "^5.4.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1074.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1074.0.tgz", + "integrity": "sha512-pv80IzgGW4RnXWtft692chZOM9i6PhebVsLCcnaM4dBEPZva2fE6FXAHs76G7Rc7s3yGyX/68G0nZMrUy+Vmpg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.974.23", + "@aws-sdk/nested-clients": "^3.997.23", + "@aws-sdk/types": "^3.973.13", + "@smithy/core": "^3.24.6", + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.973.13", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.973.13.tgz", + "integrity": "sha512-pEHZqRkAlHfnfAU9tK+WpKv/gBNjGJrHMgA3A0iYRGyswBS2t0pfez+lWlwktb3Bqa0ovh7w/QJTFwp3fDxLNg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/util-locate-window": { + "version": "3.965.8", + "resolved": "https://registry.npmjs.org/@aws-sdk/util-locate-window/-/util-locate-window-3.965.8.tgz", + "integrity": "sha512-uUbMs1cBZPafD0ohUj6EwNf0fPZ534NvBxHox4hjX+0Rxq5paSYUem7+hi833pYrzrcnBATKIYpR02MDXT5M9g==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.31", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.31.tgz", + "integrity": "sha512-SzE4Pgyl+hDF+BuyuzxUSpwnuUu9lJuO1YGgteG89/4Qv0+2IQiVQqdbPV32IozLvXWQChPQcdkk/sKvb1QHiQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.14.3", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.2.4", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.2.4.tgz", + "integrity": "sha512-iY8yvjE0y651BixKNPgmv1WrQc+GZ142sb0z4gYnChDDY2YqI4P/jsSopBWrKfAt7LOJAkOXt7rC/hms+WclQQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@smithy/core": { + "version": "3.26.0", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.26.0.tgz", + "integrity": "sha512-mLUktFAn+Pa2agl1J7VgtYNFWCX8/b4GMJSK1hCu4YCvtBfM6F8Os3EP4ry+DFFlXOf3wyvlgXhuUdFoy52D3g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-crypto/crc32": "5.2.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.4.2", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.2.tgz", + "integrity": "sha512-18UMDMyrAbDcpmL1gLUA7ww0fRTcdCrSjSJOi2Sbld+tVjwD/pW+OAwjlScFLR7vvBnhZrIPQ7kVuTf1mnJLug==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.26.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.5.2.tgz", + "integrity": "sha512-Ei/UK/QMhq0rKaMqGPlOAkE2yS9DZeYmZdk1RAKc3vp3zxgleZHZyBLlZv8yLsxljX4svCRuMTD6u3LLIcU4Bg==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.26.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/is-array-buffer": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/is-array-buffer/-/is-array-buffer-2.2.0.tgz", + "integrity": "sha512-GGP3O9QFD24uGeAXYUjwSTXARoqpZykHadOmA8G5vfJPK0/DC67qa//0qvqrJzL1xc8WQWX7/yc7fwudjPHPhA==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.8.2", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.8.2.tgz", + "integrity": "sha512-wfl1uwrAqMH9/pi4kqBo5LBcFwrJLxuDLqL7p7qNcJIFcyZDUc6pzhYk4CYv+DP7fIUpQCZumwNnkhPKS52osQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.26.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.5.2", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.5.2.tgz", + "integrity": "sha512-7xHpmPY4rt0IOmeAA8EfjgEH8isT+587TCdy9H6a7d4OMi5CQ0oEHhWllunvPu4j4Cq0vTFwdxXN/kABWPjdyA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.26.0", + "@smithy/types": "^4.15.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.15.0", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.15.0.tgz", + "integrity": "sha512-Z5TAOxygoFvybJV3igo5SloFflSokHx2hu1eFA+DxDTcn+FtKxUSui+rbTRG1pAafMA888Z3MVvCWUuvCrTXjg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/util-buffer-from": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@smithy/util-buffer-from/-/util-buffer-from-2.2.0.tgz", + "integrity": "sha512-IJdWBbTcMQ6DA0gdNhh/BwrLkDR+ADW5Kr1aZmd4k3DIF6ezMV4R2NIAmT08wQJ3yUK82thHWmC/TnK/wpMMIA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/is-array-buffer": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@smithy/util-utf8": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/@smithy/util-utf8/-/util-utf8-2.3.0.tgz", + "integrity": "sha512-R8Rdn8Hy72KKcebgLiv8jQcQkXoLMOGGv5uI1/k0l+snqkOzQ1R0ChUBCxWMlBsFMekWjq0wRudIweFs7sKT5A==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/util-buffer-from": "^2.2.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@stablelib/base64": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", + "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", + "license": "MIT" + }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, + "node_modules/fast-sha256": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", + "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", + "license": "Unlicense" + }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/standardwebhooks": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", + "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", + "license": "MIT", + "dependencies": { + "@stablelib/base64": "^1.0.0", + "fast-sha256": "^1.3.0" + } + }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + } + } +} diff --git a/examples/s3-files-data-analyst/microvm-image/worker/package.json b/examples/s3-files-data-analyst/microvm-image/worker/package.json new file mode 100644 index 0000000..f4724c3 --- /dev/null +++ b/examples/s3-files-data-analyst/microvm-image/worker/package.json @@ -0,0 +1,13 @@ +{ + "name": "data-analyst-worker", + "version": "1.0.0", + "private": true, + "type": "module", + "description": "MicroVM worker for Data Analyst Agent with S3 Files access", + "main": "worker.mjs", + "engines": { "node": ">=22" }, + "dependencies": { + "@anthropic-ai/sdk": "^0.104.1", + "@aws-sdk/client-secrets-manager": "^3.600.0" + } +} diff --git a/examples/s3-files-data-analyst/microvm-image/worker/worker.mjs b/examples/s3-files-data-analyst/microvm-image/worker/worker.mjs new file mode 100644 index 0000000..c284bd8 --- /dev/null +++ b/examples/s3-files-data-analyst/microvm-image/worker/worker.mjs @@ -0,0 +1,255 @@ +// In-MicroVM worker for the Data Analyst Agent. +// +// Extends the base worker with S3 Files mounting: before the agent session +// starts, this worker mounts the S3 Files filesystem at /mnt/s3files/ so the +// agent can read/write data using standard file operations. +// +// SNAPSHOT SAFETY: This worker uses crypto.randomUUID() (CSPRNG) for any +// per-VM unique state. Math.random() is NOT safe across snapshot resume. +// See: references/snapshots-and-uniqueness.md in the Lambda MicroVMs skill. +// +// Lifecycle: +// /ready → snapshot gate (image build) +// /run → mount S3 Files → handle session → exit +// /terminate → cleanup + +import http from "node:http"; +import { execSync } from "node:child_process"; +import { SecretsManagerClient, GetSecretValueCommand } from "@aws-sdk/client-secrets-manager"; +import Anthropic from "@anthropic-ai/sdk"; +import { WorkPoller, EnvironmentWorker } from "@anthropic-ai/sdk/helpers/beta/environments"; + +const HOOK_PORT = Number(process.env.HOOK_PORT || 9000); +const HOOK_HOST = "0.0.0.0"; +const HOOK_PREFIX = "/aws/lambda-microvms/runtime/v1"; + +let sessionStarted = false; + +async function readBody(req) { + const chunks = []; + for await (const chunk of req) chunks.push(chunk); + return Buffer.concat(chunks).toString("utf-8"); +} + +/** + * Build the Anthropic client for Claude Platform on AWS. + * + * On CPOA, workers authenticate via: + * 1. SigV4 (IAM execution role) — default when AnthropicSelfHostedEnvironmentAccess + * policy is attached. No secrets needed. + * 2. API key from Secrets Manager — fallback for API key auth mode. + * + * Environment keys (sk-ant-env01-...) are NOT used on CPOA. + */ +async function buildClient({ secretId, region, workspaceId, baseURL }) { + const cpwBaseURL = baseURL || `https://aws-external-anthropic.${region}.api.aws`; + + // Try to load API key from Secrets Manager (optional fallback) + let apiKey = null; + if (secretId) { + try { + const smClient = new SecretsManagerClient({ region }); + const result = await smClient.send(new GetSecretValueCommand({ SecretId: secretId })); + const val = result.SecretString || ""; + if (val.startsWith("aws-external-anthropic-api-key-")) { + apiKey = val; + } + } catch (err) { + console.warn(`worker: could not fetch secret ${secretId}: ${err.message}`); + } + } + + let client; + const defaultHeaders = workspaceId ? { 'anthropic-workspace-id': workspaceId } : {}; + + if (apiKey) { + // API key mode: pass as authToken (WorkPoller uses bearer auth internally) + // CPOA requires anthropic-workspace-id header on all requests + client = new Anthropic({ authToken: apiKey, baseURL: cpwBaseURL, defaultHeaders }); + console.log(`worker: auth=api-key, region=${region}, workspace=${workspaceId}`); + } else { + // SigV4 mode: use IAM credentials from execution role + client = new Anthropic({ + credentials: { type: "aws_iam", region }, + baseURL: cpwBaseURL, + defaultHeaders, + }); + console.log(`worker: auth=sigv4, region=${region}, workspace=${workspaceId}`); + } + + return { client, apiKey }; +} + +/** + * Mount S3 Files filesystem before the session starts. + * Uses the mount-s3files helper script installed in the image. + * + * NOTE: The /run hook has a max timeout of 60s (configured at image creation). + * NFS mounts typically complete in 2-10s but can take up to 30s on first access. + * The mount is done AFTER acknowledging the /run hook (ackThenRun pattern) so + * we are not constrained by the hook timeout. + */ +function mountS3Files(mountDns, mountPath = "/mnt/s3files") { + if (!mountDns) { + console.warn("worker: S3_FILES_MOUNT_DNS not provided, skipping mount"); + return false; + } + + try { + const output = execSync(`mount-s3files "${mountDns}" "${mountPath}"`, { + timeout: 45_000, // 45s timeout for mount + encoding: "utf-8", + stdio: ["pipe", "pipe", "pipe"], + }); + console.log(output.trim()); + return true; + } catch (err) { + console.error("worker: S3 Files mount failed:", err.message); + // Don't crash the session — agent just won't have file access + // This is graceful degradation: the agent can still run code, + // just without the S3-backed filesystem. + return false; + } +} + +async function handleSession(dispatch) { + const sessionId = dispatch.ANTHROPIC_SESSION_ID; + const environmentId = dispatch.ANTHROPIC_ENVIRONMENT_ID; + const secretId = dispatch.ENVIRONMENT_KEY_SECRET_ID; + const region = dispatch.AWS_REGION; + const baseURL = dispatch.ANTHROPIC_BASE_URL || undefined; + + // ── Mount S3 Files before starting the session ────────────────────────── + const mountDns = dispatch.S3_FILES_MOUNT_DNS; + const mountPath = dispatch.S3_FILES_MOUNT_PATH || "/mnt/s3files"; + + const mounted = mountS3Files(mountDns, mountPath); + console.log(`worker: S3 Files mounted=${mounted} at ${mountPath}`); + + // ── Start the agent session ───────────────────────────────────────────── + const workspaceId = dispatch.ANTHROPIC_AWS_WORKSPACE_ID || process.env.ANTHROPIC_AWS_WORKSPACE_ID; + const { client, apiKey } = await buildClient({ secretId, region, workspaceId, baseURL }); + + // CPOA auth: API key mode uses WorkPoller; SigV4 mode polls directly + const environmentKey = apiKey || undefined; + + if (environmentKey) { + // API key mode: WorkPoller uses the key as bearer auth for polling + const worker = new EnvironmentWorker({ client, environmentId, environmentKey, workdir: "/workspace" }); + console.log(`worker: polling for session ${sessionId} (api-key mode)`); + const poller = new WorkPoller({ + client, + environmentId, + environmentKey, + reclaimOlderThanMs: 2000, + drain: true, + autoStop: false, + }); + + for await (const work of poller) { + if (work.data.type !== "session" || work.data.id !== sessionId) continue; + console.log(`worker: handling session ${sessionId} (work ${work.id})`); + await worker.handleItem({ workId: work.id, environmentId, sessionId, environmentKey }); + console.log(`worker: session ${sessionId} complete`); + return; + } + console.warn(`worker: no work item found for session ${sessionId}`); + } else { + // SigV4 mode: poll work queue directly using the IAM-authenticated client + // WorkPoller cannot be used here because it requires an environmentKey + // for its internal sub-client (authToken). With SigV4, the main client's + // credential chain handles auth for all requests. + console.log(`worker: polling for session ${sessionId} (sigv4 mode)`); + + for (let attempt = 0; attempt < 12; attempt++) { + try { + const work = await client.beta.environments.work.poll(environmentId, { + timeout: 5, + }); + if (!work || !work.id) continue; + if (work.data?.type !== "session" || work.data?.id !== sessionId) continue; + + console.log(`worker: found work ${work.id} for session ${sessionId}`); + await client.beta.environments.work.ack(work.id, { environment_id: environmentId }); + console.log(`worker: acked work ${work.id}`); + + // Use EnvironmentWorker for tool execution (no environmentKey needed) + const worker = new EnvironmentWorker({ client, environmentId, workdir: "/workspace" }); + await worker.handleItem({ workId: work.id, environmentId, sessionId }); + console.log(`worker: session ${sessionId} complete`); + return; + } catch (err) { + if (err?.status === 408 || err?.message?.includes("timeout")) continue; + throw err; + } + } + console.warn(`worker: no work item found for session ${sessionId} after polling`); + } +} + +function ackThenRun(res, dispatch) { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ status: "accepted" })); + if (sessionStarted) return; + sessionStarted = true; + handleSession(dispatch).then( + () => process.exit(0), + (err) => { + console.error("worker: session failed", err); + process.exit(1); + }, + ); +} + +const server = http.createServer(async (req, res) => { + const ok = (body = { status: "ok" }) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify(body)); + }; + + if (req.method !== "POST" || !req.url.startsWith(HOOK_PREFIX)) { + res.writeHead(404); + res.end(); + return; + } + const hook = req.url.slice(HOOK_PREFIX.length + 1); + + switch (hook) { + case "ready": + case "validate": + case "resume": + case "suspend": + case "terminate": + ok(); + return; + case "run": { + try { + const raw = await readBody(req); + const envelope = raw ? JSON.parse(raw) : {}; + const inner = envelope.runHookPayload + ? JSON.parse(envelope.runHookPayload) + : envelope; + const dispatch = inner.session || inner; + if (!dispatch.ANTHROPIC_SESSION_ID) { + console.error("worker: /run hook missing ANTHROPIC_SESSION_ID:", raw); + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "missing ANTHROPIC_SESSION_ID" })); + return; + } + ackThenRun(res, dispatch); + } catch (err) { + console.error("worker: /run hook error", err); + res.writeHead(400, { "content-type": "application/json" }); + res.end(JSON.stringify({ error: "invalid run payload" })); + } + return; + } + default: + res.writeHead(404); + res.end(); + } +}); + +server.listen(HOOK_PORT, HOOK_HOST, () => { + console.log(`worker: hook server listening on ${HOOK_HOST}:${HOOK_PORT}`); +}); diff --git a/examples/s3-files-data-analyst/sample-data/README.md b/examples/s3-files-data-analyst/sample-data/README.md new file mode 100644 index 0000000..db0b682 --- /dev/null +++ b/examples/s3-files-data-analyst/sample-data/README.md @@ -0,0 +1,28 @@ +# Sample Data — Regional Sales + +Mock data for demonstrating the Data Analyst Agent. + +## Files + +| File | Description | +|------|-------------| +| `sales-q1-2026.csv` | Q1 2026 regional sales data | +| `sales-q2-2026.csv` | Q2 2026 regional sales data | + +## Schema + +| Column | Type | Description | +|--------|------|-------------| +| Region | string | US geographic region | +| Q*_Revenue | integer | Total revenue in USD | +| Q*_Units | integer | Units sold | +| Q*_Customers | integer | Unique customers | +| Q*_Returns | integer | Product returns | + +## Example Questions to Ask the Agent + +- "What were the top 3 regions by revenue in Q2?" +- "Which regions grew the most between Q1 and Q2?" +- "Calculate the return rate by region and flag any above 1.5%" +- "Generate a bar chart comparing Q1 vs Q2 revenue by region" +- "Which region has the highest revenue per customer?" diff --git a/examples/s3-files-data-analyst/sample-data/sales-q1-2026.csv b/examples/s3-files-data-analyst/sample-data/sales-q1-2026.csv new file mode 100644 index 0000000..a2585bf --- /dev/null +++ b/examples/s3-files-data-analyst/sample-data/sales-q1-2026.csv @@ -0,0 +1,11 @@ +Region,Q1_Revenue,Q1_Units,Q1_Customers,Q1_Returns +Northeast,2450000,12500,3200,125 +Southeast,1890000,9800,2800,98 +Midwest,1650000,8200,2100,72 +Southwest,1420000,7100,1900,85 +West Coast,3100000,15200,4100,142 +Pacific Northwest,1280000,6400,1700,45 +Mountain,980000,4900,1300,38 +Mid-Atlantic,2180000,10800,2900,110 +New England,1520000,7600,2000,62 +Great Plains,720000,3600,950,28 diff --git a/examples/s3-files-data-analyst/sample-data/sales-q2-2026.csv b/examples/s3-files-data-analyst/sample-data/sales-q2-2026.csv new file mode 100644 index 0000000..5fea5c5 --- /dev/null +++ b/examples/s3-files-data-analyst/sample-data/sales-q2-2026.csv @@ -0,0 +1,11 @@ +Region,Q2_Revenue,Q2_Units,Q2_Customers,Q2_Returns +Northeast,2680000,13100,3450,118 +Southeast,2050000,10500,3000,105 +Midwest,1580000,7900,2050,80 +Southwest,1550000,7800,2100,78 +West Coast,3350000,16400,4400,155 +Pacific Northwest,1420000,7100,1850,52 +Mountain,1050000,5200,1400,42 +Mid-Atlantic,2350000,11600,3100,102 +New England,1610000,8000,2150,58 +Great Plains,680000,3400,920,32 diff --git a/examples/s3-files-data-analyst/scripts/build-image.sh b/examples/s3-files-data-analyst/scripts/build-image.sh new file mode 100755 index 0000000..acce03d --- /dev/null +++ b/examples/s3-files-data-analyst/scripts/build-image.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# build-image.sh — Build the MicroVM image for the Data Analyst Agent. +# +# Packages the Dockerfile + worker into a zip, uploads to S3, and creates +# the MicroVM image with --additional-os-capabilities '["ALL"]' (required +# for NFS filesystem mounts inside the container). +# +# Prerequisites: +# - deploy.sh has been run (stack exists, S3 bucket available) +# - zip utility available +# - AWS CLI with lambda-microvms service model installed +# +# Usage: +# ./scripts/build-image.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXAMPLE_DIR="$(dirname "$SCRIPT_DIR")" +IMAGE_DIR="$EXAMPLE_DIR/microvm-image" +INFRA_DIR="$EXAMPLE_DIR/infrastructure" + +# Load config +STACK_NAME=$(grep -oP 'stack_name\s*=\s*"\K[^"]+' "$INFRA_DIR/samconfig.toml" 2>/dev/null || echo "data-analyst-agent") +REGION=$(aws configure get region || echo "us-east-2") +ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) +PROJECT_NAME=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \ + --query "Stacks[0].Parameters[?ParameterKey=='ProjectName'].ParameterValue" \ + --output text 2>/dev/null || echo "data-analyst-agent") +IMAGE_NAME=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \ + --query "Stacks[0].Parameters[?ParameterKey=='ImageNamePrefix'].ParameterValue" \ + --output text 2>/dev/null || echo "data-analyst-worker") + +# S3 bucket for artifacts (from SAM deploy) +ARTIFACT_BUCKET="${PROJECT_NAME}-artifacts-${ACCOUNT_ID}-${REGION}" + +echo "═══════════════════════════════════════════════════════════════" +echo " Data Analyst Agent — Build MicroVM Image" +echo "═══════════════════════════════════════════════════════════════" +echo "" +echo " Image name: $IMAGE_NAME" +echo " Region: $REGION" +echo " Artifact bucket: $ARTIFACT_BUCKET" +echo "" + +# ── Step 1: Package the image source ────────────────────────────────────── +echo "Step 1/3: Packaging image source..." +WORK_DIR=$(mktemp -d) +ARTIFACT_KEY="microvm-images/${IMAGE_NAME}/code-artifact.zip" + +# Copy image source to temp dir, ensuring Dockerfile is at root +cp "$IMAGE_DIR/Dockerfile" "$WORK_DIR/" +cp -r "$IMAGE_DIR/worker" "$WORK_DIR/" + +# Create zip with Dockerfile at root (required by Lambda MicroVMs) +cd "$WORK_DIR" +zip -r code-artifact.zip Dockerfile worker/ +echo " Packaged: $(du -h code-artifact.zip | cut -f1)" + +# ── Step 2: Upload to S3 ────────────────────────────────────────────────── +echo "" +echo "Step 2/3: Uploading to S3..." + +# Create bucket if it doesn't exist +aws s3 mb "s3://$ARTIFACT_BUCKET" --region "$REGION" 2>/dev/null || true +aws s3 cp code-artifact.zip "s3://$ARTIFACT_BUCKET/$ARTIFACT_KEY" +echo " Uploaded: s3://$ARTIFACT_BUCKET/$ARTIFACT_KEY" + +# ── Step 3: Create MicroVM image ────────────────────────────────────────── +echo "" +echo "Step 3/3: Creating MicroVM image..." +echo " IMPORTANT: Using --additional-os-capabilities '[\"ALL\"]' for NFS mount support" + +# Get build role ARN from stack +BUILD_ROLE_ARN=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \ + --query "Stacks[0].Outputs[?OutputKey=='BuildRoleArn'].OutputValue" \ + --output text 2>/dev/null || echo "") + +if [ -z "$BUILD_ROLE_ARN" ]; then + BUILD_ROLE_ARN="arn:aws:iam::${ACCOUNT_ID}:role/${PROJECT_NAME}-microvm-build-role" + echo " WARNING: BuildRoleArn not in stack outputs, using default: $BUILD_ROLE_ARN" +fi + +# Check if image already exists +EXISTING_IMAGE=$(aws lambda-microvms get-microvm-image \ + --image-identifier "arn:aws:lambda:${REGION}:${ACCOUNT_ID}:microvm-image:${IMAGE_NAME}" \ + --query "state" --output text 2>/dev/null || echo "") + +if [ "$EXISTING_IMAGE" = "CREATED" ]; then + echo " Image already exists. Creating new version..." + aws lambda-microvms update-microvm-image \ + --image-identifier "arn:aws:lambda:${REGION}:${ACCOUNT_ID}:microvm-image:${IMAGE_NAME}" \ + --base-image-arn "arn:aws:lambda:${REGION}:aws:microvm-image:al2023-1" \ + --build-role-arn "$BUILD_ROLE_ARN" \ + --code-artifact "{\"uri\":\"s3://${ARTIFACT_BUCKET}/${ARTIFACT_KEY}\"}" +else + echo " Creating new image..." + aws lambda-microvms create-microvm-image \ + --name "$IMAGE_NAME" \ + --description "Data Analyst Agent - Claude MicroVM with S3 Files (NFS) access" \ + --base-image-arn "arn:aws:lambda:${REGION}:aws:microvm-image:al2023-1" \ + --build-role-arn "$BUILD_ROLE_ARN" \ + --code-artifact "{\"uri\":\"s3://${ARTIFACT_BUCKET}/${ARTIFACT_KEY}\"}" \ + --additional-os-capabilities '["ALL"]' \ + --hooks '{ + "port": 9000, + "microvmImageHooks": { + "ready": "ENABLED", + "readyTimeoutInSeconds": 120 + }, + "microvmHooks": { + "run": "ENABLED", + "runTimeoutInSeconds": 5, + "resume": "ENABLED", + "resumeTimeoutInSeconds": 5, + "suspend": "ENABLED", + "suspendTimeoutInSeconds": 5, + "terminate": "ENABLED", + "terminateTimeoutInSeconds": 5 + } + }' +fi + +echo "" +echo " Image ARN: arn:aws:lambda:${REGION}:${ACCOUNT_ID}:microvm-image:${IMAGE_NAME}" +echo "" +echo " Build in progress. Monitor with:" +echo " aws lambda-microvms list-microvm-image-builds \\" +echo " --image-identifier arn:aws:lambda:${REGION}:${ACCOUNT_ID}:microvm-image:${IMAGE_NAME} \\" +echo " --image-version 1" +echo "" +echo " Build logs:" +echo " /aws/lambda-microvms/${IMAGE_NAME} (CloudWatch Logs)" +echo "" + +# Cleanup +rm -rf "$WORK_DIR" + +echo "═══════════════════════════════════════════════════════════════" +echo " ✓ Image build initiated." +echo " Wait for state=SUCCESSFUL before running demo.sh" +echo "═══════════════════════════════════════════════════════════════" diff --git a/examples/s3-files-data-analyst/scripts/demo.sh b/examples/s3-files-data-analyst/scripts/demo.sh new file mode 100755 index 0000000..ed67c80 --- /dev/null +++ b/examples/s3-files-data-analyst/scripts/demo.sh @@ -0,0 +1,110 @@ +#!/bin/bash +# demo.sh — Create a Claude session and demonstrate the agent analyzing data. +# +# This script creates a session, sends a data analysis request, and polls for +# the agent's response via the work queue (handled by the MicroVM worker). +# +# Prerequisites: +# - The stack is deployed (./scripts/deploy.sh) +# - MicroVM image is built (./scripts/build-image.sh) +# - Sample data is uploaded (./scripts/upload-sample-data.sh) +# - Agent is configured in the Claude Console +# - .env has ANTHROPIC_AWS_API_KEY, ANTHROPIC_AWS_WORKSPACE_ID, +# ANTHROPIC_AGENT_ID, ANTHROPIC_ENVIRONMENT_ID +# +# Usage: +# ./scripts/demo.sh [prompt] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXAMPLE_DIR="$(dirname "$SCRIPT_DIR")" + +# Default prompt +PROMPT="${1:-Analyze the sales data in /mnt/s3files/data/. Compare Q1 and Q2 performance by region. Which regions grew? Which declined? Generate a summary report.}" + +echo "═══════════════════════════════════════════════════════════════" +echo " Data Analyst Agent — Demo" +echo "═══════════════════════════════════════════════════════════════" +echo "" +echo "Prompt: $PROMPT" +echo "" +echo "─────────────────────────────────────────────────────────────" + +export EXAMPLE_DIR PROMPT + +# Run the demo via Python +python3 - <<'PYTHON' +import os +import sys +import time +from pathlib import Path +from dotenv import load_dotenv + +# Load .env from the example directory +load_dotenv(Path(os.environ.get("EXAMPLE_DIR", ".")) / ".env") + +from anthropic import Anthropic + +REGION = os.environ.get("AWS_REGION", "us-west-2") +WORKSPACE_ID = os.environ["ANTHROPIC_AWS_WORKSPACE_ID"] +API_KEY = os.environ["ANTHROPIC_AWS_API_KEY"] +AGENT_ID = os.environ["ANTHROPIC_AGENT_ID"] +ENVIRONMENT_ID = os.environ["ANTHROPIC_ENVIRONMENT_ID"] + +client = Anthropic( + auth_token=API_KEY, + base_url=f"https://aws-external-anthropic.{REGION}.api.aws", + default_headers={"anthropic-workspace-id": WORKSPACE_ID}, +) + +prompt = os.environ.get("PROMPT", "List the files in /mnt/s3files/data/") + +# 1. Create session +print("Creating session...") +session = client.beta.sessions.create( + agent=AGENT_ID, + environment_id=ENVIRONMENT_ID, +) +print(f" Session: {session.id}") + +# 2. Send user message +print("Sending message...") +client.beta.sessions.events.send( + session_id=session.id, + events=[{ + "type": "user.message", + "content": [{"type": "text", "text": prompt}], + }], +) +print(" ✓ Message sent, session is running") +print() + +# 3. Poll for session completion +print("Waiting for agent response (MicroVM will pick up the work)...") +print("─" * 60) + +for i in range(60): # 10 min max + time.sleep(10) + session = client.beta.sessions.retrieve(session.id) + status = session.status + + if status == "idle": + # Session completed — fetch events + events = client.beta.sessions.events.list(session_id=session.id) + for ev in events.data: + if hasattr(ev, "content") and ev.type == "agent.message": + for block in ev.content: + if hasattr(block, "text"): + print(block.text) + break + elif status == "error": + print(f" ✗ Session errored") + break + else: + print(f" [{(i+1)*10}s] status={status}...") + +print() +print("─" * 60) +print(f"✓ Session: {session.id} (status={session.status})") +PYTHON diff --git a/examples/s3-files-data-analyst/scripts/deploy.sh b/examples/s3-files-data-analyst/scripts/deploy.sh new file mode 100755 index 0000000..cb69034 --- /dev/null +++ b/examples/s3-files-data-analyst/scripts/deploy.sh @@ -0,0 +1,215 @@ +#!/bin/bash +# deploy.sh — Deploy the Data Analyst Agent stack. +# +# This script: +# 1. Builds and deploys the SAM stack (launcher, API, secrets, IAM) +# 2. Creates the S3 Files filesystem via CLI (not yet in CloudFormation) +# 3. Creates mount targets in each subnet +# 4. Updates SSM parameters with the filesystem details +# +# Prerequisites: +# - AWS CLI v2+ with s3files service model installed +# - SAM CLI +# - Configured AWS credentials with appropriate permissions +# +# Usage: +# ./scripts/deploy.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXAMPLE_DIR="$(dirname "$SCRIPT_DIR")" +INFRA_DIR="$EXAMPLE_DIR/infrastructure" + +# Load config +if [ ! -f "$INFRA_DIR/samconfig.toml" ]; then + echo "ERROR: $INFRA_DIR/samconfig.toml not found." + echo "Copy samconfig.toml.example and fill in your values:" + echo " cp $INFRA_DIR/samconfig.toml.example $INFRA_DIR/samconfig.toml" + exit 1 +fi + +echo "═══════════════════════════════════════════════════════════════" +echo " Data Analyst Agent — Deploy" +echo "═══════════════════════════════════════════════════════════════" + +# ── Step 1: SAM build + deploy ───────────────────────────────────────────── +echo "" +echo "Step 1/4: Building and deploying SAM stack..." +cd "$INFRA_DIR" +sam build +sam deploy --config-file samconfig.toml || { + echo " (Stack is already up to date or deploy had warnings)" +} + +# Extract outputs +STACK_NAME=$(grep -oP 'stack_name\s*=\s*"\K[^"]+' samconfig.toml || echo "data-analyst-agent") +REGION=$(grep -oP 'region\s*=\s*"\K[^"]+' samconfig.toml || echo "${AWS_DEFAULT_REGION:-us-west-2}") +PROJECT_NAME=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --region "$REGION" \ + --query "Stacks[0].Parameters[?ParameterKey=='ProjectName'].ParameterValue" \ + --output text 2>/dev/null || echo "data-analyst-agent") + +echo " Stack: $STACK_NAME" +echo " Project: $PROJECT_NAME" + +# ── Step 2: Create S3 Files filesystem ───────────────────────────────────── +echo "" +echo "Step 2/4: Creating S3 Files filesystem..." + +DATA_BUCKET_ARN=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --region "$REGION" \ + --query "Stacks[0].Parameters[?ParameterKey=='DataBucketArn'].ParameterValue" \ + --output text) +S3FILES_ROLE_ARN=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --region "$REGION" \ + --query "Stacks[0].Outputs[?OutputKey=='S3FilesServiceRoleArn'].OutputValue" \ + --output text 2>/dev/null || echo "") + +# Check if filesystem already exists +EXISTING_FS=$(aws s3files list-file-systems --region "$REGION" --query "fileSystems[?bucket=='${DATA_BUCKET_ARN}'].fileSystemId | [0]" --output text 2>/dev/null || echo "") + +if [ -n "$EXISTING_FS" ] && [ "$EXISTING_FS" != "None" ]; then + FS_ID="$EXISTING_FS" + echo " Using existing filesystem: $FS_ID" +else + FS_ID=$(aws s3files create-file-system \ + --bucket "$DATA_BUCKET_ARN" \ + --role-arn "$S3FILES_ROLE_ARN" \ + --region "$REGION" \ + --query "fileSystemId" \ + --output text) + echo " Created filesystem: $FS_ID" + + # Wait for filesystem to become available + echo " Waiting for filesystem to become available..." + for i in $(seq 1 30); do + STATUS=$(aws s3files get-file-system --file-system-id "$FS_ID" --query "status" --output text 2>/dev/null || echo "creating") + if [ "$STATUS" = "available" ]; then + break + fi + sleep 10 + done + echo " Status: $STATUS" +fi + +# ── Step 3: Create mount targets ────────────────────────────────────────── +echo "" +echo "Step 3/4: Creating mount targets..." + +SUBNET_IDS=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --region "$REGION" \ + --query "Stacks[0].Parameters[?ParameterKey=='SubnetIds'].ParameterValue" \ + --output text) +SG_ID=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --region "$REGION" \ + --query "Stacks[0].Outputs[?OutputKey=='MountTargetSecurityGroup'].OutputValue" \ + --output text 2>/dev/null || echo "") + +MOUNT_DNS="" +for SUBNET in $(echo "$SUBNET_IDS" | tr ',' ' '); do + echo " Creating mount target in subnet $SUBNET..." + MT_ID=$(aws s3files create-mount-target \ + --file-system-id "$FS_ID" \ + --subnet-id "$SUBNET" \ + --security-groups "$SG_ID" \ + --query "mountTargetId" \ + --output text 2>/dev/null || echo "exists") + + if [ "$MT_ID" != "exists" ]; then + echo " Mount target: $MT_ID" + fi +done + +# Get mount target IP +MOUNT_IP=$(aws s3files list-mount-targets \ + --file-system-id "$FS_ID" \ + --query "mountTargets[0].ipv4Address" \ + --output text 2>/dev/null || echo "") + +# Construct DNS name (standard S3 Files pattern: {fs-id}.s3files.{region}.amazonaws.com) +FS_DNS="${FS_ID}.s3files.${REGION}.amazonaws.com" +echo " Filesystem DNS: $FS_DNS" +if [ -n "$MOUNT_IP" ] && [ "$MOUNT_IP" != "None" ]; then + echo " Mount target IP: $MOUNT_IP" +fi + +# ── Step 4: Update SSM parameters ───────────────────────────────────────── +echo "" +echo "Step 4/5: Updating SSM parameters..." + +aws ssm put-parameter \ + --name "/${PROJECT_NAME}/s3-files-filesystem-id" \ + --value "$FS_ID" \ + --type String \ + --overwrite + +aws ssm put-parameter \ + --name "/${PROJECT_NAME}/s3-files-mount-target-dns" \ + --value "$FS_DNS" \ + --type String \ + --overwrite + +echo " Updated /${PROJECT_NAME}/s3-files-filesystem-id = $FS_ID" +echo " Updated /${PROJECT_NAME}/s3-files-mount-target-dns = $FS_DNS" + +# ── Step 5: Create VPC egress connector ──────────────────────────────────── +echo "" +echo "Step 5/5: Creating VPC egress network connector..." +echo " (Required for MicroVM to reach S3 Files mount targets via NFS)" + +# Check if connector already exists +EXISTING_CONNECTOR=$(aws ssm get-parameter --name "/${PROJECT_NAME}/vpc-egress-connector-arn" \ + --query "Parameter.Value" --output text 2>/dev/null || echo "PLACEHOLDER-SET-BY-DEPLOY-SCRIPT") + +if [ "$EXISTING_CONNECTOR" != "PLACEHOLDER-SET-BY-DEPLOY-SCRIPT" ] && [ -n "$EXISTING_CONNECTOR" ]; then + echo " Using existing connector: $EXISTING_CONNECTOR" +else + # Create network connector operator role (if not exists) + # The connector needs subnets and SGs to provision ENIs + CONNECTOR_ARN=$(aws lambda-core create-network-connector \ + --name "${PROJECT_NAME}-vpc-egress" \ + --configuration "{\"VpcEgressConfiguration\":{\"SubnetIds\":$(echo $SUBNET_IDS | tr ',' '\n' | jq -R . | jq -s .),\"SecurityGroupIds\":[\"$SG_ID\"],\"NetworkProtocol\":\"IPv4\",\"AssociatedComputeResourceTypes\":[\"MicroVm\"]}}" \ + --query "Arn" --output text 2>/dev/null || echo "") + + if [ -n "$CONNECTOR_ARN" ]; then + echo " Created connector: $CONNECTOR_ARN" + echo " Waiting for connector to become ACTIVE (may take up to 10 min)..." + for i in $(seq 1 60); do + CONN_STATE=$(aws lambda-core get-network-connector \ + --name "${PROJECT_NAME}-vpc-egress" \ + --query "State" --output text 2>/dev/null || echo "PENDING") + if [ "$CONN_STATE" = "ACTIVE" ]; then + break + fi + sleep 10 + done + echo " Connector state: $CONN_STATE" + + aws ssm put-parameter \ + --name "/${PROJECT_NAME}/vpc-egress-connector-arn" \ + --value "$CONNECTOR_ARN" \ + --type String \ + --overwrite + else + echo " WARNING: Could not create VPC egress connector." + echo " You may need to create it manually. See README for details." + fi +fi + +# ── Done ─────────────────────────────────────────────────────────────────── +echo "" +echo "═══════════════════════════════════════════════════════════════" +echo " ✓ Infrastructure deployed!" +echo "" +echo " Webhook URL:" +WEBHOOK_URL=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" --region "$REGION" \ + --query "Stacks[0].Outputs[?OutputKey=='WebhookUrl'].OutputValue" \ + --output text) +echo " $WEBHOOK_URL" +echo "" +echo " IMPORTANT: Build the MicroVM image with:" +echo " --additional-os-capabilities '[\"ALL\"]' (enables NFS mounts)" +echo " --base-image-arn arn:aws:lambda:${REGION}:aws:microvm-image:al2023-1" +echo "" +echo " Next steps:" +echo " 1. Build MicroVM image: ./scripts/build-image.sh" +echo " 2. Upload sample data: ./scripts/upload-sample-data.sh" +echo " 3. Configure agent: python agent/setup-agent.py" +echo " 4. Run demo: ./scripts/demo.sh" +echo "═══════════════════════════════════════════════════════════════" diff --git a/examples/s3-files-data-analyst/scripts/teardown.sh b/examples/s3-files-data-analyst/scripts/teardown.sh new file mode 100755 index 0000000..97fd558 --- /dev/null +++ b/examples/s3-files-data-analyst/scripts/teardown.sh @@ -0,0 +1,85 @@ +#!/bin/bash +# teardown.sh — Remove all resources created by this example. +# +# Removes: +# - S3 Files mount targets and filesystem +# - SAM CloudFormation stack +# - Sample data from S3 (but NOT the bucket itself) +# +# Usage: +# ./scripts/teardown.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXAMPLE_DIR="$(dirname "$SCRIPT_DIR")" +INFRA_DIR="$EXAMPLE_DIR/infrastructure" + +STACK_NAME=$(grep -oP 'stack_name\s*=\s*"\K[^"]+' "$INFRA_DIR/samconfig.toml" 2>/dev/null || echo "data-analyst-agent") +REGION=$(aws configure get region || echo "us-east-2") +PROJECT_NAME=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \ + --query "Stacks[0].Parameters[?ParameterKey=='ProjectName'].ParameterValue" \ + --output text 2>/dev/null || echo "data-analyst-agent") + +echo "═══════════════════════════════════════════════════════════════" +echo " Data Analyst Agent — Teardown" +echo " Stack: $STACK_NAME" +echo "═══════════════════════════════════════════════════════════════" +echo "" + +# ── Step 1: Delete S3 Files resources ───────────────────────────────────── +echo "Step 1/3: Removing S3 Files resources..." + +FS_ID=$(aws ssm get-parameter --name "/${PROJECT_NAME}/s3-files-filesystem-id" \ + --query "Parameter.Value" --output text 2>/dev/null || echo "") + +if [ -n "$FS_ID" ] && [ "$FS_ID" != "PLACEHOLDER-SET-BY-DEPLOY-SCRIPT" ]; then + # Delete mount targets first + MOUNT_TARGETS=$(aws s3files list-mount-targets --file-system-id "$FS_ID" \ + --query "mountTargets[].mountTargetId" --output text 2>/dev/null || echo "") + + for MT in $MOUNT_TARGETS; do + echo " Deleting mount target: $MT" + aws s3files delete-mount-target --mount-target-id "$MT" 2>/dev/null || true + done + + # Wait for mount targets to be deleted + sleep 10 + + # Delete filesystem + echo " Deleting filesystem: $FS_ID" + aws s3files delete-file-system --file-system-id "$FS_ID" 2>/dev/null || true +else + echo " No filesystem found, skipping." +fi + +# ── Step 2: Remove sample data ──────────────────────────────────────────── +echo "" +echo "Step 2/3: Removing sample data from S3..." + +DATA_BUCKET_ARN=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \ + --query "Stacks[0].Parameters[?ParameterKey=='DataBucketArn'].ParameterValue" \ + --output text 2>/dev/null || echo "") +BUCKET_NAME=$(echo "$DATA_BUCKET_ARN" | sed 's|arn:aws:s3:::||') + +if [ -n "$BUCKET_NAME" ]; then + echo " Removing data/ prefix from s3://$BUCKET_NAME..." + aws s3 rm "s3://$BUCKET_NAME/data/" --recursive 2>/dev/null || true + echo " Removing outputs/ prefix..." + aws s3 rm "s3://$BUCKET_NAME/outputs/" --recursive 2>/dev/null || true + echo " (Bucket itself preserved)" +else + echo " Could not determine bucket, skipping." +fi + +# ── Step 3: Delete CloudFormation stack ──────────────────────────────────── +echo "" +echo "Step 3/3: Deleting CloudFormation stack..." +aws cloudformation delete-stack --stack-name "$STACK_NAME" +echo " Waiting for stack deletion..." +aws cloudformation wait stack-delete-complete --stack-name "$STACK_NAME" 2>/dev/null || true + +echo "" +echo "═══════════════════════════════════════════════════════════════" +echo " ✓ Teardown complete." +echo "═══════════════════════════════════════════════════════════════" diff --git a/examples/s3-files-data-analyst/scripts/upload-sample-data.sh b/examples/s3-files-data-analyst/scripts/upload-sample-data.sh new file mode 100755 index 0000000..565c043 --- /dev/null +++ b/examples/s3-files-data-analyst/scripts/upload-sample-data.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# upload-sample-data.sh — Upload sample data to the S3 bucket. +# +# Uploads the CSV files from sample-data/ to the data/ prefix in your bucket. +# After upload, these files will be visible at /mnt/s3files/data/ inside the MicroVM. +# +# Usage: +# ./scripts/upload-sample-data.sh [bucket-name] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +EXAMPLE_DIR="$(dirname "$SCRIPT_DIR")" +SAMPLE_DATA_DIR="$EXAMPLE_DIR/sample-data" + +# Get bucket name from argument or stack output +BUCKET_NAME="${1:-}" + +if [ -z "$BUCKET_NAME" ]; then + STACK_NAME=$(grep -oP 'stack_name\s*=\s*"\K[^"]+' "$EXAMPLE_DIR/infrastructure/samconfig.toml" 2>/dev/null || echo "data-analyst-agent") + DATA_BUCKET_ARN=$(aws cloudformation describe-stacks --stack-name "$STACK_NAME" \ + --query "Stacks[0].Parameters[?ParameterKey=='DataBucketArn'].ParameterValue" \ + --output text 2>/dev/null || echo "") + BUCKET_NAME=$(echo "$DATA_BUCKET_ARN" | sed 's|arn:aws:s3:::||') +fi + +if [ -z "$BUCKET_NAME" ]; then + echo "ERROR: Could not determine bucket name." + echo "Usage: $0 " + exit 1 +fi + +echo "Uploading sample data to s3://$BUCKET_NAME/data/..." + +# Upload CSVs +for file in "$SAMPLE_DATA_DIR"/*.csv; do + filename=$(basename "$file") + echo " → s3://$BUCKET_NAME/data/$filename" + aws s3 cp "$file" "s3://$BUCKET_NAME/data/$filename" +done + +echo "" +echo "✓ Sample data uploaded." +echo "" +echo "Files will be accessible in the MicroVM at:" +echo " /mnt/s3files/data/sales-q1-2026.csv" +echo " /mnt/s3files/data/sales-q2-2026.csv" diff --git a/examples/s3-files-data-analyst/tests/e2e/test_full_scenario.py b/examples/s3-files-data-analyst/tests/e2e/test_full_scenario.py new file mode 100644 index 0000000..3ed6ddd --- /dev/null +++ b/examples/s3-files-data-analyst/tests/e2e/test_full_scenario.py @@ -0,0 +1,136 @@ +"""E2E test: deploy → upload data → trigger session → verify output. + +This test requires: +- A deployed stack (run deploy.sh first) +- Valid .env credentials +- Sample data uploaded to S3 + +It creates a real Claude session and verifies the agent can read +files from S3 Files and write output back. + +Usage: + python tests/e2e/test_full_scenario.py +""" + +import os +import time +from pathlib import Path + +from dotenv import load_dotenv + +# Load environment +ENV_PATH = Path(__file__).resolve().parents[2] / ".env" +load_dotenv(ENV_PATH) + + +def test_agent_reads_s3_files(): + """Create a session, ask agent to list files, verify it sees S3 data.""" + from anthropic import AnthropicAWS + + client = AnthropicAWS() + + agent_id = os.environ["AGENT_ID"] + environment_id = os.environ["ANTHROPIC_ENVIRONMENT_ID"] + + print("Creating session...") + session = client.beta.managed_agents.sessions.create( + agent_id=agent_id, + environment_id=environment_id, + ) + print(f" Session: {session.id}") + + # Ask the agent to list files — this proves S3 Files is mounted + print("Sending request: list files in /mnt/s3files/data/") + response = client.beta.managed_agents.sessions.messages.create( + session_id=session.id, + messages=[{ + "role": "user", + "content": "List all files in /mnt/s3files/data/ and show me the first 3 lines of each CSV.", + }], + max_tokens=2048, + ) + + # Extract text from response + response_text = "" + for block in response.content: + if hasattr(block, "text"): + response_text += block.text + + print(f" Response length: {len(response_text)} chars") + + # Verify the agent saw our sample data + assert "sales-q1-2026" in response_text.lower() or "q1" in response_text.lower(), \ + f"Agent didn't find Q1 sales file. Response: {response_text[:500]}" + + assert "sales-q2-2026" in response_text.lower() or "q2" in response_text.lower(), \ + f"Agent didn't find Q2 sales file. Response: {response_text[:500]}" + + print(" ✓ Agent successfully read files from S3 Files mount") + return session.id + + +def test_agent_writes_output(): + """Ask agent to write analysis output, verify it appears in S3.""" + import boto3 + from anthropic import AnthropicAWS + + client = AnthropicAWS() + s3 = boto3.client("s3") + + agent_id = os.environ["AGENT_ID"] + environment_id = os.environ["ANTHROPIC_ENVIRONMENT_ID"] + bucket_name = os.environ["DATA_BUCKET_NAME"] + + print("\nCreating session for write test...") + session = client.beta.managed_agents.sessions.create( + agent_id=agent_id, + environment_id=environment_id, + ) + print(f" Session: {session.id}") + + # Ask agent to generate and save a report + print("Sending request: generate report") + response = client.beta.managed_agents.sessions.messages.create( + session_id=session.id, + messages=[{ + "role": "user", + "content": ( + "Read /mnt/s3files/data/sales-q1-2026.csv and write a one-paragraph " + "summary to /mnt/s3files/outputs/reports/test-summary.md" + ), + }], + max_tokens=2048, + ) + + # Wait for S3 Files sync (close-to-open consistency, typically <5s) + print(" Waiting for S3 sync...") + time.sleep(10) + + # Check if the output file appeared in S3 + try: + result = s3.get_object( + Bucket=bucket_name, + Key="outputs/reports/test-summary.md", + ) + content = result["Body"].read().decode("utf-8") + print(f" Output file found: {len(content)} chars") + assert len(content) > 10, "Output file is too short" + print(" ✓ Agent successfully wrote output to S3 via S3 Files") + except s3.exceptions.NoSuchKey: + print(" ✗ Output file not found in S3 (sync may be delayed)") + raise + + +if __name__ == "__main__": + print("=" * 60) + print(" E2E Test: Data Analyst Agent with S3 Files") + print("=" * 60) + print() + + test_agent_reads_s3_files() + test_agent_writes_output() + + print() + print("=" * 60) + print(" All E2E tests passed ✓") + print("=" * 60) diff --git a/examples/s3-files-data-analyst/tests/unit/test_mount_config.sh b/examples/s3-files-data-analyst/tests/unit/test_mount_config.sh new file mode 100755 index 0000000..259ea04 --- /dev/null +++ b/examples/s3-files-data-analyst/tests/unit/test_mount_config.sh @@ -0,0 +1,90 @@ +#!/bin/bash +# test_mount_config.sh — Unit tests for mount-s3files.sh +# +# Tests the mount script's argument validation and error handling +# WITHOUT requiring actual NFS or AWS access. + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +MOUNT_SCRIPT="$SCRIPT_DIR/../../microvm-image/worker/mount-s3files.sh" + +PASSED=0 +FAILED=0 + +assert_exit_code() { + local expected="$1" + local actual="$2" + local desc="$3" + + if [ "$actual" -eq "$expected" ]; then + echo " ✓ $desc" + PASSED=$((PASSED + 1)) + else + echo " ✗ $desc (expected exit $expected, got $actual)" + FAILED=$((FAILED + 1)) + fi +} + +echo "mount-s3files.sh — unit tests" +echo "──────────────────────────────" + +# Test 1: Missing DNS argument → exit 1 +echo "" +echo "Test: missing DNS argument" +EXIT_CODE=0 +bash "$MOUNT_SCRIPT" 2>/dev/null || EXIT_CODE=$? +assert_exit_code 1 "$EXIT_CODE" "exits with code 1 when no DNS provided" + +# Test 2: Script is executable +echo "" +echo "Test: script is executable" +if [ -x "$MOUNT_SCRIPT" ]; then + echo " ✓ script has execute permission" + PASSED=$((PASSED + 1)) +else + echo " ✗ script missing execute permission" + FAILED=$((FAILED + 1)) +fi + +# Test 3: Script uses bash strict mode +echo "" +echo "Test: bash strict mode" +if grep -q "set -euo pipefail" "$MOUNT_SCRIPT"; then + echo " ✓ uses 'set -euo pipefail'" + PASSED=$((PASSED + 1)) +else + echo " ✗ missing strict mode" + FAILED=$((FAILED + 1)) +fi + +# Test 4: NFS version is 4.1 +echo "" +echo "Test: NFS version 4.1" +if grep -q "nfsvers=4.1" "$MOUNT_SCRIPT"; then + echo " ✓ uses NFS v4.1 (required by S3 Files)" + PASSED=$((PASSED + 1)) +else + echo " ✗ wrong NFS version" + FAILED=$((FAILED + 1)) +fi + +# Test 5: Creates output directories +echo "" +echo "Test: creates output directories" +if grep -q "outputs/reports" "$MOUNT_SCRIPT" && grep -q "outputs/charts" "$MOUNT_SCRIPT"; then + echo " ✓ creates outputs/reports and outputs/charts" + PASSED=$((PASSED + 1)) +else + echo " ✗ missing output directory creation" + FAILED=$((FAILED + 1)) +fi + +# ── Summary ──────────────────────────────────────────────────────────────── +echo "" +echo "──────────────────────────────" +echo "Results: $PASSED passed, $FAILED failed" + +if [ "$FAILED" -gt 0 ]; then + exit 1 +fi diff --git a/examples/s3-files-data-analyst/tests/unit/test_payload.py b/examples/s3-files-data-analyst/tests/unit/test_payload.py new file mode 100644 index 0000000..83df507 --- /dev/null +++ b/examples/s3-files-data-analyst/tests/unit/test_payload.py @@ -0,0 +1,77 @@ +"""Unit tests for the extended payload that includes S3 Files config.""" + +import json +import sys +from pathlib import Path + +# Add the base sample's functions to the path for import +BASE_FUNCTIONS = Path(__file__).resolve().parents[4] / "src" / "functions" +sys.path.insert(0, str(BASE_FUNCTIONS)) + + +def test_payload_includes_s3_files_fields(): + """The run-hook payload should include S3 Files mount configuration.""" + # Simulate what the extended launcher would produce + # (In the real implementation, payload.py is extended to include these) + dispatch = { + "version": "1", + "session": { + "ANTHROPIC_SESSION_ID": "sesn_test123", + "ANTHROPIC_ENVIRONMENT_ID": "env_test", + "ENVIRONMENT_KEY_SECRET_ID": "arn:aws:secretsmanager:us-east-2:123:secret:test", + "AWS_REGION": "us-east-2", + # S3 Files fields (added by this example) + "S3_FILES_MOUNT_DNS": "fs-abc123.s3files.us-east-2.amazonaws.com", + "S3_FILES_MOUNT_PATH": "/mnt/s3files", + }, + } + + payload_json = json.dumps(dispatch) + parsed = json.loads(payload_json) + session = parsed["session"] + + # Verify S3 Files fields are present + assert "S3_FILES_MOUNT_DNS" in session, "Missing S3_FILES_MOUNT_DNS" + assert "S3_FILES_MOUNT_PATH" in session, "Missing S3_FILES_MOUNT_PATH" + assert session["S3_FILES_MOUNT_DNS"].endswith(".amazonaws.com"), \ + "Mount DNS should be a valid AWS endpoint" + assert session["S3_FILES_MOUNT_PATH"].startswith("/"), \ + "Mount path should be absolute" + + # Verify base fields still present + assert "ANTHROPIC_SESSION_ID" in session + assert "ANTHROPIC_ENVIRONMENT_ID" in session + assert "ENVIRONMENT_KEY_SECRET_ID" in session + assert "AWS_REGION" in session + + # Verify no secrets in payload + forbidden_keys = ["ANTHROPIC_API_KEY", "ANTHROPIC_ENVIRONMENT_KEY"] + for key in forbidden_keys: + assert key not in session, f"Secret key {key} must not appear in payload" + + print("✓ All payload assertions passed") + + +def test_payload_without_s3_files_is_valid(): + """Base payload without S3 Files fields should still be valid.""" + dispatch = { + "version": "1", + "session": { + "ANTHROPIC_SESSION_ID": "sesn_test456", + "ANTHROPIC_ENVIRONMENT_ID": "env_test", + "ENVIRONMENT_KEY_SECRET_ID": "arn:aws:secretsmanager:us-east-2:123:secret:test", + "AWS_REGION": "us-east-2", + }, + } + + session = dispatch["session"] + + # S3 Files fields are optional — worker handles gracefully + assert "S3_FILES_MOUNT_DNS" not in session + print("✓ Base payload (no S3 Files) is valid") + + +if __name__ == "__main__": + test_payload_includes_s3_files_fields() + test_payload_without_s3_files_is_valid() + print("\nAll tests passed ✓")