Add environment control package

.github/workflows/copilot-setup-steps.yml

@@ -12,12 +12,15 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "latest"
+          enable-cache: true
 
       - name: Set up Python
         run: uv python install 3.11
 
       - name: Install all dependencies
         run: uv sync --all-extras
+
+      - name: Install Playwright browsers
+        run: uv run playwright install chromium --with-deps

.github/workflows/docs-publish.yml

@@ -35,9 +35,9 @@ jobs:
           fetch-depth: 0
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "latest"
+          enable-cache: true
 
       - name: Set up Python
         run: uv python install 3.11

.github/workflows/lint.yml

@@ -9,15 +9,15 @@ on:
 jobs:
   lint:
     name: Lint & Type Check
-    runs-on: ubuntu-slim
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "latest"
+          enable-cache: true
 
       - name: Set up Python
         run: uv python install 3.11

.github/workflows/publish-pypi.yml

@@ -30,9 +30,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
-        with:
-          version: "latest"
+        uses: astral-sh/setup-uv@v7
 
       - name: Set up Python
         run: uv python install 3.11

.github/workflows/test-env-ctrl.yml

@@ -0,0 +1,43 @@
+name: Test Environment Control
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "packages/environment_control/**"
+      - "tests/integration/environment_control/**"
+      - "src/webarena_verified/environments/env_ctrl_client/**"
+  push:
+    branches: [main]
+    paths:
+      - "packages/environment_control/**"
+      - "tests/integration/environment_control/**"
+      - "src/webarena_verified/environments/env_ctrl_client/**"
+
+jobs:
+  test:
+    name: Test Environment Control
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install 3.11
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Install Playwright browsers
+        run: uv run playwright install chromium --with-deps
+
+      - name: Build Docker image
+        run: docker build -t webarena-verified:test .
+
+      - name: Run environment control tests
+        run: uv run pytest tests/integration/environment_control/ -m docker --webarena-verified-docker-img webarena-verified:test

.github/workflows/test.yml

@@ -15,18 +15,21 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Install uv
-        uses: astral-sh/setup-uv@v5
+        uses: astral-sh/setup-uv@v7
         with:
-          version: "latest"
+          enable-cache: true
 
       - name: Set up Python
         run: uv python install 3.11
 
       - name: Install dependencies
         run: uv sync --all-extras
 
+      - name: Install Playwright browsers
+        run: uv run playwright install chromium --with-deps
+
       - name: Build Docker image
         run: docker build -t webarena-verified:test .
 
       - name: Run tests
-        run: uv run pytest --webarena-verified-docker-img webarena-verified:test
+        run: uv run pytest --webarena-verified-docker-img webarena-verified:test --ignore=tests/integration/environment_control/

Dockerfile

@@ -3,7 +3,7 @@
 # ============================================================
 # Stage 1: Builder
 # ============================================================
-FROM ghcr.io/astral-sh/uv:0.9-python3.12-bookworm-slim AS builder
+FROM ghcr.io/astral-sh/uv:0.9-python3.11-bookworm-slim AS builder
 
 WORKDIR /app
 
@@ -26,7 +26,7 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 # ============================================================
 # Stage 2: Runtime (minimal)
 # ============================================================
-FROM python:3.12-slim-bookworm AS runtime
+FROM python:3.11-slim-bookworm AS runtime
 
 WORKDIR /app
 

docs/getting_started/environments.md → docs/environments/docker_images.md

@@ -1,6 +1,6 @@
-# Environments
+# Docker Images
 
-WebArena-Verified uses Docker containers to provide isolated, reproducible environments for each website in the benchmark. To improve reset time and reduce storage requirements, we provide recipes to create **slim images** - optimized versions of the original environments while keeping the exact content and functionality.
+To improve reset time and reduce storage requirements, we provide recipes to create **slim images** - optimized versions of the original environments while keeping the exact content and functionality.
 
 ## Size Improvements
 
@@ -105,11 +105,13 @@ docker exec admin-slim magento-init http://localhost:7780
    ```
 
 **Container not initializing:**
+
 - Check container logs: `docker logs admin-slim`
 - Verify MAGENTO_BASE_URL is set correctly
 - Wait ~30 seconds for initialization to complete
 
 **Database reset not working:**
+
 - Ensure archive exists: `docker exec admin-slim ls -lh /var/backups/mysql/data.tar.gz`
 - Check services status: `docker exec admin-slim supervisorctl status`
 
@@ -123,6 +125,7 @@ bash create_slim_image.sh
 ```
 
 **Note:** The script reuses existing data if available:
+
 - PostgreSQL archive (~1.6GB)
 - Optimized submission images (~6GB)
 - This saves ~30-60 minutes on subsequent runs
@@ -150,11 +153,13 @@ docker exec reddit-slim postmill-init
 ### Troubleshooting
 
 **Container not initializing:**
+
 - Check container logs: `docker logs reddit-slim`
 - Wait ~2-3 minutes for initial data extraction
 - Check initialization status: `docker exec reddit-slim cat /run/postmill.env`
 
 **Database reset not working:**
+
 - Ensure archives exist:
   ```bash
   docker exec reddit-slim ls -lh /var/backups/pgsql/data.tar.gz
@@ -164,6 +169,7 @@ docker exec reddit-slim postmill-init
 - Verify database integrity: `docker exec reddit-slim postmill-init` (shows validation output)
 
 **Patches not applied:**
+
 1. Check rate limits removed:
    ```bash
    docker exec reddit-slim grep -c '@RateLimit' /var/www/html/src/DataObject/SubmissionData.php

docs/environments/environment_control.md

@@ -0,0 +1,129 @@
+# Environment Control
+
+A lightweight utility for WebArena test environment containers that enables self-contained environment management.
+
+## Why?
+
+Without this package, setting up a test environment requires:
+
+1. Starting the Docker container
+2. Using `docker exec` to run setup commands (e.g., configuring external URLs)
+
+This approach has two main issues:
+
+1. **Version mismatch** - Setup commands may change between versions, requiring maintenance of different setup scripts for different environment versions.
+2. **Limited container access** - In some deployment scenarios (e.g., Kubernetes, managed container services), accessing the internal Docker daemon or running `docker exec` is either not allowed or difficult to configure.
+
+## How It Works
+
+The package is installed inside each test environment container and exposes environment management operations through two interfaces:
+
+- **CLI** - For direct access via `docker exec`
+- **HTTP API** - For remote access without requiring Docker access
+
+This makes each test environment site fully self-contained and provides a consistent interface across all test environments (shopping, reddit, gitlab, etc.), regardless of their underlying implementation.
+
+## Architecture
+
+```mermaid
+graph LR
+    subgraph "Host / Test Runner"
+        Client[WebArena Client]
+    end
+
+    subgraph "Shopping Container"
+        App[Shopping Site]
+        EnvCtrl[env-ctrl server :8877]
+    end
+
+    Client -->|"HTTP: /health, /reset, /config"| EnvCtrl
+    EnvCtrl -->|manages| App
+```
+
+The client library communicates with `env-ctrl` running inside each container, allowing operations like health checks, resets, and configuration changes without direct Docker access.
+
+## Dashboard
+
+The env-ctrl server includes a web-based dashboard for manual environment management. Access it by navigating to the server's root URL (e.g., `http://localhost:8877/`).
+
+![Environment Control Dashboard](../assets/env-ctrl-dashboard.png)
+
+The dashboard provides:
+
+- **Environment status** - Shows the current environment type and health status
+- **Action buttons** - Initialize, Start, Stop, Restart, and Refresh Status
+- **Console output** - Real-time feedback from operations
+
+This is useful for debugging and manual testing without needing to use the CLI or write code.
+
+## Usage
+
+### Via Docker Exec (CLI)
+
+When you have Docker access, you can use the CLI directly:
+
+```bash
+# List available environment types
+docker exec <container> env-ctrl list
+
+# Check status
+docker exec <container> env-ctrl status
+
+# Start the HTTP server
+docker exec <container> env-ctrl serve --port 8877
+```
+
+### Via HTTP API
+
+When the server is running, interact with it over HTTP:
+
+```bash
+curl http://localhost:8877/health
+curl -X POST http://localhost:8877/reset
+```
+
+### Python Client
+
+For programmatic access, use the client classes from the `webarena_verified` library:
+
+```python
+from webarena_verified.environments.env_ctrl_client import HttpClient
+
+# Connect to the env-ctrl server
+client = HttpClient(base_url="http://localhost:8877")
+
+# Check environment status
+result = client.status()
+print(result["success"])  # True if healthy
+
+# Wait for environment to be ready
+result = client.wait_until_ready(timeout=60.0)
+
+# Start/stop the environment
+client.start(wait=True)
+client.stop()
+```
+
+**Available methods:**
+
+| Method | Description |
+|--------|-------------|
+| `status()` | Get environment health status |
+| `init()` | Initialize the environment |
+| `start(wait=False)` | Start the environment |
+| `stop()` | Stop the environment |
+| `restart(wait=False)` | Restart the environment |
+| `wait_until_ready(timeout, interval)` | Poll until environment is ready |
+| `cleanup()` | Remove logs, caches, temp files |
+| `config()` | Get build configuration |
+
+**Alternative: Docker client**
+
+If you have Docker access and prefer to use `docker exec` instead of HTTP:
+
+```python
+from webarena_verified.environments.env_ctrl_client import EnvCtrlDockerClient
+
+client = EnvCtrlDockerClient(container_name="shopping-slim")
+result = client.status()
+```

docs/environments/index.md

@@ -0,0 +1,16 @@
+# Environments
+
+WebArena-Verified uses Docker containers to provide isolated, reproducible test environments for each website in the benchmark.
+
+## Overview
+
+Each test environment (Shopping, Reddit, GitLab, etc.) runs as a self-contained Docker container with:
+
+- The application and its dependencies
+- Pre-populated test data
+- Environment control utilities for management via CLI or HTTP
+
+## Sections
+
+- [Docker Images](docker_images.md) - Instructions for building and running slim Docker images
+- [Environment Control](environment_control.md) - The `environment_control` package for managing environments

mkdocs.yml

@@ -57,13 +57,18 @@ plugins:
   - redirects:
       redirect_maps:
         getting_started/index.md: index.md
+        getting_started/environments.md: environments/index.md
 
 markdown_extensions:
   - admonition
   - pymdownx.details
   - attr_list
   - md_in_html
-  - pymdownx.superfences
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
   - pymdownx.highlight:
       anchor_linenums: true
   - pymdownx.inlinehilite
@@ -94,10 +99,13 @@ nav:
           - Configuration: getting_started/configuration.md
           - Subset Manager: getting_started/subset_manager.md
           - Utilities: getting_started/utilities.md
-          - Environments: getting_started/environments.md
           - Changelog: changelog/index.md
       - Subsets:
           - WebArena-Verified Hard: getting_started/hard_subset.md
+  - Environments:
+      - Overview: environments/index.md
+      - Docker Images: environments/docker_images.md
+      - Environment Control: environments/environment_control.md
   - Evaluation:
       - Quick Start: evaluation/index.md
       - Handling of Unachievable Tasks: evaluation/handling_of_unachievable_tasks.md