diff --git a/.github/workflows/test-docker-gitlab.yml b/.github/workflows/test-docker-gitlab.yml new file mode 100644 index 0000000..75bb3b7 --- /dev/null +++ b/.github/workflows/test-docker-gitlab.yml @@ -0,0 +1,43 @@ +name: Test Docker GitLab + +on: + pull_request: + branches: [main] + paths: + - "dev/environments/docker/sites/gitlab/**" + - "tests/integration/environments/gitlab/**" + - "tests/integration/environments/conftest.py" + push: + branches: [main] + paths: + - "dev/environments/docker/sites/gitlab/**" + - "tests/integration/environments/gitlab/**" + - "tests/integration/environments/conftest.py" + +jobs: + test: + name: Test gitlab + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Set up Python + run: uv python install 3.11 + + - name: Install dependencies + run: uv sync --all-extras + + - name: Install Playwright browsers + run: uv run playwright install chromium --with-deps + + - name: Build Docker image + run: uv run invoke envs.docker.build --site=gitlab --tag=test + + - name: Run integration tests + run: uv run invoke envs.docker.test --site=gitlab --tag=test diff --git a/.github/workflows/test-docker-map.yml.disabled b/.github/workflows/test-docker-map.yml.disabled new file mode 100644 index 0000000..ac9046a --- /dev/null +++ b/.github/workflows/test-docker-map.yml.disabled @@ -0,0 +1,54 @@ +name: Test Docker Map + +on: + pull_request: + branches: [main] + paths: + - "dev/environments/docker/sites/map/**" + - "tests/integration/environments/map/**" + - "tests/integration/environments/conftest.py" + push: + branches: [main] + paths: + - "dev/environments/docker/sites/map/**" + - "tests/integration/environments/map/**" + - "tests/integration/environments/conftest.py" + +jobs: + test: + name: Test map + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Set up Python + run: uv python install 3.11 + + - name: Install dependencies + run: uv sync --all-extras + + - name: Install Playwright browsers + run: uv run playwright install chromium --with-deps + + - name: Restore map test data cache + id: map-data-cache + uses: actions/cache@v4 + with: + path: data/map + key: map-monaco-data-v1 + + - name: Generate map test data + if: steps.map-data-cache.outputs.cache-hit != 'true' + run: uv run invoke dev.ci.generate-map-data + + - name: Build Docker image + run: uv run invoke envs.docker.build --site=map --tag=test + + - name: Run integration tests + run: uv run invoke envs.docker.test --site=map --tag=test --data-dir=data/map diff --git a/.github/workflows/test-docker-reddit.yml b/.github/workflows/test-docker-reddit.yml new file mode 100644 index 0000000..1198e69 --- /dev/null +++ b/.github/workflows/test-docker-reddit.yml @@ -0,0 +1,43 @@ +name: Test Docker Reddit + +on: + pull_request: + branches: [main] + paths: + - "dev/environments/docker/sites/reddit/**" + - "tests/integration/environments/reddit/**" + - "tests/integration/environments/conftest.py" + push: + branches: [main] + paths: + - "dev/environments/docker/sites/reddit/**" + - "tests/integration/environments/reddit/**" + - "tests/integration/environments/conftest.py" + +jobs: + test: + name: Test reddit + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Set up Python + run: uv python install 3.11 + + - name: Install dependencies + run: uv sync --all-extras + + - name: Install Playwright browsers + run: uv run playwright install chromium --with-deps + + - name: Build Docker image + run: uv run invoke envs.docker.build --site=reddit --tag=test + + - name: Run integration tests + run: uv run invoke envs.docker.test --site=reddit --tag=test diff --git a/.github/workflows/test-docker-shopping-admin.yml b/.github/workflows/test-docker-shopping-admin.yml new file mode 100644 index 0000000..af194b1 --- /dev/null +++ b/.github/workflows/test-docker-shopping-admin.yml @@ -0,0 +1,43 @@ +name: Test Docker Shopping Admin + +on: + pull_request: + branches: [main] + paths: + - "dev/environments/docker/sites/shopping_admin/**" + - "tests/integration/environments/shopping_admin/**" + - "tests/integration/environments/conftest.py" + push: + branches: [main] + paths: + - "dev/environments/docker/sites/shopping_admin/**" + - "tests/integration/environments/shopping_admin/**" + - "tests/integration/environments/conftest.py" + +jobs: + test: + name: Test shopping_admin + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Set up Python + run: uv python install 3.11 + + - name: Install dependencies + run: uv sync --all-extras + + - name: Install Playwright browsers + run: uv run playwright install chromium --with-deps + + - name: Build Docker image + run: uv run invoke envs.docker.build --site=shopping_admin --tag=test + + - name: Run integration tests + run: uv run invoke envs.docker.test --site=shopping_admin --tag=test diff --git a/.github/workflows/test-docker-shopping.yml b/.github/workflows/test-docker-shopping.yml new file mode 100644 index 0000000..4bcd2d7 --- /dev/null +++ b/.github/workflows/test-docker-shopping.yml @@ -0,0 +1,43 @@ +name: Test Docker Shopping + +on: + pull_request: + branches: [main] + paths: + - "dev/environments/docker/sites/shopping/**" + - "tests/integration/environments/shopping/**" + - "tests/integration/environments/conftest.py" + push: + branches: [main] + paths: + - "dev/environments/docker/sites/shopping/**" + - "tests/integration/environments/shopping/**" + - "tests/integration/environments/conftest.py" + +jobs: + test: + name: Test shopping + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Set up Python + run: uv python install 3.11 + + - name: Install dependencies + run: uv sync --all-extras + + - name: Install Playwright browsers + run: uv run playwright install chromium --with-deps + + - name: Build Docker image + run: uv run invoke envs.docker.build --site=shopping --tag=test + + - name: Run integration tests + run: uv run invoke envs.docker.test --site=shopping --tag=test diff --git a/.github/workflows/test-docker-wikipedia.yml b/.github/workflows/test-docker-wikipedia.yml new file mode 100644 index 0000000..69c013a --- /dev/null +++ b/.github/workflows/test-docker-wikipedia.yml @@ -0,0 +1,46 @@ +name: Test Docker Wikipedia + +on: + pull_request: + branches: [main] + paths: + - "dev/environments/docker/sites/wikipedia/**" + - "tests/integration/environments/wikipedia/**" + - "tests/integration/environments/conftest.py" + push: + branches: [main] + paths: + - "dev/environments/docker/sites/wikipedia/**" + - "tests/integration/environments/wikipedia/**" + - "tests/integration/environments/conftest.py" + +jobs: + test: + name: Test wikipedia + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v7 + with: + enable-cache: true + + - name: Set up Python + run: uv python install 3.11 + + - name: Install dependencies + run: uv sync --all-extras + + - name: Install Playwright browsers + run: uv run playwright install chromium --with-deps + + - name: Download CI ZIM + run: uv run invoke dev.ci.setup-wikipedia + + - name: Build Docker image + run: uv run invoke envs.docker.build --site=wikipedia --tag=test + + - name: Run integration tests + run: uv run invoke envs.docker.test --site=wikipedia --tag=test --data-dir=data/wikipedia diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fbe7163..6f647aa 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,4 +32,4 @@ jobs: run: docker build -t webarena-verified:test . - name: Run tests - run: uv run pytest --webarena-verified-docker-img webarena-verified:test --ignore=tests/integration/environment_control/ + run: uv run pytest --webarena-verified-docker-img webarena-verified:test --ignore=tests/integration/environment_control/ --ignore=tests/integration/environments/ diff --git a/.gitignore b/.gitignore index a86ae85..f109d56 100644 --- a/.gitignore +++ b/.gitignore @@ -241,5 +241,5 @@ demo_config.json tmp/ eval_log.txt -# Generated Gatus config -dev/environments/docker/monitoring/gatus/config.yaml +# CI test data (downloaded/generated by inv dev.ci.* tasks) +data/ diff --git a/README.md b/README.md index 80a4207..d054caf 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ WebArena-Verified is the verified release of the WebArena benchmark. It distribu ## ๐Ÿ“ข Announcements +- **February 2, 2026**: Optimized Docker images for all WebArena environments are now available on [Docker Hub](https://hub.docker.com/u/am1n3e)! Images are up to 92% smaller than originals, include auto-login headers, plus a single container for Map (beta) (previously 5 separate containers). See the [Environments documentation](https://servicenow.github.io/webarena-verified/environments/). - **February 2, 2026**: WebArena-Verified is now available via Docker and uvx! Run `uvx webarena-verified --help` or `docker run am1n3e/webarena-verified:latest --help` to get started. - **January 7, 2026**: WebArena-Verified is now available on PyPI! Install it easily with `pip install webarena-verified`. - **December 2, 2025**: We are presenting WebArena-Verified at the [Scaling Environments for Agents (SEA) Workshop](https://sea-workshop.github.io/) at NeurIPS 2025 on December 7th in San Diego. Come see us! @@ -30,31 +31,25 @@ WebArena-Verified is the verified release of the WebArena benchmark. It distribu ## ๐Ÿš€ Quick Start -### Installation +### Using uvx (Recommended) -Install from PyPI: +The fastest way to try WebArena-Verified without installing anything: ```bash -pip install webarena-verified -``` - -Or for development, clone and install from source: - -```bash -git clone https://github.com/ServiceNow/webarena-verified.git -cd webarena-verified -uv sync +uvx webarena-verified --help ``` -Verify the CLI is working: +Run evaluation directly: ```bash -webarena-verified --help +uvx webarena-verified eval-tasks \ + --task-ids 108 \ + --output-dir examples/agent_logs/demo ``` -### Evaluate with Docker +### Using Docker -You can run evaluation directly using the Docker image by mounting your output directory: +Run evaluation using the Docker image by mounting your output directory: ```bash docker run --rm \ @@ -73,6 +68,83 @@ output/ โ”‚ โ””โ”€โ”€ ... ``` +### Using pip + +Install from PyPI: + +```bash +pip install webarena-verified +``` + +Verify the CLI is working: + +```bash +webarena-verified --help +``` + +For development, clone and install from source: + +```bash +git clone https://github.com/ServiceNow/webarena-verified.git +cd webarena-verified +uv sync +``` + +## ๐ŸŒ Run WebArena Environments + +### Using the CLI (Recommended) + +Start and manage WebArena environments using the built-in CLI: + +```bash +# Start a site (waits for services to be ready) +webarena-verified env start --site shopping +webarena-verified env start --site shopping_admin +webarena-verified env start --site reddit +webarena-verified env start --site gitlab + +# Check status +webarena-verified env status --site shopping + +# Stop a site +webarena-verified env stop --site shopping + +# Stop all running sites +webarena-verified env stop-all +``` + +For sites requiring data setup (Wikipedia, Map): + +```bash +# Wikipedia - download data first (~100GB) +webarena-verified env setup init --site wikipedia --data-dir ./downloads +webarena-verified env start --site wikipedia --data-dir ./downloads + +# Map - download data first (~60GB) +webarena-verified env setup init --site map --data-dir ./downloads +webarena-verified env start --site map +``` + +### Using Docker Directly + +You can also run environments directly with Docker: + +```bash +# Shopping (Magento) +docker run -d --name webarena-verified-shopping -p 7770:80 -p 7771:8877 am1n3e/webarena-verified-shopping + +# Shopping Admin +docker run -d --name webarena-verified-shopping_admin -p 7780:80 -p 7781:8877 am1n3e/webarena-verified-shopping_admin + +# Reddit (Postmill) +docker run -d --name webarena-verified-reddit -p 9999:80 -p 9998:8877 am1n3e/webarena-verified-reddit + +# GitLab +docker run -d --name webarena-verified-gitlab -p 8023:8023 -p 8024:8877 am1n3e/webarena-verified-gitlab +``` + +See the [Environments documentation](https://servicenow.github.io/webarena-verified/environments/) for detailed setup instructions, credentials, and configuration options. + ## ๐Ÿงช Evaluate A Task Evaluate a task using the CLI or programmatically: diff --git a/dev/__init__.py b/dev/__init__.py index d15764a..e69de29 100644 --- a/dev/__init__.py +++ b/dev/__init__.py @@ -1 +0,0 @@ -"""Contributing tasks and utilities.""" diff --git a/dev/ci_tasks.py b/dev/ci_tasks.py new file mode 100644 index 0000000..beaa559 --- /dev/null +++ b/dev/ci_tasks.py @@ -0,0 +1,429 @@ +"""CI helper tasks for automated testing.""" + +from __future__ import annotations + +import platform +import shutil +from pathlib import Path +from typing import TYPE_CHECKING + +from invoke import task + +if TYPE_CHECKING: + from invoke.context import Context + +from dev.utils import logging_utils +from dev.utils.path_utils import get_repo_root + + +def _get_docker_platform_flag() -> str: + """Return --platform flag if running on ARM64 (Apple Silicon) to use amd64 images.""" + if platform.machine() in ("arm64", "aarch64"): + return "--platform linux/amd64 " + return "" + + +# CI-specific ZIM file for Wikipedia tests (small ~2.7MB file) +CI_WIKIPEDIA_ZIM_URL = "https://download.kiwix.org/zim/wikipedia/wikipedia_en_ray-charles_maxi_2026-02.zim" +CI_WIKIPEDIA_ZIM_NAME = "wikipedia_en_ray-charles_maxi_2026-02.zim" + +# Monaco OSM data URL for map CI tests (~656KB) +MONACO_PBF_URL = "https://download.geofabrik.de/europe/monaco-latest.osm.pbf" +MONACO_PBF_NAME = "monaco-latest.osm.pbf" + +# OSRM routing profiles and file naming +# Directory names -> OSRM profile lua file names +OSRM_PROFILES = {"car": "car", "bike": "bicycle", "foot": "foot"} +# Use us-northeast-latest prefix to match supervisord.conf expectations +OSRM_OUTPUT_PREFIX = "us-northeast-latest" + + +@task(name="setup-wikipedia") +@logging_utils.with_banner() +def setup_wikipedia(ctx: Context, output_dir: str | None = None) -> None: + """Download tiny Wikipedia ZIM file for CI testing. + + Downloads the Ray Charles ZIM (~2.7MB) instead of the full Wikipedia (~100GB). + The ZIM file is saved to the output directory for mounting at runtime. + + Args: + output_dir: Directory to save the ZIM file (default: data/wikipedia at repo root). + """ + repo_root = get_repo_root() + default_dir = repo_root / "data" / "wikipedia" + target_dir = Path(output_dir) if output_dir else default_dir + target_dir.mkdir(parents=True, exist_ok=True) + + zim_path = target_dir / CI_WIKIPEDIA_ZIM_NAME + + if zim_path.exists(): + logging_utils.print_info(f"ZIM file already exists: {zim_path}") + return + + logging_utils.print_info("Downloading CI Wikipedia ZIM file...") + logging_utils.print_info(f" URL: {CI_WIKIPEDIA_ZIM_URL}") + logging_utils.print_info(f" To: {zim_path}") + + ctx.run(f'curl -L -o "{zim_path}" "{CI_WIKIPEDIA_ZIM_URL}"') + + logging_utils.print_success(f"Downloaded: {zim_path}") + + +# ============================================================================== +# Map CI Tasks (Monaco test data) +# ============================================================================== + + +@task(name="generate-map-data") +@logging_utils.with_banner() +def generate_map_data(ctx: Context, output_dir: str | None = None) -> None: + """Generate Monaco test data for map CI using Docker containers. + + Produces volume-ready data that can be mounted directly by the map container. + The output structure matches what the map entrypoint expects: + - database/postgres/ (PostgreSQL 15 tile data) + - nominatim/postgres/ (PostgreSQL 14 Nominatim data) + - website/postgres/ (PostgreSQL 14 OSM website data via osmosis) + - routing/{car,bike,foot}/ (OSRM routing files) + + This data can be cached in CI (GitHub Actions cache) for fast subsequent runs. + + Args: + output_dir: Directory to save generated data (default: data/map at repo root). + """ + repo_root = get_repo_root() + default_dir = repo_root / "data" / "map" + output_path = Path(output_dir).resolve() if output_dir else default_dir + output_path.mkdir(parents=True, exist_ok=True) + + pbf_path = output_path / MONACO_PBF_NAME + + # Create subdirectories matching map container volume structure + (output_path / "database").mkdir(parents=True, exist_ok=True) + (output_path / "nominatim").mkdir(parents=True, exist_ok=True) + (output_path / "website").mkdir(parents=True, exist_ok=True) + for profile in OSRM_PROFILES: + (output_path / "routing" / profile).mkdir(parents=True, exist_ok=True) + + # Step 1: Download Monaco PBF + logging_utils.print_info("Step 1/5: Downloading Monaco OSM PBF...") + if pbf_path.exists(): + logging_utils.print_info(f" PBF already exists: {pbf_path}") + else: + ctx.run(f'curl -L -o "{pbf_path}" "{MONACO_PBF_URL}"') + logging_utils.print_success(f" Downloaded: {pbf_path}") + + # Step 2: Generate tile database (PostgreSQL data directory) + logging_utils.print_info("Step 2/5: Generating tile database...") + tile_db_path = output_path / "database" / "postgres" + if tile_db_path.exists() and (tile_db_path / "PG_VERSION").exists(): + logging_utils.print_info(f" Tile database exists: {tile_db_path}") + else: + _generate_tile_database(ctx, output_path, pbf_path) + logging_utils.print_success(f" Created: {tile_db_path}") + + # Step 3: Generate OSRM routing data + logging_utils.print_info("Step 3/5: Processing OSRM routing data...") + _generate_osrm_data(ctx, output_path, pbf_path) + + # Step 4: Generate Nominatim database (PostgreSQL data directory) + logging_utils.print_info("Step 4/5: Generating Nominatim database...") + nominatim_db_path = output_path / "nominatim" / "postgres" + if nominatim_db_path.exists() and (nominatim_db_path / "PG_VERSION").exists(): + logging_utils.print_info(f" Nominatim database exists: {nominatim_db_path}") + else: + _generate_nominatim_database(ctx, output_path, pbf_path) + logging_utils.print_success(f" Created: {nominatim_db_path}") + + # Step 5: Generate website database (PostgreSQL data directory via osmosis) + logging_utils.print_info("Step 5/5: Generating website database (osmosis)...") + website_db_path = output_path / "website" / "postgres" + if website_db_path.exists() and (website_db_path / "PG_VERSION").exists(): + logging_utils.print_info(f" Website database exists: {website_db_path}") + else: + _generate_website_database(ctx, output_path, pbf_path) + logging_utils.print_success(f" Created: {website_db_path}") + + # Keep PBF file (may be needed at runtime) + logging_utils.print_info(f"Keeping PBF file: {pbf_path}") + + logging_utils.print_success(f"Monaco test data generated in: {output_path}") + logging_utils.print_info("Data structure:") + logging_utils.print_info(" database/postgres/ - Tile database (PostgreSQL 15)") + logging_utils.print_info(" nominatim/postgres/ - Nominatim database (PostgreSQL 14)") + logging_utils.print_info(" website/postgres/ - Website database (PostgreSQL 14, osmosis)") + logging_utils.print_info(" routing/{car,bike,foot}/ - OSRM routing files") + + +def _generate_tile_database(ctx: Context, output_path: Path, pbf_path: Path) -> None: + """Generate PostgreSQL data directory for tile database using the tile server's import.""" + container_name = "map-ci-tile-import" + tile_db_path = output_path / "database" / "postgres" + + # Clean up any existing data and container + if tile_db_path.exists(): + shutil.rmtree(tile_db_path) + ctx.run(f"docker rm -f {container_name} 2>/dev/null || true") + + try: + # Run tile server import - it exits when complete + # This downloads external data too (water polygons ~900MB) which takes time + logging_utils.print_info(" Running tile server import (downloads ~900MB external data)...") + ctx.run( + f"docker run --name {container_name} " + f'-v "{pbf_path}:/data/region.osm.pbf:ro" ' + f"-e THREADS=2 " + f"-e SKIP_DOWNLOAD= " + f"overv/openstreetmap-tile-server import", + timeout=1800, # 30 min timeout for download + import + ) + + # Import completed, now we need to restart to copy data + # Start container again just to access the data + logging_utils.print_info(" Restarting container to copy data...") + ctx.run(f"docker start {container_name}") + ctx.run("sleep 5") + + # Stop PostgreSQL cleanly before copying data + logging_utils.print_info(" Stopping PostgreSQL...") + ctx.run( + f"docker exec {container_name} bash -c '" + f'su - renderer -c "pg_ctl stop -D /data/database/postgres -m fast" 2>/dev/null || ' + f"pkill -INT postgres || true'", + warn=True, + ) + ctx.run("sleep 3") + + # Copy PostgreSQL data directory out of container + logging_utils.print_info(" Copying database files...") + ctx.run(f'docker cp {container_name}:/data/database/postgres "{tile_db_path}"') + + finally: + ctx.run(f"docker rm -f {container_name} 2>/dev/null || true") + + +def _generate_osrm_data(ctx: Context, output_path: Path, pbf_path: Path) -> None: + """Generate OSRM routing data for all profiles.""" + osrm_image = "ghcr.io/project-osrm/osrm-backend:v5.27.1" + # Name the PBF copy to match the expected output prefix (so osrm-extract outputs us-northeast-latest.osrm.*) + osrm_pbf_name = f"{OSRM_OUTPUT_PREFIX}.osm.pbf" + + for profile_dir_name, profile_lua_name in OSRM_PROFILES.items(): + profile_dir = output_path / "routing" / profile_dir_name + osrm_marker = profile_dir / f"{OSRM_OUTPUT_PREFIX}.osrm.mldgr" + + # Check if already processed + if osrm_marker.exists(): + logging_utils.print_info(f" {profile_dir_name}: Already processed") + continue + + logging_utils.print_info(f" {profile_dir_name}: Processing...") + + # Copy PBF with the expected output name (osrm-extract uses input filename for output) + profile_pbf = profile_dir / osrm_pbf_name + shutil.copy(pbf_path, profile_pbf) + + # Extract (use the lua profile name, not the directory name) + ctx.run( + f'docker run --rm -v "{profile_dir}:/data" {osrm_image} ' + f"osrm-extract -p /opt/{profile_lua_name}.lua /data/{osrm_pbf_name}" + ) + + # Partition + ctx.run(f'docker run --rm -v "{profile_dir}:/data" {osrm_image} osrm-partition /data/{OSRM_OUTPUT_PREFIX}.osrm') + + # Customize + ctx.run(f'docker run --rm -v "{profile_dir}:/data" {osrm_image} osrm-customize /data/{OSRM_OUTPUT_PREFIX}.osrm') + + # Clean up PBF copy and intermediate files to save space + profile_pbf.unlink(missing_ok=True) + for f in profile_dir.glob("*.osm.pbf"): + f.unlink(missing_ok=True) + + logging_utils.print_success(f" {profile_dir_name}: Done") + + +def _generate_nominatim_database(ctx: Context, output_path: Path, pbf_path: Path) -> None: + """Generate PostgreSQL data directory for Nominatim database.""" + container_name = "map-ci-nominatim-import" + nominatim_path = output_path / "nominatim" + nominatim_db_path = nominatim_path / "postgres" + + # Clean up any existing data and container + if nominatim_db_path.exists(): + shutil.rmtree(nominatim_db_path) + ctx.run(f"docker rm -f {container_name} 2>/dev/null || true") + + try: + # Start Nominatim container with import mode + # Use --platform linux/amd64 on ARM64 Macs since mediagis/nominatim has no arm64 image + platform_flag = _get_docker_platform_flag() + logging_utils.print_info(" Starting Nominatim container...") + ctx.run( + f"docker run -d --name {container_name} {platform_flag}" + f'-v "{pbf_path}:/nominatim/data.osm.pbf" ' + f"-e PBF_PATH=/nominatim/data.osm.pbf " + f"--shm-size=1g " + f"mediagis/nominatim:4.2" + ) + + # Wait for import to complete (use sudo -u nominatim for peer auth) + logging_utils.print_info(" Waiting for Nominatim import...") + ctx.run( + f"docker exec {container_name} bash -c '" + f"for i in $(seq 1 300); do " + f' if sudo -u nominatim psql -d nominatim -c "SELECT 1" >/dev/null 2>&1; then ' + f' if sudo -u nominatim psql -d nominatim -c "SELECT 1 FROM placex LIMIT 1" >/dev/null 2>&1; then ' + f' echo "Import complete"; exit 0; ' + f" fi; " + f" fi; " + f" sleep 5; " + f"done; " + f'echo "Timeout waiting for import"; exit 1\'', + timeout=1800, + ) + + # Stop PostgreSQL cleanly before copying data + logging_utils.print_info(" Stopping PostgreSQL...") + stop_cmd = ( + "sudo -u postgres pg_ctl stop -D /var/lib/postgresql/14/main -m fast || service postgresql stop || true" + ) + ctx.run(f"docker exec {container_name} bash -c '{stop_cmd}'") + + # Copy PostgreSQL data directory out of container + logging_utils.print_info(" Copying database files...") + ctx.run(f'docker cp {container_name}:/var/lib/postgresql/14/main "{nominatim_db_path}"') + + finally: + ctx.run(f"docker rm -f {container_name} 2>/dev/null || true") + + +def _generate_website_database(ctx: Context, output_path: Path, pbf_path: Path) -> None: + """Generate PostgreSQL data directory for OSM website database using Rails migrations.""" + container_name = "map-ci-website-import" + website_path = output_path / "website" + website_db_path = website_path / "postgres" + + # Same commit as used in map Dockerfile + osm_website_commit = "d4a014d3a6ca3f8f7d03528d39e4707dc256bc60" + + # Clean up any existing data and container + if website_db_path.exists(): + shutil.rmtree(website_db_path) + ctx.run(f"docker rm -f {container_name} 2>/dev/null || true") + + try: + # Use --platform linux/amd64 on ARM64 Macs + platform_flag = _get_docker_platform_flag() + + # Start PostgreSQL 14 with PostGIS using default postgres superuser + # The map container entrypoint expects the postgres user to exist + logging_utils.print_info(" Starting PostgreSQL 14 with PostGIS...") + ctx.run( + f"docker run -d --name {container_name} {platform_flag}-e POSTGRES_PASSWORD=postgres postgis/postgis:14-3.3" + ) + + # Wait for PostgreSQL to be ready + logging_utils.print_info(" Waiting for PostgreSQL to be ready...") + ctx.run( + f"docker exec {container_name} bash -c '" + f"for i in $(seq 1 30); do " + f" pg_isready -U postgres >/dev/null 2>&1 && exit 0; " + f" sleep 2; " + f"done; exit 1'", + timeout=120, + ) + + # Create openstreetmap user and database (as required by OSM website) + logging_utils.print_info(" Creating openstreetmap user and database...") + ctx.run( + f"docker exec {container_name} psql -U postgres -c " + f"\"CREATE USER openstreetmap WITH PASSWORD 'openstreetmap' CREATEDB;\"" + ) + ctx.run( + f'docker exec {container_name} psql -U postgres -c "CREATE DATABASE openstreetmap OWNER openstreetmap;"' + ) + ctx.run( + f"docker exec {container_name} psql -U postgres -d openstreetmap -c " + f'"CREATE EXTENSION IF NOT EXISTS postgis;"' + ) + ctx.run( + f"docker exec {container_name} psql -U postgres -d openstreetmap -c " + f'"CREATE EXTENSION IF NOT EXISTS btree_gist;"' + ) + + # Install Ruby and dependencies for Rails migrations + logging_utils.print_info(" Installing Ruby and dependencies...") + ctx.run(f"docker exec {container_name} apt-get update -qq") + ctx.run( + f"docker exec {container_name} apt-get install -y -qq " + f"ruby ruby-dev build-essential libpq-dev git libxml2-dev libxslt-dev " + f"libyaml-dev libffi-dev libssl-dev zlib1g-dev nodejs npm libarchive-dev" + ) + ctx.run(f"docker exec {container_name} gem install bundler -v 2.4.22 --no-document") + + # Clone OSM website at specific commit + logging_utils.print_info(f" Cloning OSM website (commit {osm_website_commit[:8]})...") + ctx.run( + f"docker exec {container_name} git clone https://github.com/openstreetmap/openstreetmap-website.git /app" + ) + ctx.run(f"docker exec -w /app {container_name} git checkout {osm_website_commit}") + + # Configure database.yml + logging_utils.print_info(" Configuring database...") + ctx.run( + f"docker exec {container_name} bash -c '" + f"cat > /app/config/database.yml << EOF\n" + f"production:\n" + f" adapter: postgresql\n" + f" database: openstreetmap\n" + f" username: openstreetmap\n" + f" password: openstreetmap\n" + f" host: localhost\n" + f" encoding: utf8\n" + f"EOF'" + ) + + # Configure storage.yml for Active Storage (required by migrations) + logging_utils.print_info(" Configuring Active Storage...") + ctx.run( + f"docker exec {container_name} bash -c '" + f"cat > /app/config/storage.yml << EOF\n" + f"local:\n" + f" service: Disk\n" + f' root: <%%= Rails.root.join("storage") %%>\n' + f"EOF'" + ) + + # Install bundle dependencies (minimal for migrations) + logging_utils.print_info(" Installing bundle dependencies...") + ctx.run( + f"docker exec -w /app {container_name} bundle config set --local without 'development test'", + ) + ctx.run( + f"docker exec -w /app {container_name} bundle install", + timeout=600, + ) + + # Run Rails migrations to create the schema + logging_utils.print_info(" Running Rails migrations...") + ctx.run( + f"docker exec -w /app -e RAILS_ENV=production {container_name} bundle exec rails db:migrate", + timeout=300, + ) + + # Stop PostgreSQL cleanly before copying data + logging_utils.print_info(" Stopping PostgreSQL...") + ctx.run( + f"docker exec {container_name} bash -c '" + f'su postgres -c "pg_ctl stop -D /var/lib/postgresql/data -m fast" || true\'', + warn=True, + ) + ctx.run("sleep 3") + + # Copy PostgreSQL data directory out of container + logging_utils.print_info(" Copying database files...") + ctx.run(f'docker cp {container_name}:/var/lib/postgresql/data "{website_db_path}"') + + finally: + ctx.run(f"docker rm -f {container_name} 2>/dev/null || true") diff --git a/dev/code_tasks.py b/dev/code_tasks.py new file mode 100644 index 0000000..e9fd3af --- /dev/null +++ b/dev/code_tasks.py @@ -0,0 +1,20 @@ +"""Code quality tasks (linting, formatting, type checking).""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from invoke.tasks import task + +if TYPE_CHECKING: + from invoke.context import Context + + +@task(name="format") +def format_and_check(c: Context) -> None: + """Format code using ruff and run type checking.""" + c.run("uv run ruff check src dev --fix --unsafe-fixes") + c.run("uv run ruff format src dev") + c.run("uv run ty check src dev") + # Verify environment_control package is Python 3.9 compatible + c.run("uv run vermin -t=3.9 --eval-annotations --no-tips packages/environment_control/environment_control/") diff --git a/dev/data_tasks.py b/dev/data_tasks.py new file mode 100644 index 0000000..1850cf0 --- /dev/null +++ b/dev/data_tasks.py @@ -0,0 +1,150 @@ +"""Dataset formatting tasks.""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import TYPE_CHECKING + +from compact_json import EolStyle, Formatter +from invoke.tasks import task + +if TYPE_CHECKING: + from invoke.context import Context + +# Dataset path +DATASET_FILE = Path("assets/dataset/webarena-verified.json") + + +@task(name="format") +def format_json(c: Context) -> None: + """Format dataset JSON file.""" + data = load_json(DATASET_FILE) + save_json(DATASET_FILE, data) + + +def load_json(file_path: Path) -> list[dict]: + """Load JSON file. + + Args: + file_path: Path to JSON file + + Returns: + Parsed JSON data + + Raises: + FileNotFoundError: If file doesn't exist + ValueError: If file is invalid JSON + """ + if not file_path.exists(): + raise FileNotFoundError(f"File not found: {file_path}") + + try: + with open(file_path) as f: + data = json.load(f) + except json.JSONDecodeError as e: + raise ValueError(f"Invalid JSON in {file_path}: {e}") from e + + if not isinstance(data, list): + raise ValueError(f"Expected JSON array in {file_path}, got {type(data).__name__}") + + return data + + +def check_git_status(file_path: Path) -> bool: + """Check if the file has uncommitted changes in git. + + Args: + file_path: Path to the file to check + + Returns: + True if file is safe to modify (not tracked or no uncommitted changes) + False if file has uncommitted changes + """ + try: + # Check if file is tracked by git + result = subprocess.run( + ["git", "ls-files", "--error-unmatch", str(file_path)], + capture_output=True, + text=True, + check=False, + ) + + # If file is not tracked, it's safe to modify + if result.returncode != 0: + return True + + # File is tracked, check for uncommitted changes + result = subprocess.run( + ["git", "status", "--porcelain", str(file_path)], + capture_output=True, + text=True, + check=True, + ) + + # If output is empty, no uncommitted changes + return len(result.stdout.strip()) == 0 + + except subprocess.CalledProcessError: + # If git command fails, assume it's safe (not a git repo) + return True + + +def save_json(file_path: Path, data: list[dict], skip_git_check: bool = False) -> None: + """Save JSON data to file with consistent formatting and key ordering. + + Args: + file_path: Path to output file + data: List of task objects + skip_git_check: Skip git status check + + Raises: + ValueError: If file has uncommitted changes in git + """ + # Check git status before modifying file + if not skip_git_check and not check_git_status(file_path): + raise ValueError( + f"{file_path} has uncommitted changes. Please commit or stash them first before running transforms." + ) + + # Define the desired key order + key_order = [ + "sites", + "task_id", + "intent_template_id", + "start_url", + "start_urls", + "intent", + "intent_template", + "instantiation_dict", + "retrieved_data_format_spec", + "start_url_context", + "eval", + "revision", + ] + + # Reorder keys in each task + ordered_data = [] + for task_data in data: + ordered_task = {} + # Add keys in the specified order if they exist + for key in key_order: + if key in task_data: + ordered_task[key] = task_data[key] + # Add any remaining keys in alphabetical order + remaining_keys = sorted(set(task_data.keys()) - set(key_order)) + for key in remaining_keys: + ordered_task[key] = task_data[key] + ordered_data.append(ordered_task) + + # Configure compact_json formatter + formatter = Formatter() + formatter.indent_spaces = 2 + formatter.max_inline_complexity = 10 + formatter.json_eol_style = EolStyle.LF + formatter.omit_trailing_whitespace = True + + # Ensure directory exists and write formatted output + file_path.parent.mkdir(parents=True, exist_ok=True) + formatter.dump(ordered_data, output_file=str(file_path)) diff --git a/dev/docs_tasks.py b/dev/docs_tasks.py new file mode 100644 index 0000000..d22f0ce --- /dev/null +++ b/dev/docs_tasks.py @@ -0,0 +1,66 @@ +"""Documentation tasks.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from invoke.tasks import task + +if TYPE_CHECKING: + from invoke.context import Context + + +@task +def serve(c: Context) -> None: + """Serve the documentation locally with live reload.""" + c.run("uv run mkdocs serve") + + +@task +def build(c: Context) -> None: + """Build the documentation site.""" + c.run("uv run mkdocs build") + + +@task +def deploy(c: Context) -> None: + """Deploy documentation to GitHub Pages. + + Safety checks: + - Ensures current branch is main + - Ensures local main is up-to-date with remote main + """ + # Check current branch + result = c.run("git branch --show-current", hide=True) + assert result is not None + current_branch = result.stdout.strip() + + if current_branch != "main": + print(f"ERROR: Cannot deploy docs from branch '{current_branch}'") + print("You must be on the 'main' branch to deploy documentation.") + raise SystemExit(1) + + # Fetch remote to get latest state + print("Fetching remote updates...") + c.run("git fetch origin main", hide=True) + + # Check if local main matches remote main + local_result = c.run("git rev-parse main", hide=True) + remote_result = c.run("git rev-parse origin/main", hide=True) + assert local_result is not None + assert remote_result is not None + local_commit = local_result.stdout.strip() + remote_commit = remote_result.stdout.strip() + + if local_commit != remote_commit: + print("ERROR: Local 'main' branch does not match remote 'origin/main'") + print(f" Local: {local_commit}") + print(f" Remote: {remote_commit}") + print("\nPlease either:") + print(" - Pull latest changes: git pull origin main") + print(" - Push your changes: git push origin main") + raise SystemExit(1) + + print("โœ“ Safety checks passed: on main branch and in sync with remote") + print("Deploying documentation to GitHub Pages...") + c.run("uv run mkdocs gh-deploy") diff --git a/dev/env_tasks.py b/dev/env_tasks.py new file mode 100644 index 0000000..0a6d3b9 --- /dev/null +++ b/dev/env_tasks.py @@ -0,0 +1,17 @@ +"""Development environment setup tasks.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from invoke.tasks import task + +if TYPE_CHECKING: + from invoke.context import Context + + +@task +def init(c: Context) -> None: + """Initialize development environment: sync all dependencies and install pre-commit hooks.""" + c.run("uv sync --all-extras") + c.run("uv run pre-commit install") diff --git a/dev/environments/README.md b/dev/environments/README.md new file mode 100644 index 0000000..9763cde --- /dev/null +++ b/dev/environments/README.md @@ -0,0 +1,196 @@ +# Docker Image Management + +Tools for creating and managing optimized Docker images for WebArena sites. + +## Quick Start + +```bash +# List available sites +inv envs.sites + +# Pull and start a site +inv envs.docker.pull --site shopping_admin +inv envs.docker.start --site shopping_admin +inv envs.docker.check --site shopping_admin + +# Stop when done +inv envs.docker.stop --site shopping_admin +``` + +## Creating Base Images + +Build optimized base images for sites. This applies patches, runs cleanup, and squashes layers. + +### Shopping Admin + +```bash +inv envs.docker.pull --site shopping_admin --original +inv envs.docker.create-base-img --site shopping_admin + +# Test locally +inv envs.docker.start --site shopping_admin +inv envs.docker.check --site shopping_admin + +# Publish (tag must be semver, e.g., 1.0.0) +inv envs.docker.publish --site shopping_admin --tag 1.0.0 +``` + +### Reddit + +```bash +inv envs.docker.pull --site reddit --original +inv envs.docker.create-base-img --site reddit + +# Test locally +inv envs.docker.start --site reddit +inv envs.docker.check --site reddit + +# Publish +inv envs.docker.publish --site reddit --tag 1.0.0 +``` + +### GitLab + +```bash +inv envs.docker.pull --site gitlab --original +inv envs.docker.create-base-img --site gitlab + +# Test locally +inv envs.docker.start --site gitlab +inv envs.docker.check --site gitlab + +# Publish +inv envs.docker.publish --site gitlab --tag 1.0.0 +``` + +--- + +## Available Sites + +| Site | Default Port | Env-Ctrl Port | Image | +|------|--------------|---------------|-------| +| `shopping_admin` | 7780 | 7781 | `am1n3e/webarena-verified-shopping_admin` | +| `shopping` | 7770 | 7771 | `am1n3e/webarena-verified-shopping` | +| `gitlab` | 8023 | 8024 | `am1n3e/webarena-verified-gitlab` | +| `reddit` | 9999 | 9998 | `am1n3e/webarena-verified-reddit` | +| `wikipedia` | 8888 | 8889 | `am1n3e/webarena-verified-wikipedia` | +| `map` | 3030 | 3031 | `am1n3e/webarena-verified-map` | + +## Command Reference + +### Container Lifecycle + +```bash +inv envs.docker.start --site # Start container +inv envs.docker.start --site --original # Start with original image +inv envs.docker.start --site --port 8080 # Custom port +inv envs.docker.stop --site # Stop and remove container +inv envs.docker.check --site # Health check +``` + +### Image Management + +```bash +inv envs.docker.pull --site # Pull from Docker Hub +inv envs.docker.pull --site --original # Download original tar file +inv envs.docker.build --site # Build from Dockerfile +inv envs.docker.create-base-img --site # Create optimized base image +inv envs.docker.publish --site --tag 1.0.0 # Push to Docker Hub +``` + +### Testing + +```bash +inv envs.docker.test --site # Run integration tests +inv envs.docker.test --site --headed # Run with visible browser +``` + +## Base Image Pipeline + +The `create-base-img` command creates optimized images: + +``` +Original Image โ†’ Start Container โ†’ Run Setup Scripts โ†’ Squash โ†’ Base Image +``` + +Setup scripts in `sites//scripts/` run in numeric order: + +| Script | Purpose | +|--------|---------| +| `00_apply_patches.sh` | Bootstrap env-ctrl, copy entrypoint, apply patches | +| `10_post_patch.sh` | Post-patch configuration (optional) | +| `20_optimize.sh` | Site-specific optimizations (optional) | +| `60_cleanup.sh` | Remove logs, caches, temp files | +| `90_verify.sh` | Verify setup completed correctly | + +All patching is done via shell scripts with explicit `cp` commands - no Python patching at runtime. + +## Directory Structure + +``` +dev/environments/ +โ”œโ”€โ”€ settings.py # Site registry (ports, images, paths) +โ”œโ”€โ”€ tasks.py # Top-level tasks (envs.sites) +โ””โ”€โ”€ docker/ + โ”œโ”€โ”€ tasks.py # Docker tasks (envs.docker.*) + โ”œโ”€โ”€ sites/ # Site-specific configs + โ”‚ โ”œโ”€โ”€ shopping_admin/ + โ”‚ โ”‚ โ”œโ”€โ”€ Dockerfile + โ”‚ โ”‚ โ”œโ”€โ”€ docker_overrides/ # Patch files + โ”‚ โ”‚ โ””โ”€โ”€ scripts/ # Setup scripts + โ”‚ โ”œโ”€โ”€ shopping/ + โ”‚ โ”œโ”€โ”€ reddit/ + โ”‚ โ”œโ”€โ”€ gitlab/ + โ”‚ โ”œโ”€โ”€ wikipedia/ + โ”‚ โ””โ”€โ”€ map/ + โ””โ”€โ”€ utils/ # Shared utilities + โ”œโ”€โ”€ containers.py # Container operations + โ”œโ”€โ”€ create_base_img.py + โ”œโ”€โ”€ downloads.py + โ”œโ”€โ”€ dockerfile.py + โ””โ”€โ”€ sites.py +``` + +## In-Container Operations (env-ctrl) + +The `env-ctrl` CLI runs inside containers for runtime operations: + +```bash +env-ctrl init --base-url http://localhost:7780/ # Set base URL +env-ctrl start --wait # Start services +env-ctrl stop # Stop services +env-ctrl status # Health check +env-ctrl serve # Start REST API server +``` + +### Architecture + +``` +packages/environment_control/ +โ”œโ”€โ”€ cli.py # CLI entry point +โ”œโ”€โ”€ server/app.py # REST API server +โ””โ”€โ”€ ops/ + โ”œโ”€โ”€ base.py # BaseOps (abstract base class) + โ”œโ”€โ”€ mixins/ + โ”‚ โ””โ”€โ”€ supervisor.py # SupervisorMixin + โ””โ”€โ”€ sites/ + โ”œโ”€โ”€ shopping_admin.py # ShoppingAdminOps + โ”œโ”€โ”€ shopping.py # ShoppingOps + โ”œโ”€โ”€ reddit.py # RedditOps + โ”œโ”€โ”€ gitlab.py # GitlabOps + โ”œโ”€โ”€ wikipedia.py # WikipediaOps + โ””โ”€โ”€ map.py # MapOps +``` + +Site ops classes inherit from `BaseOps` and optionally `SupervisorMixin`: +- `_init()` - Set base URL, configure site +- `_start()` / `_stop()` - Manage services +- `_get_health()` - Check service health + +## Environment Variables + +Override settings with `WA_DEV__` prefix: +```bash +WA_DEV__SHOPPING_ADMIN__PORT=7777 +WA_DEV__SHOPPING_ADMIN__HOSTNAME=myhost.local +``` diff --git a/src/webarena_verified/environments/patches/reddit/__init__.py b/dev/environments/__init__.py similarity index 100% rename from src/webarena_verified/environments/patches/reddit/__init__.py rename to dev/environments/__init__.py diff --git a/dev/environments/docker/README.md b/dev/environments/docker/README.md new file mode 100644 index 0000000..a410808 --- /dev/null +++ b/dev/environments/docker/README.md @@ -0,0 +1,67 @@ +# Docker Site Optimizations + +This directory contains optimized Docker images for WebArena sites. Each site has patches and configurations that fix issues with the original images. + +## What Are Optimizations? + +The original WebArena Docker images have various issues: +- **Data integrity bugs** - Vote counts reset, scores miscalculated +- **Testing friction** - Manual login required, rate limits block automation +- **Network issues** - Private network restrictions break internal requests +- **Missing tooling** - No runtime control or health checks + +Our optimized images fix these issues while preserving the original functionality. + +## How Optimizations Are Applied + +### Base Image Pipeline + +``` +Original Image โ†’ Start Container โ†’ Run Setup Scripts โ†’ Squash โ†’ Base Image +``` + +1. **Pull original image** - Download the unmodified WebArena image +2. **Start container** - Run container from original image +3. **Apply patches** - Copy override files and run setup scripts +4. **Squash layers** - Create single-layer optimized image +5. **Publish** - Push to Docker Hub + +### Setup Scripts + +Each site has scripts in `sites//scripts/` that run in order: + +| Script | Purpose | +|--------|---------| +| `00_apply_patches.sh` | Copy override files, install env-ctrl | +| `10_post_patch.sh` | Post-patch commands (compile, enable modules) | +| `20_optimize.sh` | Site-specific optimizations | +| `60_cleanup.sh` | Remove logs, caches, temp files | +| `90_verify.sh` | Verify patches applied correctly | + +### Override Files + +Patch files live in `sites//docker_overrides/`. These are copied into the container during `00_apply_patches.sh`. + +## Runtime Control (env-ctrl) + +All optimized images include the `env-ctrl` CLI for runtime operations: + +```bash +env-ctrl init --base-url http://localhost:7780/ # Configure base URL +env-ctrl start --wait # Start services +env-ctrl stop # Stop services +env-ctrl status # Health check +``` + +## Sites + +| Site | Port | Fixes | +|------|------|-------| +| [shopping_admin](sites/shopping_admin/) | 7780 | Header auth, mass action protection | +| [shopping](sites/shopping/) | 7770 | Header auth for customers | +| [reddit](sites/reddit/) | 9999 | Vote system, header auth, URL rewriting, rate limits | +| [gitlab](sites/gitlab/) | 8023 | - | +| [wikipedia](sites/wikipedia/) | 8888 | - | +| [map](sites/map/) | 3030 | Combined image with tiles + routing | + +See each site's README for specific fixes. diff --git a/dev/environments/docker/__init__.py b/dev/environments/docker/__init__.py new file mode 100644 index 0000000..b8ea199 --- /dev/null +++ b/dev/environments/docker/__init__.py @@ -0,0 +1,5 @@ +"""Docker tasks sub-namespace for envs.docker.*""" + +from .tasks import ns + +__all__ = ["ns"] diff --git a/dev/environments/docker/sites/gitlab/Dockerfile b/dev/environments/docker/sites/gitlab/Dockerfile new file mode 100644 index 0000000..8ca4acd --- /dev/null +++ b/dev/environments/docker/sites/gitlab/Dockerfile @@ -0,0 +1,43 @@ +# GitLab Environment - Final Image +# +# Base image created by: ./create_optimized_image.sh +# This Dockerfile adds env-ctrl on top of the optimized base. +# +# Build (from repo root): +# docker build -t gitlab:latest -f dev/environments/docker/sites/gitlab/Dockerfile . +# +# Run: +# docker run -d -p 8023:8023 -p 8877:8877 gitlab:latest + +ARG BASE_IMAGE=am1n3e/webarena-verified-gitlab-base:latest +FROM ${BASE_IMAGE} +ARG ENV_CTRL_PORT=8877 + +# Environment control configuration +ENV WA_ENV_CTRL_TYPE=gitlab +ENV WA_ENV_CTRL_AVAILABLE=true +ENV WA_ENV_CTRL_ENABLE=true +ENV WA_ENV_CTRL_PORT=${ENV_CTRL_PORT} +ENV WA_ENV_CTRL_EXTERNAL_SITE_URL=http://localhost:8023 +ENV WA_CONTAINER_PORT=8023 + +# Copy environment_control package and env-ctrl CLI +COPY packages/environment_control/environment_control /usr/local/environment_control +COPY --chmod=755 packages/environment_control/environment_control/env-ctrl /usr/local/bin/env-ctrl + +# Symlink GitLab's embedded python3 so env-ctrl can find it +RUN ln -sf /opt/gitlab/embedded/bin/python3 /usr/local/bin/python3 + +# Copy GitLab configuration variants (small ~3GB, default ~6GB, large ~10GB) +COPY dev/environments/docker/sites/gitlab/docker_overrides/gitlab.rb /opt/gitlab-configs/gitlab.rb +COPY dev/environments/docker/sites/gitlab/docker_overrides/gitlab.small.rb /opt/gitlab-configs/gitlab.small.rb +COPY dev/environments/docker/sites/gitlab/docker_overrides/gitlab.large.rb /opt/gitlab-configs/gitlab.large.rb + +# Copy entrypoint that starts env-ctrl independently +COPY --chmod=755 dev/environments/docker/sites/gitlab/docker_overrides/entrypoint.sh /opt/entrypoint.sh + +# Use custom entrypoint +ENTRYPOINT ["/opt/entrypoint.sh"] + +# Expose ports (gitlab: 8023, env-ctrl) +EXPOSE 8023 ${ENV_CTRL_PORT} diff --git a/dev/environments/docker/sites/gitlab/docker_overrides/entrypoint.sh b/dev/environments/docker/sites/gitlab/docker_overrides/entrypoint.sh new file mode 100644 index 0000000..69f955d --- /dev/null +++ b/dev/environments/docker/sites/gitlab/docker_overrides/entrypoint.sh @@ -0,0 +1,103 @@ +#!/bin/bash +set -euo pipefail + +# GitLab Slim Image Entrypoint +# Manages runsvdir and GitLab services directly (bypasses /assets/wrapper). +# Note: gitlab-ctl reconfigure is run at build time, not here. + +SLEEP_INTERVAL="${SLEEP_INTERVAL:-5}" +MAX_WAIT_ATTEMPTS=60 + +# Default external URL (baked at build time) +DEFAULT_EXTERNAL_URL="http://localhost:8023" +WA_ENV_CTRL_EXTERNAL_SITE_URL="${WA_ENV_CTRL_EXTERNAL_SITE_URL:-$DEFAULT_EXTERNAL_URL}" +WA_ENV_CTRL_SKIP_RECONFIGURE="${WA_ENV_CTRL_SKIP_RECONFIGURE:-false}" + +RUNSVDIR_PID="" + +cleanup() { + echo "Shutting down..." + gitlab-ctl stop 2>/dev/null || true + [ -n "$RUNSVDIR_PID" ] && kill "$RUNSVDIR_PID" 2>/dev/null || true +} +trap cleanup SIGTERM SIGINT + +echo "Starting runsvdir..." +/opt/gitlab/embedded/bin/runsvdir-start & +RUNSVDIR_PID=$! + +# Wait for runsvdir to be fully ready +echo "Waiting for runsvdir to be ready..." +for i in $(seq 1 $MAX_WAIT_ATTEMPTS); do + if gitlab-ctl status >/dev/null 2>&1; then + echo "runsvdir is ready (attempt $i)" + break + fi + sleep "$SLEEP_INTERVAL" +done +if ! gitlab-ctl status >/dev/null 2>&1; then + echo "ERROR: runsvdir failed to start" + exit 1 +fi + +# Reconfigure if external URL differs from baked default +if [ "$WA_ENV_CTRL_SKIP_RECONFIGURE" = "true" ]; then + echo "Skipping reconfigure: WA_ENV_CTRL_SKIP_RECONFIGURE=true" +elif [ "$WA_ENV_CTRL_EXTERNAL_SITE_URL" = "$DEFAULT_EXTERNAL_URL" ]; then + echo "Skipping reconfigure: external URL matches default ($DEFAULT_EXTERNAL_URL)" +else + echo "Reconfigure required: external URL changed from $DEFAULT_EXTERNAL_URL to $WA_ENV_CTRL_EXTERNAL_SITE_URL" + + # Update gitlab.rb with new external_url + echo "Updating external_url in /etc/gitlab/gitlab.rb..." + sed -i "s|external_url 'http://localhost:8023'|external_url '$WA_ENV_CTRL_EXTERNAL_SITE_URL'|" /etc/gitlab/gitlab.rb + + # Extract port from URL and update nginx listen_port + NEW_PORT=$(echo "$WA_ENV_CTRL_EXTERNAL_SITE_URL" | sed -n 's|.*:\([0-9]*\)$|\1|p') + if [ -n "$NEW_PORT" ]; then + echo "Updating nginx listen_port to $NEW_PORT..." + sed -i "s|nginx\['listen_port'\] = 8023|nginx['listen_port'] = $NEW_PORT|" /etc/gitlab/gitlab.rb + fi + + echo "Running gitlab-ctl reconfigure..." + gitlab-ctl reconfigure + echo "Reconfigure completed" +fi + +# Start env-ctrl with auto-restart (if enabled) +if [ "${WA_ENV_CTRL_ENABLE:-}" = "true" ]; then + echo "Running env-ctrl init..." + if ! /usr/local/bin/env-ctrl init; then + echo "ERROR: env-ctrl init failed" + exit 1 + fi + + echo "Starting env-ctrl on port ${WA_ENV_CTRL_PORT:-8877}..." + ( + while true; do + /usr/local/bin/env-ctrl serve --port "${WA_ENV_CTRL_PORT:-8877}" || true + echo "env-ctrl exited, restarting in ${SLEEP_INTERVAL}s..." + sleep "$SLEEP_INTERVAL" + done + ) & + ENV_CTRL_PID=$! + sleep "$SLEEP_INTERVAL" + if ! kill -0 $ENV_CTRL_PID 2>/dev/null; then + echo "ERROR: env-ctrl failed to start" + exit 1 + fi +fi + +# Start any services that aren't running (reconfigure already done at build time) +echo "Starting GitLab services..." +if ! gitlab-ctl start; then + echo "ERROR: Failed to start GitLab services" + exit 1 +fi + +echo "GitLab is ready" + +# Keep container running by waiting on runsvdir +wait $RUNSVDIR_PID +echo "runsvdir exited unexpectedly" +exit 1 diff --git a/dev/environments/docker/sites/gitlab/docker_overrides/gitlab.large.rb b/dev/environments/docker/sites/gitlab/docker_overrides/gitlab.large.rb new file mode 100644 index 0000000..2cd0e82 --- /dev/null +++ b/dev/environments/docker/sites/gitlab/docker_overrides/gitlab.large.rb @@ -0,0 +1,64 @@ +# Performance GitLab configuration for responsive testing +# Target: ~8-10GB RAM +# Use case: Multi-user testing, faster CI, complex workflows + +# External URL - Set to generic internal URL +# For K8s deployments, the actual external URL is handled by Ingress +external_url 'http://localhost:8023' + +# Allow GitLab to be accessed from any hostname (important for K8s/proxy environments) +nginx['listen_addresses'] = ['0.0.0.0'] +nginx['listen_port'] = 8023 + +# Trust proxy headers from reverse proxies (K8s Ingress, nginx, etc.) +gitlab_rails['trusted_proxies'] = [ + '10.0.0.0/8', # Private network + '172.16.0.0/12', # Private network + '192.168.0.0/16', # Private network + '127.0.0.1', # Localhost +] + +# Disable production monitoring services to reduce resource usage +prometheus_monitoring['enable'] = false +alertmanager['enable'] = false +gitlab_exporter['enable'] = false +postgres_exporter['enable'] = false +redis_exporter['enable'] = false +gitlab_kas['enable'] = false + +# Puma configuration (~1.5GB) +puma['worker_processes'] = 6 +puma['min_threads'] = 2 +puma['max_threads'] = 8 + +# Sidekiq - high concurrency for fast background jobs (~1GB) +sidekiq['max_concurrency'] = 25 + +# Disable Grafana (not needed for testing) +grafana['enable'] = false + +# Disable node_exporter (not needed for testing) +node_exporter['enable'] = false + +# PostgreSQL - generous allocation (~3GB) +postgresql['max_connections'] = 100 +postgresql['shared_buffers'] = '1GB' +postgresql['work_mem'] = '64MB' +postgresql['effective_cache_size'] = '2GB' +postgresql['maintenance_work_mem'] = '256MB' + +# Redis - generous for caching (~1GB) +redis['maxclients'] = 500 +redis['maxmemory'] = '1gb' +redis['maxmemory_policy'] = 'allkeys-lru' + +# Disable usage statistics +gitlab_rails['usage_ping_enabled'] = false +gitlab_rails['seat_link_enabled'] = false + +# Disable Gitaly backup (reduces reconfigure time) +gitaly['configuration'] = { + backup: { + go_cloud_url: '', + }, +} diff --git a/dev/environments/docker/sites/gitlab/docker_overrides/gitlab.rb b/dev/environments/docker/sites/gitlab/docker_overrides/gitlab.rb new file mode 100644 index 0000000..4a0fadc --- /dev/null +++ b/dev/environments/docker/sites/gitlab/docker_overrides/gitlab.rb @@ -0,0 +1,64 @@ +# Optimized GitLab configuration for development/testing +# This file can be mounted or copied into the container before first start + +# External URL - Set to generic internal URL +# For K8s deployments, the actual external URL is handled by Ingress +external_url 'http://localhost:8023' + +# Allow GitLab to be accessed from any hostname (important for K8s/proxy environments) +nginx['listen_addresses'] = ['0.0.0.0'] +nginx['listen_port'] = 8023 + +# Trust proxy headers from reverse proxies (K8s Ingress, nginx, etc.) +# Using private networks + localhost (safer than 0.0.0.0/0) +gitlab_rails['trusted_proxies'] = [ + '10.0.0.0/8', # Private network + '172.16.0.0/12', # Private network + '192.168.0.0/16', # Private network + '127.0.0.1', # Localhost +] + +# Disable production monitoring services to reduce resource usage +prometheus_monitoring['enable'] = false +alertmanager['enable'] = false +gitlab_exporter['enable'] = false +postgres_exporter['enable'] = false +redis_exporter['enable'] = false +gitlab_kas['enable'] = false + +# Puma configuration (~800MB) +puma['worker_processes'] = 4 +puma['min_threads'] = 1 +puma['max_threads'] = 4 + +# Sidekiq - good concurrency for background jobs (~600MB) +sidekiq['max_concurrency'] = 15 + +# Disable Grafana (not needed for testing) +grafana['enable'] = false + +# Disable node_exporter (not needed for testing) +node_exporter['enable'] = false + +# PostgreSQL - solid memory allocation (~1.5GB) +postgresql['max_connections'] = 50 +postgresql['shared_buffers'] = '512MB' +postgresql['work_mem'] = '32MB' +postgresql['effective_cache_size'] = '1GB' +postgresql['maintenance_work_mem'] = '128MB' + +# Redis - generous for caching (~512MB) +redis['maxclients'] = 200 +redis['maxmemory'] = '512mb' +redis['maxmemory_policy'] = 'allkeys-lru' + +# Disable usage statistics +gitlab_rails['usage_ping_enabled'] = false +gitlab_rails['seat_link_enabled'] = false + +# Disable Gitaly backup (reduces reconfigure time) +gitaly['configuration'] = { + backup: { + go_cloud_url: '', + }, +} diff --git a/dev/environments/docker/sites/gitlab/docker_overrides/gitlab.small.rb b/dev/environments/docker/sites/gitlab/docker_overrides/gitlab.small.rb new file mode 100644 index 0000000..e3dbf15 --- /dev/null +++ b/dev/environments/docker/sites/gitlab/docker_overrides/gitlab.small.rb @@ -0,0 +1,63 @@ +# Minimal GitLab configuration for resource-constrained environments +# Target: ~3GB RAM +# Use case: CI testing, low-traffic development + +# External URL - Set to generic internal URL +# For K8s deployments, the actual external URL is handled by Ingress +external_url 'http://localhost:8023' + +# Allow GitLab to be accessed from any hostname (important for K8s/proxy environments) +nginx['listen_addresses'] = ['0.0.0.0'] +nginx['listen_port'] = 8023 + +# Trust proxy headers from reverse proxies (K8s Ingress, nginx, etc.) +gitlab_rails['trusted_proxies'] = [ + '10.0.0.0/8', # Private network + '172.16.0.0/12', # Private network + '192.168.0.0/16', # Private network + '127.0.0.1', # Localhost +] + +# Disable production monitoring services to reduce resource usage +prometheus_monitoring['enable'] = false +alertmanager['enable'] = false +gitlab_exporter['enable'] = false +postgres_exporter['enable'] = false +redis_exporter['enable'] = false +gitlab_kas['enable'] = false + +# Puma configuration (~400MB) +puma['worker_processes'] = 2 +puma['min_threads'] = 1 +puma['max_threads'] = 2 + +# Sidekiq - minimal concurrency (~300MB) +sidekiq['max_concurrency'] = 5 + +# Disable Grafana (not needed for testing) +grafana['enable'] = false + +# Disable node_exporter (not needed for testing) +node_exporter['enable'] = false + +# PostgreSQL - minimal allocation (~500MB) +postgresql['max_connections'] = 20 +postgresql['shared_buffers'] = '128MB' +postgresql['work_mem'] = '8MB' +postgresql['effective_cache_size'] = '256MB' + +# Redis - minimal (~128MB) +redis['maxclients'] = 100 +redis['maxmemory'] = '128mb' +redis['maxmemory_policy'] = 'allkeys-lru' + +# Disable usage statistics +gitlab_rails['usage_ping_enabled'] = false +gitlab_rails['seat_link_enabled'] = false + +# Disable Gitaly backup (reduces reconfigure time) +gitaly['configuration'] = { + backup: { + go_cloud_url: '', + }, +} diff --git a/dev/environments/docker/sites/gitlab/scripts/00_apply_patches.sh b/dev/environments/docker/sites/gitlab/scripts/00_apply_patches.sh new file mode 100755 index 0000000..ff18227 --- /dev/null +++ b/dev/environments/docker/sites/gitlab/scripts/00_apply_patches.sh @@ -0,0 +1,28 @@ +#!/bin/bash +# +# Bootstrap script for GitLab container +# Sets up env-ctrl package and CLI wrapper +# Note: Entrypoint is added by Dockerfile, not here +# +set -euo pipefail + +WA_ENV_CTRL_ROOT="${WA_ENV_CTRL_ROOT:-/opt}" + +echo "========================================" +echo "[bootstrap] Setting up env-ctrl..." +echo "========================================" + +echo "[bootstrap] Copying env-ctrl package to ${WA_ENV_CTRL_ROOT}/environment_control" +cp -r "${BUILD_ENV_CTRL_SRC}" "${WA_ENV_CTRL_ROOT}/environment_control" + +echo "[bootstrap] Creating env-ctrl wrapper at /usr/local/bin/env-ctrl" +cat > /usr/local/bin/env-ctrl << EOF +#!/bin/sh +export PYTHONPATH="${WA_ENV_CTRL_ROOT}" +exec python3 -m environment_control.cli "\$@" +EOF +chmod +x /usr/local/bin/env-ctrl + +echo "========================================" +echo "[bootstrap] Done" +echo "========================================" diff --git a/dev/environments/docker/sites/gitlab/scripts/20_optimize.sh b/dev/environments/docker/sites/gitlab/scripts/20_optimize.sh new file mode 100755 index 0000000..1f2f9d7 --- /dev/null +++ b/dev/environments/docker/sites/gitlab/scripts/20_optimize.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Image optimization script for GitLab container +# NO-OP: GitLab has no user-uploaded images that need optimization +# +# Main size contributors that must be kept: +# - Git repositories (~13GB) +# - PostgreSQL data (~12GB) +# - Uploads (~3.4GB) +# - Binaries (~2.7GB) +# +set -euo pipefail + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" + +echo "========================================" +echo "[optimize] GitLab image optimization..." +echo "========================================" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Would skip optimization (no-op for GitLab)" + exit 0 +fi + +echo "[optimize] No image optimization needed for GitLab" +echo "[optimize] GitLab stores git repositories and database data, not user images" + +echo "========================================" +echo "[optimize] Done (no-op)" +echo "========================================" diff --git a/dev/environments/docker/sites/gitlab/scripts/60_cleanup.sh b/dev/environments/docker/sites/gitlab/scripts/60_cleanup.sh new file mode 100755 index 0000000..7dd4d73 --- /dev/null +++ b/dev/environments/docker/sites/gitlab/scripts/60_cleanup.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# +# Cleanup script for GitLab container +# Removes logs, caches, and temporary files to reduce image size +# +set -euo pipefail +set -x # Echo all commands + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Skipping cleanup" + exit 0 +fi + +echo "========================================" +echo "[cleanup] Starting cleanup..." +echo "========================================" + +echo "[cleanup] Removing GitLab logs..." +rm -rf /var/log/gitlab/*/*.log +rm -rf /var/log/gitlab/*/*/*.log + +echo "[cleanup] Removing runit svlogd logs..." +# svlogd uses @timestamp.u and 'current' files, not .log extension +rm -f /var/log/gitlab/*/@* +rm -f /var/log/gitlab/*/current +rm -f /var/log/gitlab/*/*/@* +rm -f /var/log/gitlab/*/*/current + +echo "[cleanup] Removing system logs..." +rm -rf /var/log/*.log +rm -rf /var/log/*/*.log + +echo "[cleanup] Removing Prometheus data (monitoring not needed for testing)..." +rm -rf /var/opt/gitlab/prometheus/data/* + +echo "[cleanup] Removing GitLab tmp and cache..." +rm -rf /var/opt/gitlab/gitlab-rails/shared/tmp/* +rm -rf /var/opt/gitlab/gitlab-rails/shared/cache/* +rm -rf /var/opt/gitlab/gitlab-rails/tmp/cache/* + +echo "[cleanup] Removing backups..." +rm -rf /var/opt/gitlab/backups/* + +echo "[cleanup] Removing apt cache..." +rm -rf /var/cache/apt/archives/* +rm -rf /var/lib/apt/lists/* + +echo "[cleanup] Removing temp files..." +rm -rf /tmp/* + +echo "[cleanup] Removing stale PID files..." +rm -f /var/opt/gitlab/gitaly/gitaly.pid +rm -f /opt/gitlab/embedded/cookbooks/cache/cinc-client-running.pid + +echo "[cleanup] Removing shell history and vim swap files..." +rm -f /root/.bash_history +rm -f /root/.viminfo + +# ----------------------------------------- +# Shutdown services +# ----------------------------------------- + +echo "[cleanup] Shutting down GitLab services..." +if command -v gitlab-ctl &> /dev/null; then + echo "[cleanup] Stopping all GitLab services..." + gitlab-ctl stop 2>/dev/null || true + sleep 2 +else + echo "[cleanup] gitlab-ctl not found, skipping service shutdown" +fi + +echo "========================================" +echo "[cleanup] Resetting PostgreSQL WAL files..." +echo "========================================" + +PG_DATA="/var/opt/gitlab/postgresql/data" +PG_CTL="/opt/gitlab/embedded/bin/pg_ctl" +PG_RESETWAL="/opt/gitlab/embedded/bin/pg_resetwal" + +echo "[cleanup] WAL size before: $(du -sh ${PG_DATA}/pg_wal/ 2>/dev/null | cut -f1)" + +# Start PostgreSQL to run CHECKPOINT +echo "[cleanup] Starting PostgreSQL..." +su - gitlab-psql -s /bin/sh -c "${PG_CTL} start -D ${PG_DATA} -w -t 60" + +# Run CHECKPOINT to flush all data to disk +echo "[cleanup] Running CHECKPOINT..." +su - gitlab-psql -s /bin/sh -c "/opt/gitlab/embedded/bin/psql -h /var/opt/gitlab/postgresql -d gitlabhq_production -c 'CHECKPOINT;'" + +# Stop PostgreSQL cleanly +echo "[cleanup] Stopping PostgreSQL..." +su - gitlab-psql -s /bin/sh -c "${PG_CTL} stop -D ${PG_DATA} -m fast -w -t 60" + +# Reset WAL files +echo "[cleanup] Running pg_resetwal..." +su - gitlab-psql -s /bin/sh -c "${PG_RESETWAL} -f ${PG_DATA}" + +echo "[cleanup] WAL size after: $(du -sh ${PG_DATA}/pg_wal/ 2>/dev/null | cut -f1)" + +echo "========================================" +echo "[cleanup] Done" +echo "========================================" diff --git a/dev/environments/docker/sites/gitlab/scripts/70_configure.sh b/dev/environments/docker/sites/gitlab/scripts/70_configure.sh new file mode 100755 index 0000000..0c8b025 --- /dev/null +++ b/dev/environments/docker/sites/gitlab/scripts/70_configure.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# +# Run gitlab-ctl reconfigure during build (not at runtime) +# This eliminates ~2 minute reconfigure overhead on every container start +# +set -euo pipefail + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" +SLEEP_INTERVAL=5 + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Skipping reconfigure" + exit 0 +fi + +echo "========================================" +echo "[configure] Starting GitLab reconfigure..." +echo "========================================" + +cleanup() { + echo "[configure] Cleaning up..." + gitlab-ctl stop 2>/dev/null || true + pkill -f runsvdir-start 2>/dev/null || true +} +trap cleanup EXIT + +echo "[configure] Starting runsvdir..." +/opt/gitlab/embedded/bin/runsvdir-start & +# Give runsvdir time to initialize before polling +sleep "$SLEEP_INTERVAL" + +echo "[configure] Waiting for runsvdir to be ready..." +for i in {1..120}; do + if gitlab-ctl status >/dev/null 2>&1; then + echo "[configure] runsvdir ready (attempt $i)" + break + fi + sleep "$SLEEP_INTERVAL" +done +if ! gitlab-ctl status >/dev/null 2>&1; then + echo "[configure] ERROR: runsvdir failed to start" + exit 1 +fi + +echo "[configure] Copying optimized gitlab.rb..." +cp /build-site/docker_overrides/gitlab.rb /etc/gitlab/gitlab.rb + +echo "[configure] Running gitlab-ctl reconfigure..." +gitlab-ctl reconfigure + +echo "[configure] Stopping services..." +gitlab-ctl stop +sleep "$SLEEP_INTERVAL" + +# Verify services stopped +if gitlab-ctl status 2>&1 | grep -q "^run:"; then + echo "[configure] WARNING: Some services still running, forcing stop..." + gitlab-ctl kill + sleep "$SLEEP_INTERVAL" +fi + +echo "========================================" +echo "[configure] Done" +echo "========================================" diff --git a/dev/environments/docker/sites/gitlab/scripts/90_verify.sh b/dev/environments/docker/sites/gitlab/scripts/90_verify.sh new file mode 100755 index 0000000..cf89844 --- /dev/null +++ b/dev/environments/docker/sites/gitlab/scripts/90_verify.sh @@ -0,0 +1,96 @@ +#!/bin/bash +# +# Verify GitLab database integrity after cleanup +# Starts PostgreSQL, runs checks, then stops it +# +set -euo pipefail + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Skipping verification" + exit 0 +fi + +echo "========================================" +echo "[verify] Starting database verification..." +echo "========================================" + +PG_DATA="/var/opt/gitlab/postgresql/data" +PG_CTL="/opt/gitlab/embedded/bin/pg_ctl" +PSQL="/opt/gitlab/embedded/bin/psql -h /var/opt/gitlab/postgresql -d gitlabhq_production -t -A" + +# Start PostgreSQL +echo "[verify] Starting PostgreSQL..." +su - gitlab-psql -s /bin/sh -c "${PG_CTL} start -D ${PG_DATA} -w -t 60" + +# Run verification queries +echo "" +echo "=== TABLE ROW COUNTS ===" + +verify_count() { + local table="$1" + local min_expected="$2" + local count + count=$(su - gitlab-psql -s /bin/sh -c "${PSQL} -c \"SELECT COUNT(*) FROM ${table};\"" 2>/dev/null || echo "ERROR") + + if [ "$count" = "ERROR" ]; then + echo " ${table}: ERROR - query failed" + return 1 + elif [ "$count" -lt "$min_expected" ]; then + echo " ${table}: ${count} (WARN: expected >= ${min_expected})" + return 1 + else + echo " ${table}: ${count} โœ“" + return 0 + fi +} + +ERRORS=0 + +# Core tables with minimum expected counts +verify_count "users" 100 || ((ERRORS++)) +verify_count "projects" 50 || ((ERRORS++)) +verify_count "namespaces" 100 || ((ERRORS++)) +verify_count "issues" 1000 || ((ERRORS++)) +verify_count "merge_requests" 1000 || ((ERRORS++)) +verify_count "notes" 1000 || ((ERRORS++)) +verify_count "members" 10 || ((ERRORS++)) +verify_count "uploads" 100 || ((ERRORS++)) + +echo "" +echo "=== DATABASE SIZE ===" +DB_SIZE=$(su - gitlab-psql -s /bin/sh -c "${PSQL} -c \"SELECT pg_size_pretty(pg_database_size('gitlabhq_production'));\"" 2>/dev/null) +echo " Total: ${DB_SIZE}" + +echo "" +echo "=== WAL STATUS ===" +WAL_SIZE=$(du -sh ${PG_DATA}/pg_wal/ 2>/dev/null | cut -f1) +echo " WAL directory: ${WAL_SIZE}" + +echo "" +echo "=== SAMPLE DATA CHECK ===" +# Verify root user exists +ROOT_USER=$(su - gitlab-psql -s /bin/sh -c "${PSQL} -c \"SELECT username FROM users WHERE id = 1;\"" 2>/dev/null) +if [ "$ROOT_USER" = "root" ]; then + echo " Root user: exists โœ“" +else + echo " Root user: MISSING" + ((ERRORS++)) +fi + +# Stop PostgreSQL +echo "" +echo "[verify] Stopping PostgreSQL..." +su - gitlab-psql -s /bin/sh -c "${PG_CTL} stop -D ${PG_DATA} -m fast -w -t 60" + +echo "" +echo "========================================" +if [ "$ERRORS" -gt 0 ]; then + echo "[verify] FAILED - ${ERRORS} error(s) found" + echo "========================================" + exit 1 +else + echo "[verify] PASSED - All checks OK" + echo "========================================" +fi diff --git a/dev/environments/docker/sites/map/Dockerfile b/dev/environments/docker/sites/map/Dockerfile new file mode 100644 index 0000000..92c09df --- /dev/null +++ b/dev/environments/docker/sites/map/Dockerfile @@ -0,0 +1,243 @@ +# ============================================================================ +# SOURCE IMAGES (for COPY --from) +# All source images use linux/amd64 to match the base image architecture +# ============================================================================ +ARG BUILDPLATFORM=linux/amd64 +FROM --platform=$BUILDPLATFORM overv/openstreetmap-tile-server@sha256:b6a79da39b6d0758368f7c62d22e49dd3ec59e78b194a5ef9dee2723b1f3fa79 AS tile-server +FROM --platform=$BUILDPLATFORM ghcr.io/project-osrm/osrm-backend:v5.27.1 AS osrm-backend +FROM --platform=$BUILDPLATFORM mediagis/nominatim:4.2 AS nominatim + +# ============================================================================ +# BASE STAGE - Binaries and packages (rarely changes) +# ============================================================================ +FROM --platform=$BUILDPLATFORM am1n3e/openstreetmap-website-web:base AS base + +ENV DEBIAN_FRONTEND=noninteractive + +# Add PostgreSQL apt repository for PostgreSQL 15 +RUN apt-get update \ + && apt-get install --no-install-recommends -y curl ca-certificates gnupg \ + && curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/postgresql-keyring.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/postgresql-keyring.gpg] http://apt.postgresql.org/pub/repos/apt jammy-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Install PostgreSQL 14 & 15, supervisor, git, and tile server dependencies +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + git \ + postgresql-14 \ + postgresql-contrib-14 \ + postgresql-14-postgis-3 \ + postgresql-15 \ + postgresql-15-postgis-3 \ + supervisor \ + apache2 \ + libapache2-mod-tile \ + renderd \ + osm2pgsql \ + osmium-tool \ + fonts-noto-cjk \ + fonts-noto-hinted \ + fonts-noto-unhinted \ + fonts-hanazono \ + fontconfig \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Copy tile server configurations from tile-server image +COPY --from=tile-server /etc/renderd.conf /etc/renderd.conf +COPY --from=tile-server /etc/postgresql/15/main/postgresql.custom.conf.tmpl /etc/postgresql/15/main/postgresql.custom.conf.tmpl +COPY --from=tile-server /home/renderer/src/openstreetmap-carto-backup /home/renderer/src/openstreetmap-carto-backup +COPY --from=tile-server /home/renderer/src/regional /home/renderer/src/regional +COPY --from=tile-server /run.sh /tile-server-run.sh + +# Install carto (Node.js tool to compile map styles to mapnik.xml) +RUN npm install -g carto + +# Create renderer user and setup directories +RUN useradd -m -s /bin/bash renderer \ + && mkdir -p /data/database /data/tiles /data/style /var/cache/renderd/tiles /run/renderd /var/log/tiles \ + && chown -R renderer: /data /var/cache/renderd /run/renderd /var/log/tiles \ + && ln -sf /data/style /home/renderer/src/openstreetmap-carto + +# Pre-build mapnik.xml from default style (will be copied to /data/style at runtime if empty) +RUN cd /home/renderer/src/openstreetmap-carto-backup \ + && carto project.mml > mapnik.xml + +# Copy OSRM binary from osrm-backend image +COPY --from=osrm-backend /usr/local/bin/osrm-routed /usr/local/bin/osrm-routed + +# Install OSRM runtime dependencies (boost libraries) +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + libboost-program-options1.74.0 \ + libboost-filesystem1.74.0 \ + libboost-iostreams1.74.0 \ + libboost-thread1.74.0 \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# === NOMINATIM === +# Copy Nominatim binaries and configs from nominatim stage +COPY --from=nominatim /usr/local/bin/nominatim /usr/local/bin/nominatim +COPY --from=nominatim /usr/local/lib/nominatim /usr/local/lib/nominatim +COPY --from=nominatim /usr/local/lib/python3.10 /usr/local/lib/python3.10 +COPY --from=nominatim /usr/local/share/nominatim /usr/local/share/nominatim +COPY --from=nominatim /usr/local/etc/nominatim /usr/local/etc/nominatim +COPY --from=nominatim /nominatim /nominatim-base + +# Install Nominatim PHP and Python dependencies, plus brotli for asset patching +RUN apt-get update && apt-get install --no-install-recommends -y \ + php8.1-fpm \ + php8.1-pgsql \ + php8.1-intl \ + php8.1-cgi \ + libapache2-mod-php8.1 \ + python3-yaml \ + python3-icu \ + python3-psycopg2 \ + python3-dotenv \ + brotli \ + python3-psutil \ + python3-jinja2 \ + python3-datrie \ + python3-sqlalchemy \ + python3-asyncpg \ + sudo \ + locales \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen \ + && locale-gen + +# Create Nominatim user and directories +RUN useradd -m -s /bin/bash nominatim \ + && mkdir -p /nominatim /data/nominatim/postgres /data/nominatim/flatnode /var/log/nominatim \ + && chown -R nominatim: /nominatim /var/log/nominatim + +# ============================================================================ +# FINAL STAGE - PostgreSQL configs and Rails compilation (iterate quickly) +# ============================================================================ +FROM base AS final + +ARG ENV_CTRL_PORT=8877 + +ENV DEBIAN_FRONTEND=noninteractive +ENV OSM_COMMIT=d4a014d3a6ca3f8f7d03528d39e4707dc256bc60 +ENV PG_VERSION=15 + +# Environment control configuration +ENV WA_ENV_CTRL_TYPE=map +ENV WA_ENV_CTRL_AVAILABLE=true +ENV WA_ENV_CTRL_ENABLE=true +ENV WA_ENV_CTRL_PORT=${ENV_CTRL_PORT} +ENV WA_ENV_CTRL_EXTERNAL_SITE_URL=http://localhost:8080 +ENV WA_CONTAINER_PORT=8080 + +# Drop auto-created clusters and create fresh ones with explicit ports +RUN pg_dropcluster --stop 14 main 2>/dev/null || true \ + && pg_dropcluster --stop 15 main 2>/dev/null || true + +# Initialize PostgreSQL 14 cluster (website database) on port 5433 +RUN pg_createcluster 14 main -p 5433 --start \ + && pg_ctlcluster 14 main stop \ + && echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/14/main/pg_hba.conf \ + && echo "listen_addresses='*'" >> /etc/postgresql/14/main/postgresql.conf + +# Initialize PostgreSQL 15 cluster (tile database) on port 5432 (renderd default) +RUN pg_createcluster 15 main -p 5432 --start \ + && pg_ctlcluster 15 main stop \ + && echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/15/main/pg_hba.conf \ + && echo "listen_addresses='*'" >> /etc/postgresql/15/main/postgresql.conf \ + && mkdir -p /etc/postgresql/15/main/conf.d + +# Initialize PostgreSQL 14 cluster for Nominatim on port 5434 +RUN pg_createcluster 14 nominatim -p 5434 --start \ + && pg_ctlcluster 14 nominatim stop \ + && echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/14/nominatim/pg_hba.conf \ + && echo "listen_addresses='*'" >> /etc/postgresql/14/nominatim/postgresql.conf + +# Configure Apache for tile serving on port 8080, OSRM proxy, and Nominatim on port 8085 +RUN echo "Listen 8080" >> /etc/apache2/ports.conf \ + && echo "Listen 8085" >> /etc/apache2/ports.conf \ + && a2enmod tile headers proxy proxy_http php8.1 +COPY dev/environments/docker/sites/map/docker_overrides/tile-server.conf /etc/apache2/sites-available/tile-server.conf +COPY dev/environments/docker/sites/map/docker_overrides/nominatim.conf /etc/apache2/sites-available/nominatim.conf +RUN a2ensite tile-server nominatim + +# Install image optimization tools for Rails asset pipeline (image_optim gem) +RUN apt-get update \ + && apt-get install --no-install-recommends -y \ + pngcrush \ + advancecomp \ + optipng \ + pngquant \ + jhead \ + jpegoptim \ + libjpeg-turbo-progs \ + gifsicle \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && npm install -g svgo + +# Clone OSM website app at exact commit +RUN rm -rf /app/* /app/.[!.]* \ + && git clone https://github.com/openstreetmap/openstreetmap-website.git /app \ + && cd /app && git checkout $OSM_COMMIT \ + && rm -rf /app/.git + +WORKDIR /app + +# Install Rails dependencies +RUN bundle install && bundle exec bin/yarn install + +# Setup Rails config files +RUN cp config/example.storage.yml config/storage.yml +COPY dev/environments/docker/sites/map/docker_overrides/database.yml config/database.yml +# Copy local settings BEFORE asset precompilation so URLs are baked into JS +COPY dev/environments/docker/sites/map/docker_overrides/settings.local.yml config/settings.local.yml + +# Production mode configuration +ENV RAILS_ENV=production +ENV SECRET_KEY_BASE=1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b + +# Patch tile URL to use local tile server via Apache proxy (relative URL) +RUN sed -i 's|https://tile.openstreetmap.org/|/tile/|g' \ + vendor/assets/leaflet/leaflet.osm.js + +# Remove non-OSRM routing engines (only local OSRM is available) +RUN rm -f app/assets/javascripts/index/directions/graphhopper.js \ + app/assets/javascripts/index/directions/fossgis_valhalla.js + +# Precompile assets at build time +# - RAILS_SERVE_STATIC_FILES enables serving precompiled assets +# - i18n:js:export generates JavaScript translations for embed.js.erb +RUN SECRET_KEY_BASE=$SECRET_KEY_BASE \ + RAILS_ENV=production \ + RAILS_SERVE_STATIC_FILES=true \ + bundle exec rails i18n:js:export assets:precompile + +# Patch compiled JS to use relative URLs for browser requests +# Server-side Rails uses absolute URLs from settings.local.yml (http://localhost:8080/...) +# Client-side browser JS needs relative URLs (/nominatim/, /osrm/) to work with any host port +RUN find public/assets -name "*.js" -exec \ + sed -i 's|http://localhost:8080/nominatim/|/nominatim/|g; s|http://localhost:8080/osrm/|/osrm/|g' {} \; + +# Supervisor config +RUN mkdir -p /var/log/supervisor +COPY dev/environments/docker/sites/map/docker_overrides/supervisord.conf /etc/supervisor/conf.d/supervisord.conf +COPY dev/environments/docker/sites/map/docker_overrides/entrypoint.sh /entrypoint.sh +COPY dev/environments/docker/sites/map/scripts/warmup_tiles.py /warmup_tiles.py +RUN chmod +x /entrypoint.sh + +# Copy environment_control package and env-ctrl CLI +COPY packages/environment_control/environment_control /usr/local/environment_control +COPY --chmod=755 packages/environment_control/environment_control/env-ctrl /usr/local/bin/env-ctrl + +# Ports: Apache unified (8080 - proxies Rails + tiles + OSRM + Nominatim), internal services below +# Internal: Rails (3000), Nominatim (8085), OSRM routing (5000-5002), PostgreSQL 15 tiles (5432), PostgreSQL 14 website (5433), PostgreSQL 14 nominatim (5434), env-ctrl +EXPOSE 8080 3000 5000 5001 5002 8085 5432 5433 5434 ${ENV_CTRL_PORT} + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"] diff --git a/dev/environments/docker/sites/map/docker_overrides/database.yml b/dev/environments/docker/sites/map/docker_overrides/database.yml new file mode 100644 index 0000000..73f24b6 --- /dev/null +++ b/dev/environments/docker/sites/map/docker_overrides/database.yml @@ -0,0 +1,26 @@ +development: + adapter: postgresql + database: openstreetmap + username: openstreetmap + password: openstreetmap + host: localhost + port: 5433 + encoding: utf8 + +test: + adapter: postgresql + database: osm_test + username: openstreetmap + password: openstreetmap + host: localhost + port: 5433 + encoding: utf8 + +production: + adapter: postgresql + database: openstreetmap + username: openstreetmap + password: openstreetmap + host: localhost + port: 5433 + encoding: utf8 diff --git a/dev/environments/docker/sites/map/docker_overrides/entrypoint.sh b/dev/environments/docker/sites/map/docker_overrides/entrypoint.sh new file mode 100755 index 0000000..bd46778 --- /dev/null +++ b/dev/environments/docker/sites/map/docker_overrides/entrypoint.sh @@ -0,0 +1,264 @@ +#!/bin/bash +set -e + +# PostgreSQL 14 (website database) +PG14_DATA=/var/lib/postgresql/14/main +PG14_CONFIG=/etc/postgresql/14/main + +# PostgreSQL 15 (tile database) +PG15_DATA=/data/database/postgres +PG15_CONFIG=/etc/postgresql/15/main + +# Nominatim (geocoding) +NOMINATIM_PG_DATA=/data/nominatim/postgres +NOMINATIM_PG_CONFIG=/etc/postgresql/14/nominatim +NOMINATIM_FLATNODE=/data/nominatim/flatnode +NOMINATIM_PROJECT=/nominatim + +# Initialize PostgreSQL 14 (website database) on port 5433 +init_postgres_14() { + echo "=== Initializing PostgreSQL 14 (website) ===" + + # Check if cluster exists (handles mounted empty volume) + if [ ! -f "$PG14_DATA/PG_VERSION" ]; then + echo "Initializing PostgreSQL 14 cluster..." + mkdir -p "$PG14_DATA" + chown -R postgres: "$PG14_DATA" + su - postgres -c "/usr/lib/postgresql/14/bin/initdb -D $PG14_DATA" + echo "host all all 0.0.0.0/0 trust" >> "$PG14_DATA/pg_hba.conf" + fi + + # Ensure correct ownership and permissions + chown -R postgres: "$PG14_DATA" + chmod 700 "$PG14_DATA" + + su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D $PG14_DATA -o \"-c config_file=$PG14_CONFIG/postgresql.conf\" start -w" + + # Check if user exists + if ! su - postgres -c "psql -p 5433 -tAc \"SELECT 1 FROM pg_roles WHERE rolname='openstreetmap'\"" | grep -q 1; then + echo "Creating openstreetmap user..." + su - postgres -c "psql -p 5433 -c \"CREATE USER openstreetmap SUPERUSER PASSWORD 'openstreetmap';\"" + fi + + # Check if database exists + if ! su - postgres -c "psql -p 5433 -tAc \"SELECT 1 FROM pg_database WHERE datname='openstreetmap'\"" | grep -q 1; then + echo "Creating openstreetmap database..." + su - postgres -c "createdb -p 5433 -O openstreetmap openstreetmap" + fi + + # Run migrations if schema not initialized + cd /app + if ! su - postgres -c "psql -p 5433 -d openstreetmap -tAc \"SELECT 1 FROM information_schema.tables WHERE table_name='users'\"" | grep -q 1; then + echo "Running database migrations..." + bundle exec rails db:migrate + fi + + echo "Stopping PostgreSQL 14 after initialization..." + su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D $PG14_DATA stop -m fast" +} + +# Initialize PostgreSQL 15 (tile database) on port 5432 +init_postgres_15() { + echo "=== Initializing PostgreSQL 15 (tiles) ===" + + # Check if tile data is mounted + if [ ! -d "$PG15_DATA" ] || [ ! -f "$PG15_DATA/PG_VERSION" ]; then + echo "WARNING: Tile database not found at $PG15_DATA" + echo "Mount tile data to /data/database/postgres/ for tile server functionality" + return 0 + fi + + # Fix ownership and permissions + chown -R postgres: $PG15_DATA + chmod 700 $PG15_DATA + + # Update data_directory in postgresql.conf to use mounted data + sed -i "s|data_directory = '.*'|data_directory = '$PG15_DATA'|" "$PG15_CONFIG/postgresql.conf" + + # Apply custom PostgreSQL config + if [ -f "$PG15_CONFIG/postgresql.custom.conf.tmpl" ]; then + cp "$PG15_CONFIG/postgresql.custom.conf.tmpl" "$PG15_CONFIG/conf.d/postgresql.custom.conf" + echo "autovacuum = on" >> "$PG15_CONFIG/conf.d/postgresql.custom.conf" + fi + + # Start PG15 temporarily to ensure renderer user exists + echo "Starting PostgreSQL 15 to verify renderer user..." + su - postgres -c "/usr/lib/postgresql/15/bin/pg_ctl -D $PG15_DATA -o \"-c config_file=$PG15_CONFIG/postgresql.conf\" start -w" + + # Create renderer user if not exists + if ! su - postgres -c "psql -p 5432 -tAc \"SELECT 1 FROM pg_roles WHERE rolname='renderer'\"" | grep -q 1; then + echo "Creating renderer user..." + su - postgres -c "psql -p 5432 -c \"CREATE USER renderer SUPERUSER PASSWORD 'renderer';\"" + fi + + # Check if gis database exists + if ! su - postgres -c "psql -p 5432 -tAc \"SELECT 1 FROM pg_database WHERE datname='gis'\"" | grep -q 1; then + echo "Creating gis database..." + su - postgres -c "createdb -p 5432 -O renderer gis" + su - postgres -c "psql -p 5432 -d gis -c 'CREATE EXTENSION IF NOT EXISTS postgis;'" + su - postgres -c "psql -p 5432 -d gis -c 'CREATE EXTENSION IF NOT EXISTS hstore;'" + fi + + echo "Stopping PostgreSQL 15 after initialization..." + su - postgres -c "/usr/lib/postgresql/15/bin/pg_ctl -D $PG15_DATA stop -m fast" + + echo "PostgreSQL 15 tile database ready" +} + +# Setup tile server styles +init_tile_styles() { + echo "=== Initializing tile styles ===" + + # If no custom style mounted, use default osm-carto + if [ ! "$(ls -A /data/style/ 2>/dev/null)" ]; then + echo "Copying default OpenStreetMap Carto style..." + cp -r /home/renderer/src/openstreetmap-carto-backup/* /data/style/ + chown -R renderer: /data/style/ + fi + + # Build mapnik.xml if not present + if [ ! -f /data/style/mapnik.xml ]; then + echo "Building mapnik.xml from style..." + cd /data/style/ + if command -v carto &> /dev/null; then + carto project.mml > mapnik.xml || echo "WARNING: carto build failed" + else + echo "WARNING: carto not installed, skipping mapnik.xml generation" + fi + fi + + # Ensure renderd directories exist with correct permissions + mkdir -p /run/renderd /var/cache/renderd/tiles + chown -R renderer: /run/renderd /var/cache/renderd +} + +# Initialize OSRM routing data +init_osrm_routing() { + echo "=== Initializing OSRM routing ===" + + local ROUTING_BASE=/data/routing + local profiles=("car" "bike" "foot") + local found=0 + + for profile in "${profiles[@]}"; do + local data_file="$ROUTING_BASE/$profile/us-northeast-latest.osrm.mldgr" + if [ -f "$data_file" ]; then + echo "OSRM $profile routing data found" + found=$((found + 1)) + else + echo "WARNING: OSRM $profile routing data not found at $ROUTING_BASE/$profile/" + fi + done + + if [ $found -eq 0 ]; then + echo "No routing data found. Mount routing data to /data/routing/{car,bike,foot}/" + echo "Expected files: us-northeast-latest.osrm.* (from osrm_routing.tar)" + else + echo "Found $found of 3 routing profiles" + fi +} + +# Patch JS assets to use relative URLs for Nominatim and OSRM +# This fixes the issue where precompiled assets have absolute URLs baked in +# Handles .js, .js.gz (gzip), and .js.br (brotli) files +patch_js_assets() { + echo "=== Patching JS assets for relative URLs ===" + + local ASSETS_DIR=/app/public/assets + local patched=0 + + # Find JS files that need patching by checking .gz files (fastest to scan) + for gz_file in "$ASSETS_DIR"/*.js.gz; do + [ -f "$gz_file" ] || continue + + # Check if file contains the hardcoded URLs + if zgrep -q "localhost:8080" "$gz_file" 2>/dev/null; then + local js_file="${gz_file%.gz}" + local br_file="${js_file}.br" + + # Patch the uncompressed JS file + if [ -f "$js_file" ]; then + sed -i 's|http://localhost:8080/nominatim/|/nominatim/|g; s|http://localhost:8080/osrm/|/osrm/|g' "$js_file" + + # Regenerate the gzip file + gzip -kf "$js_file" + + # Regenerate the brotli file if it exists + if [ -f "$br_file" ] && command -v brotli &> /dev/null; then + brotli -f "$js_file" + fi + + patched=$((patched + 1)) + echo "Patched: $(basename "$js_file")" + fi + fi + done + + if [ $patched -gt 0 ]; then + echo "Patched $patched JS asset(s) for relative URLs" + else + echo "No JS assets needed patching" + fi +} + +# Initialize Nominatim geocoding database +init_nominatim() { + echo "=== Initializing Nominatim ===" + + # Check if Nominatim data is mounted + if [ ! -d "$NOMINATIM_PG_DATA" ] || [ ! -f "$NOMINATIM_PG_DATA/PG_VERSION" ]; then + echo "WARNING: Nominatim database not found at $NOMINATIM_PG_DATA" + echo "Mount Nominatim data to /data/nominatim/ for geocoding functionality" + return 0 + fi + + # Fix ownership and permissions for PostgreSQL data + # PostgreSQL requires 0700 permissions on data directory + chown -R postgres: $NOMINATIM_PG_DATA + chmod 700 $NOMINATIM_PG_DATA + + # Fix ownership for flatnode and project directories + if [ -d "$NOMINATIM_FLATNODE" ]; then + chown -R nominatim: $NOMINATIM_FLATNODE + fi + chown -R nominatim: $NOMINATIM_PROJECT + + # Update PostgreSQL data directory + sed -i "s|data_directory = '.*'|data_directory = '$NOMINATIM_PG_DATA'|" "$NOMINATIM_PG_CONFIG/postgresql.conf" + + # Start PostgreSQL for Nominatim temporarily + echo "Starting PostgreSQL 14 (nominatim) to refresh website..." + su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D $NOMINATIM_PG_DATA -o \"-c config_file=$NOMINATIM_PG_CONFIG/postgresql.conf\" start -w" + + # Copy base nominatim project files if /nominatim is empty + if [ ! -f "$NOMINATIM_PROJECT/.env" ]; then + echo "Copying Nominatim project files..." + cp -a /nominatim-base/. $NOMINATIM_PROJECT/ + fi + + # Configure nominatim to use port 5434 + if ! grep -q "NOMINATIM_DATABASE_DSN" "$NOMINATIM_PROJECT/.env" 2>/dev/null; then + echo 'NOMINATIM_DATABASE_DSN="pgsql:host=/var/run/postgresql;port=5434;dbname=nominatim"' >> "$NOMINATIM_PROJECT/.env" + fi + chown -R nominatim: $NOMINATIM_PROJECT + + # Only refresh if website files are missing + cd $NOMINATIM_PROJECT + if [ ! -f "$NOMINATIM_PROJECT/website/search.php" ]; then + sudo -E -u nominatim nominatim refresh --website --functions 2>&1 || echo "WARNING: Nominatim refresh failed" + fi + + # Stop after init (supervisor will restart) + su - postgres -c "/usr/lib/postgresql/14/bin/pg_ctl -D $NOMINATIM_PG_DATA stop -m fast" + + echo "Nominatim initialized" +} + +patch_js_assets +init_postgres_14 +init_postgres_15 +init_nominatim +init_tile_styles +init_osrm_routing + +exec "$@" diff --git a/dev/environments/docker/sites/map/docker_overrides/nominatim.conf b/dev/environments/docker/sites/map/docker_overrides/nominatim.conf new file mode 100644 index 0000000..671cc26 --- /dev/null +++ b/dev/environments/docker/sites/map/docker_overrides/nominatim.conf @@ -0,0 +1,14 @@ + + DocumentRoot /nominatim/website + CustomLog /var/log/apache2/nominatim-access.log combined + ErrorLog /var/log/apache2/nominatim-error.log + LogLevel info + + + Options FollowSymLinks MultiViews + DirectoryIndex search.php + Require all granted + + + AddType text/html .php + diff --git a/dev/environments/docker/sites/map/docker_overrides/settings.local.yml b/dev/environments/docker/sites/map/docker_overrides/settings.local.yml new file mode 100644 index 0000000..ca79daa --- /dev/null +++ b/dev/environments/docker/sites/map/docker_overrides/settings.local.yml @@ -0,0 +1,5 @@ +# Local services via Apache proxy +# Server-side: Rails makes HTTP requests to these URLs (absolute for internal container) +# Client-side: JS assets are patched post-compile in Dockerfile to use relative URLs +fossgis_osrm_url: "http://localhost:8080/osrm/" +nominatim_url: "http://localhost:8080/nominatim/" diff --git a/dev/environments/docker/sites/map/docker_overrides/supervisord.conf b/dev/environments/docker/sites/map/docker_overrides/supervisord.conf new file mode 100644 index 0000000..fc381dc --- /dev/null +++ b/dev/environments/docker/sites/map/docker_overrides/supervisord.conf @@ -0,0 +1,109 @@ +[unix_http_server] +file=/var/run/supervisor.sock + +[supervisorctl] +serverurl=unix:///var/run/supervisor.sock + +[supervisord] +nodaemon=true +user=root +logfile=/var/log/supervisor/supervisord.log +pidfile=/var/run/supervisord.pid + +[program:postgresql-14] +command=/usr/lib/postgresql/14/bin/postgres -D /var/lib/postgresql/14/main -c config_file=/etc/postgresql/14/main/postgresql.conf +user=postgres +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/postgresql-14.log +stderr_logfile=/var/log/supervisor/postgresql-14_err.log +priority=10 + +[program:postgresql-15] +command=/usr/lib/postgresql/15/bin/postgres -D /data/database/postgres -c config_file=/etc/postgresql/15/main/postgresql.conf +user=postgres +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/postgresql-15.log +stderr_logfile=/var/log/supervisor/postgresql-15_err.log +priority=10 + +[program:postgresql-nominatim] +command=/usr/lib/postgresql/14/bin/postgres -D /data/nominatim/postgres -c config_file=/etc/postgresql/14/nominatim/postgresql.conf +user=postgres +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/postgresql-nominatim.log +stderr_logfile=/var/log/supervisor/postgresql-nominatim_err.log +priority=10 + +[program:rails] +command=bundle exec rails server -b 0.0.0.0 -p 3000 +directory=/app +user=root +environment=RAILS_ENV="production",SECRET_KEY_BASE="1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b3c4d5e6f7a8b9c0d1e2f3a4b5c6d7e8f9a0b1c2d3e4f5a6b7c8d9e0f1a2b",RAILS_SERVE_STATIC_FILES="true" +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/rails.log +stderr_logfile=/var/log/supervisor/rails_err.log +priority=20 + +[program:apache2] +command=/usr/sbin/apache2ctl -D FOREGROUND +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/apache2.log +stderr_logfile=/var/log/supervisor/apache2_err.log +priority=20 + +[program:renderd] +command=/usr/bin/renderd -f -c /etc/renderd.conf +user=renderer +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/renderd.log +stderr_logfile=/var/log/supervisor/renderd_err.log +priority=30 + +[program:osrm-car] +command=/usr/local/bin/osrm-routed --algorithm mld --mmap --max-table-size 1000 -t 4 --port 5000 /data/routing/car/us-northeast-latest.osrm +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/osrm-car.log +stderr_logfile=/var/log/supervisor/osrm-car_err.log +priority=30 + +[program:osrm-bike] +command=/usr/local/bin/osrm-routed --algorithm mld --mmap --max-table-size 1000 -t 4 --port 5001 /data/routing/bike/us-northeast-latest.osrm +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/osrm-bike.log +stderr_logfile=/var/log/supervisor/osrm-bike_err.log +priority=30 + +[program:osrm-foot] +command=/usr/local/bin/osrm-routed --algorithm mld --mmap --max-table-size 1000 -t 4 --port 5002 /data/routing/foot/us-northeast-latest.osrm +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/osrm-foot.log +stderr_logfile=/var/log/supervisor/osrm-foot_err.log +priority=30 + +[program:env-ctrl] +command=/bin/sh -c '[ "$WA_ENV_CTRL_ENABLE" != "true" ] && exit 0; exec /usr/local/bin/env-ctrl serve --port %(ENV_WA_ENV_CTRL_PORT)s' +autostart=true +autorestart=unexpected +startsecs=0 +stdout_logfile=/var/log/supervisor/env-ctrl.log +stderr_logfile=/var/log/supervisor/env-ctrl.log +priority=40 + +[program:env-ctrl-init] +command=/bin/sh -c '[ "$WA_ENV_CTRL_ENABLE" != "true" ] && exit 0; exec /usr/local/bin/env-ctrl init' +autostart=true +autorestart=false +startsecs=0 +exitcodes=0 +priority=999 +stdout_logfile=/var/log/supervisor/env-ctrl-init.log +stderr_logfile=/var/log/supervisor/env-ctrl-init.log diff --git a/dev/environments/docker/sites/map/docker_overrides/tile-server.conf b/dev/environments/docker/sites/map/docker_overrides/tile-server.conf new file mode 100644 index 0000000..50d9528 --- /dev/null +++ b/dev/environments/docker/sites/map/docker_overrides/tile-server.conf @@ -0,0 +1,36 @@ + + ServerAdmin webmaster@localhost + + AddTileConfig /tile/ default + LoadTileConfigFile /etc/renderd.conf + ModTileRenderdSocketName /run/renderd/renderd.sock + ModTileRequestTimeout 0 + ModTileMissingRequestTimeout 30 + + # OSRM routing proxy - maps FOSSGIS-style URLs to local OSRM + # /osrm/routed-{profile}/route/v1/... -> localhost:{port}/route/v1/... + ProxyPass /osrm/routed-car/ http://localhost:5000/ + ProxyPassReverse /osrm/routed-car/ http://localhost:5000/ + ProxyPass /osrm/routed-bike/ http://localhost:5001/ + ProxyPassReverse /osrm/routed-bike/ http://localhost:5001/ + ProxyPass /osrm/routed-foot/ http://localhost:5002/ + ProxyPassReverse /osrm/routed-foot/ http://localhost:5002/ + + # Nominatim geocoding proxy + ProxyPass /nominatim/ http://localhost:8085/ + ProxyPassReverse /nominatim/ http://localhost:8085/ + + # Rails website proxy (catch-all for everything else) + # Exclude paths handled by mod_tile and other proxies + ProxyPreserveHost On + ProxyPass /tile/ ! + ProxyPass / http://localhost:3000/ + ProxyPassReverse / http://localhost:3000/ + + DocumentRoot /var/www/html + + ErrorLog ${APACHE_LOG_DIR}/tile-error.log + CustomLog ${APACHE_LOG_DIR}/tile-access.log combined + + Header set Access-Control-Allow-Origin "*" + diff --git a/dev/environments/docker/sites/map/scripts/docker_setup.py b/dev/environments/docker/sites/map/scripts/docker_setup.py new file mode 100644 index 0000000..ea8284c --- /dev/null +++ b/dev/environments/docker/sites/map/scripts/docker_setup.py @@ -0,0 +1,146 @@ +"""Docker setup for map site. + +Sets up Docker volumes for the OpenStreetMap site including: +- Tile database (osm_tile_server.tar) +- Routing data for car/bike/foot (osrm_routing.tar) +- Nominatim geocoding data (nominatim_volumes.tar) + +Uses a single container for all download and extraction operations. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from dev.environments.docker.utils import docker_setup_helpers +from dev.environments.settings import get_settings +from dev.utils import logging_utils +from webarena_verified.types.task import WebArenaSite + +if TYPE_CHECKING: + from invoke.context import Context + +SITE = WebArenaSite.MAP +SCRIPT_PATH = Path(__file__).parent / "setup_volumes.sh" + +# Volumes that need data extraction from tar files (must match setup_volumes.sh) +# Maps volume suffix -> source tar file +VOLUMES_WITH_DATA = { + "tile-db": "osm_tile_server.tar", + "routing-car": "osrm_routing.tar", + "routing-bike": "osrm_routing.tar", + "routing-foot": "osrm_routing.tar", + "nominatim-db": "nominatim_volumes.tar", + "nominatim-flatnode": "nominatim_volumes.tar", +} + +# Volumes initialized empty (container populates at runtime) +VOLUMES_EMPTY = [ + "website-db", # PostgreSQL 14 for OSM website - initialized by entrypoint + "tiles", # Tile cache - rendered on demand by renderd + "style", # Map style - copied from backup by entrypoint +] + +# All volume suffixes +ALL_VOLUME_SUFFIXES = list(VOLUMES_WITH_DATA.keys()) + VOLUMES_EMPTY + + +def _create_volumes(ctx: Context, volume_prefix: str) -> None: + """Create all required Docker volumes.""" + for suffix in ALL_VOLUME_SUFFIXES: + vol_name = docker_setup_helpers.get_volume_name(volume_prefix, SITE.value, suffix) + docker_setup_helpers.create_volume(ctx, vol_name) + + +def _build_docker_command(data_dir: Path, volume_prefix: str, data_urls: tuple[str, ...]) -> str: + """Build docker run command with all volume mounts.""" + # Mount data directory and script + mounts = [ + f"-v {data_dir}:/data", + f"-v {SCRIPT_PATH}:/setup.sh:ro", + ] + + # Mount only volumes that need data extraction + for suffix in VOLUMES_WITH_DATA: + vol_name = docker_setup_helpers.get_volume_name(volume_prefix, SITE.value, suffix) + mounts.append(f"-v {vol_name}:/volumes/{suffix}") + + mount_args = " ".join(mounts) + urls_str = " ".join(data_urls) + + return f'docker run --rm -e DATA_URLS="{urls_str}" {mount_args} alpine sh /setup.sh' + + +def _print_dry_run_info(ctx: Context, data_urls: tuple[str, ...], data_dir: Path, volume_prefix: str) -> None: + """Print what would happen in dry-run mode, checking actual state.""" + # Check which tars exist + logging_utils.print_info("Downloads:") + for url in data_urls: + filename = url.rsplit("/", 1)[-1] + file_path = data_dir / filename + if file_path.exists(): + size_gb = file_path.stat().st_size / (1024 * 1024 * 1024) + logging_utils.print_info(f" [SKIP] {filename} ({size_gb:.1f}GB exists)") + else: + logging_utils.print_info(f" [DOWNLOAD] {filename}") + logging_utils.print_info(f" {url}") + + # Check volumes that need data extraction + logging_utils.print_info("") + logging_utils.print_info("Volumes (with data extraction):") + for suffix, tar_name in VOLUMES_WITH_DATA.items(): + vol_name = docker_setup_helpers.get_volume_name(volume_prefix, SITE.value, suffix) + + if not docker_setup_helpers.volume_exists(ctx, vol_name): + logging_utils.print_info(f" [CREATE + EXTRACT] {vol_name} (from {tar_name})") + elif docker_setup_helpers.volume_is_empty(ctx, vol_name): + logging_utils.print_info(f" [EXTRACT] {vol_name} (from {tar_name})") + else: + logging_utils.print_info(f" [SKIP] {vol_name} (has data)") + + # Check empty volumes (container-initialized) + logging_utils.print_info("") + logging_utils.print_info("Volumes (container-initialized):") + for suffix in VOLUMES_EMPTY: + vol_name = docker_setup_helpers.get_volume_name(volume_prefix, SITE.value, suffix) + + if not docker_setup_helpers.volume_exists(ctx, vol_name): + logging_utils.print_info(f" [CREATE] {vol_name}") + else: + logging_utils.print_info(f" [SKIP] {vol_name} (exists)") + + +def setup(ctx: Context, data_dir: Path, dry_run: bool = False, keep_downloads: bool = False) -> None: + """Set up map Docker volumes from data files. + + Uses a single container to download missing tars and extract to empty volumes. + + Args: + ctx: Invoke context for running commands. + data_dir: Directory to store/find data files. + dry_run: Show what would be done without doing it. + keep_downloads: Not used (tars are kept in data_dir for reuse). + """ + settings = get_settings() + volume_prefix = settings.volume_prefix + data_urls = settings.map.data_urls + + if dry_run: + _print_dry_run_info(ctx, data_urls, data_dir, volume_prefix) + logging_utils.print_info("") + logging_utils.print_info("Dry run complete - no changes made") + return + + # Ensure data directory exists + data_dir.mkdir(parents=True, exist_ok=True) + + # Create volumes first (outside container) + _create_volumes(ctx, volume_prefix) + + # Run setup script in container + cmd = _build_docker_command(data_dir, volume_prefix, data_urls) + logging_utils.print_info("Running setup in container...") + ctx.run(cmd, pty=True) + + logging_utils.print_success("Setup complete!") diff --git a/dev/environments/docker/sites/map/scripts/import-osm-data.sh b/dev/environments/docker/sites/map/scripts/import-osm-data.sh new file mode 100755 index 0000000..32666dc --- /dev/null +++ b/dev/environments/docker/sites/map/scripts/import-osm-data.sh @@ -0,0 +1,57 @@ +#!/bin/bash +# Import OSM data into the website database (PostgreSQL 14 on port 5433) +# +# Usage: +# ./import-osm-data.sh [container-name] +# +# Example: +# ./import-osm-data.sh us-northeast-latest.osm.pbf osm-website + +set -e + +PBF_FILE="${1:?Usage: $0 [container-name]}" +CONTAINER="${2:-osm-website}" + +# Database connection settings (PostgreSQL 14 - website database) +DB_HOST="127.0.0.1" +DB_PORT="5433" +DB_NAME="openstreetmap" +DB_USER="openstreetmap" +DB_PASS="openstreetmap" + +echo "=== OSM Data Import ===" +echo "PBF file: $PBF_FILE" +echo "Container: $CONTAINER" + +# Check if PBF file exists +[ -f "$PBF_FILE" ] || { echo "ERROR: PBF file not found: $PBF_FILE"; exit 1; } + +# Check if container is running +docker ps --format '{{.Names}}' | grep -q "^${CONTAINER}$" || { echo "ERROR: Container '$CONTAINER' is not running"; exit 1; } + +# Copy PBF file into container +echo "Copying PBF file into container..." +docker cp "$PBF_FILE" "$CONTAINER:/tmp/import.osm.pbf" + +# Import using osmosis (truncate first, then import) +echo "Importing OSM data..." +echo "Started at: $(date)" +docker exec "$CONTAINER" osmosis \ + --truncate-apidb host="$DB_HOST" port="$DB_PORT" database="$DB_NAME" user="$DB_USER" password="$DB_PASS" \ + --read-pbf file=/tmp/import.osm.pbf \ + --log-progress interval=10 \ + --write-apidb host="$DB_HOST" port="$DB_PORT" database="$DB_NAME" user="$DB_USER" password="$DB_PASS" validateSchemaVersion=no + +echo "Finished at: $(date)" + +# Cleanup +docker exec "$CONTAINER" rm -f /tmp/import.osm.pbf + +# Show final counts +echo "" +echo "=== Import complete ===" +docker exec "$CONTAINER" bash -c "PGPASSWORD=$DB_PASS psql -h $DB_HOST -p $DB_PORT -U $DB_USER -d $DB_NAME -c \" +SELECT 'nodes' as type, COUNT(*) as count FROM current_nodes +UNION ALL SELECT 'ways', COUNT(*) FROM current_ways +UNION ALL SELECT 'relations', COUNT(*) FROM current_relations; +\"" diff --git a/dev/environments/docker/sites/map/scripts/setup_volumes.sh b/dev/environments/docker/sites/map/scripts/setup_volumes.sh new file mode 100755 index 0000000..84e9e06 --- /dev/null +++ b/dev/environments/docker/sites/map/scripts/setup_volumes.sh @@ -0,0 +1,96 @@ +#!/bin/sh +# Setup script for map site volumes +# Downloads tars (if missing) and extracts to volumes (if empty) +# +# Environment variables: +# DATA_URLS - space-separated list of tar URLs to download +# +# Volume mounts expected: +# /data - directory for tar files +# /volumes/tile-db +# /volumes/routing-car +# /volumes/routing-bike +# /volumes/routing-foot +# /volumes/nominatim-db +# /volumes/nominatim-flatnode + +set -e + +echo "=== Map Volume Setup ===" + +# Install aria2c for fast parallel downloads +echo "Installing aria2c..." +apk add --no-cache aria2 + +# Download tars if not present +echo "" +echo "=== Downloading tars ===" +for url in $DATA_URLS; do + filename=$(basename "$url") + if [ -f "/data/$filename" ]; then + echo "SKIP: $filename already exists" + else + echo "Downloading $filename..." + aria2c -x 16 -s 16 --file-allocation=none -d /data -o "$filename" "$url" + fi +done + +# Extract to volumes if empty +echo "" +echo "=== Extracting to volumes ===" + +# tile-db from osm_tile_server.tar +if [ -n "$(ls -A /volumes/tile-db 2>/dev/null)" ]; then + echo "SKIP: tile-db is not empty" +else + echo "Extracting tile-db..." + tar -xf /data/osm_tile_server.tar -C /volumes/tile-db \ + --strip-components=6 'projects/ogma3/docker/volumes/osm-data/_data' +fi + +# routing-car from osrm_routing.tar +if [ -n "$(ls -A /volumes/routing-car 2>/dev/null)" ]; then + echo "SKIP: routing-car is not empty" +else + echo "Extracting routing-car..." + tar -xf /data/osrm_routing.tar -C /volumes/routing-car --strip-components=1 'car' +fi + +# routing-bike from osrm_routing.tar +if [ -n "$(ls -A /volumes/routing-bike 2>/dev/null)" ]; then + echo "SKIP: routing-bike is not empty" +else + echo "Extracting routing-bike..." + tar -xf /data/osrm_routing.tar -C /volumes/routing-bike --strip-components=1 'bike' +fi + +# routing-foot from osrm_routing.tar +if [ -n "$(ls -A /volumes/routing-foot 2>/dev/null)" ]; then + echo "SKIP: routing-foot is not empty" +else + echo "Extracting routing-foot..." + tar -xf /data/osrm_routing.tar -C /volumes/routing-foot --strip-components=1 'foot' +fi + +# nominatim-db from nominatim_volumes.tar +# Path: projects/metis2/docker/docker/volumes/nominatim-data/_data (7 components) +if [ -n "$(ls -A /volumes/nominatim-db 2>/dev/null)" ]; then + echo "SKIP: nominatim-db is not empty" +else + echo "Extracting nominatim-db..." + tar -xf /data/nominatim_volumes.tar -C /volumes/nominatim-db \ + --strip-components=7 'projects/metis2/docker/docker/volumes/nominatim-data/_data' +fi + +# nominatim-flatnode from nominatim_volumes.tar +# Path: projects/metis2/docker/docker/volumes/nominatim-flatnode/_data (7 components) +if [ -n "$(ls -A /volumes/nominatim-flatnode 2>/dev/null)" ]; then + echo "SKIP: nominatim-flatnode is not empty" +else + echo "Extracting nominatim-flatnode..." + tar -xf /data/nominatim_volumes.tar -C /volumes/nominatim-flatnode \ + --strip-components=7 'projects/metis2/docker/docker/volumes/nominatim-flatnode/_data' +fi + +echo "" +echo "=== Setup complete ===" diff --git a/dev/environments/docker/sites/map/scripts/warmup_tiles.py b/dev/environments/docker/sites/map/scripts/warmup_tiles.py new file mode 100644 index 0000000..31e7e34 --- /dev/null +++ b/dev/environments/docker/sites/map/scripts/warmup_tiles.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python3 +"""Warm up tile cache for US Northeast region. + +Fetches tiles via HTTP to trigger rendering and caching. + +Usage (inside container): + python3 /warmup_tiles.py + python3 /warmup_tiles.py --zoom-max 12 --parallel 16 +""" + +import argparse +import math +import urllib.request +from concurrent.futures import ThreadPoolExecutor, as_completed + +# US Northeast bounding box (matches OSRM routing data) +BBOX = {"min_lon": -80.5, "max_lon": -66.9, "min_lat": 37.0, "max_lat": 47.5} + + +def lat_lon_to_tile(lat: float, lon: float, zoom: int) -> tuple[int, int]: + """Convert lat/lon to tile coordinates.""" + n = 2**zoom + x = int((lon + 180) / 360 * n) + lat_rad = math.radians(lat) + y = int((1 - math.asinh(math.tan(lat_rad)) / math.pi) / 2 * n) + return x, y + + +def fetch_tile(url: str) -> tuple[str, int | str]: + """Fetch a single tile, return URL and HTTP status or error message.""" + try: + with urllib.request.urlopen(url, timeout=30) as resp: + resp.read() + return url, resp.status + except Exception as e: + return url, str(e) + + +def generate_tiles(zoom_max: int) -> list[tuple[int, int, int]]: + """Generate all tile coordinates for the bounding box.""" + tiles = [] + for z in range(zoom_max + 1): + x_min, y_max = lat_lon_to_tile(BBOX["min_lat"], BBOX["min_lon"], z) + x_max, y_min = lat_lon_to_tile(BBOX["max_lat"], BBOX["max_lon"], z) + for x in range(x_min, x_max + 1): + for y in range(y_min, y_max + 1): + tiles.append((z, x, y)) + return tiles + + +def main() -> None: + """Warm up tile cache with command-line arguments.""" + parser = argparse.ArgumentParser(description="Warm up tile cache") + parser.add_argument("--zoom-max", type=int, default=10, help="Max zoom level (default: 10)") + parser.add_argument("--parallel", type=int, default=8, help="Parallel requests (default: 8)") + parser.add_argument("--url", default="http://localhost:8080/tile", help="Tile server URL") + args = parser.parse_args() + + tiles = generate_tiles(args.zoom_max) + urls = [f"{args.url}/{z}/{x}/{y}.png" for z, x, y in tiles] + + print(f"Tiles to fetch: {len(urls)} (zoom 0-{args.zoom_max})") + + completed, errors = 0, 0 + with ThreadPoolExecutor(max_workers=args.parallel) as executor: + futures = {executor.submit(fetch_tile, url): url for url in urls} + for future in as_completed(futures): + url, status = future.result() + completed += 1 + if status != 200: + errors += 1 + if errors <= 5: + print(f" Error: {url} -> {status}") + if completed % 100 == 0 or completed == len(urls): + print(f"Progress: {completed}/{len(urls)} ({errors} errors)") + + print(f"Done: {completed - errors}/{completed} tiles cached") + + +if __name__ == "__main__": + main() diff --git a/dev/environments/docker/sites/reddit/Dockerfile b/dev/environments/docker/sites/reddit/Dockerfile new file mode 100644 index 0000000..0dcdc38 --- /dev/null +++ b/dev/environments/docker/sites/reddit/Dockerfile @@ -0,0 +1,38 @@ +# Reddit Environment - Final Image +# +# Base image created by: ./create_optimized_image.sh +# This Dockerfile adds env-ctrl on top of the optimized base. +# +# Build (from repo root): +# docker build -t reddit:latest -f dev/environments/docker/sites/reddit/Dockerfile . +# +# Run: +# docker run -d -p 9999:80 -p 8877:8877 reddit:latest + +ARG BASE_IMAGE=am1n3e/webarena-verified-reddit-base:latest +FROM ${BASE_IMAGE} +ARG ENV_CTRL_PORT=8877 + +WORKDIR /var/www/html + +# Environment control configuration +ENV WA_ENV_CTRL_TYPE=reddit +ENV WA_ENV_CTRL_AVAILABLE=true +ENV WA_ENV_CTRL_ENABLE=true +ENV WA_ENV_CTRL_PORT=${ENV_CTRL_PORT} +ENV WA_ENV_CTRL_EXTERNAL_SITE_URL=http://localhost:9999 +ENV WA_CONTAINER_PORT=80 + +# Copy environment_control package and env-ctrl CLI +COPY packages/environment_control/environment_control /usr/local/environment_control +COPY --chmod=755 packages/environment_control/environment_control/env-ctrl /usr/local/bin/env-ctrl + +# Copy env-ctrl supervisor configs (server + init) +COPY dev/environments/docker/sites/reddit/docker_overrides/etc/supervisor.d/env-ctrl.ini /etc/supervisor.d/env-ctrl.ini +COPY dev/environments/docker/sites/reddit/docker_overrides/etc/supervisor.d/env-ctrl-init.ini /etc/supervisor.d/env-ctrl-init.ini + +ENTRYPOINT ["/docker-entrypoint.sh"] +CMD ["supervisord", "-n", "-j", "/supervisord.pid"] + +# Expose ports (nginx: 80, env-ctrl) +EXPOSE 80 ${ENV_CTRL_PORT} diff --git a/dev/environments/docker/sites/reddit/README.md b/dev/environments/docker/sites/reddit/README.md new file mode 100644 index 0000000..ee26b22 --- /dev/null +++ b/dev/environments/docker/sites/reddit/README.md @@ -0,0 +1,44 @@ +# Reddit (Postmill) + +Port: 9999 + +## Fixes + +### Vote System + +**Problem:** Original code recalculates `netScore` from the votes collection. The imported database has `net_score` values but no vote records, causing first vote to reset score to ยฑ1. + +**Fix:** Use increment/decrement instead of recalculating. New vote adds ยฑ1, vote change swings ยฑ2, retract subtracts. + +**Files:** `Submission.php`, `Comment.php`, `Votable.php`, `VoteManager.php` + +### Header Authentication + +**Problem:** UI login via Playwright is slow and requires maintaining test credentials. + +**Fix:** Custom Symfony authenticator accepts `X-Postmill-Auto-Login: username:password` header. + +**Files:** `HeaderAutologinAuthenticator.php`, `security.yaml` + +**Usage:** +```python +context = await browser.new_context( + extra_http_headers={"X-Postmill-Auto-Login": "MarvelsGrantMan136:test1234"} +) +``` + +### URL Rewriting + +**Problem:** Symfony's `NoPrivateNetworkHttpClient` blocks requests to localhost/private IPs. Container can't fetch URLs referencing its own hostname. + +**Fix:** Custom HTTP client rewrites all external URLs to `http://localhost/` before making requests. + +**Files:** `UrlRewritingHttpClient.php`, `http_client.yaml` + +### Rate Limit Removal + +**Problem:** Postmill rate limits submissions, blocking automated testing. + +**Fix:** Removed `@RateLimit` annotations. + +**Files:** `SubmissionData.php` diff --git a/dev/environments/docker/sites/reddit/docker_overrides/Comment.php b/dev/environments/docker/sites/reddit/docker_overrides/Comment.php new file mode 100644 index 0000000..e2c523a --- /dev/null +++ b/dev/environments/docker/sites/reddit/docker_overrides/Comment.php @@ -0,0 +1,544 @@ +&Selectable + */ + private $children; + + /** + * @ORM\OneToMany(targetEntity="CommentVote", mappedBy="comment", + * fetch="EXTRA_LAZY", cascade={"persist"}, orphanRemoval=true) + * + * @var Collection&Selectable + */ + private $votes; + + /** + * @ORM\Column(type="text", options={"default": "visible"}) + * + * @var string + */ + private $visibility = self::VISIBILITY_VISIBLE; + + private ?\ArrayObject $cachedUserThreadVisibilityMap = null; + + /** + * @ORM\Column(type="inet", nullable=true) + * + * @var string|null + */ + private $ip; + + /** + * @ORM\Column(type="datetimetz_immutable", nullable=true) + * + * @var \DateTimeImmutable|null + */ + private $editedAt; + + /** + * @ORM\Column(type="boolean", options={"default": false}) + * + * @var bool + */ + private $moderated = false; + + /** + * @ORM\Column(type="text") + * + * @var string + */ + private $userFlag = UserFlags::FLAG_NONE; + + /** + * @ORM\OneToMany(targetEntity="CommentNotification", mappedBy="comment", cascade={"remove"}, orphanRemoval=true) + * + * @var Collection&Selectable + */ + private $notifications; + + /** + * @ORM\OneToMany(targetEntity="CommentMention", mappedBy="comment", cascade={"remove"}, orphanRemoval=true) + * + * @var Collection&Selectable + */ + private $mentions; + + /** + * @ORM\Column(type="integer") + * + * @var int + */ + private $netScore = 0; + + /** + * @ORM\Column(type="tsvector", nullable=true) + */ + private $searchDoc; + + /** + * @param Submission|Comment $parent + */ + public function __construct(string $body, User $user, $parent, ?string $ip) { + if ($ip !== null && !filter_var($ip, FILTER_VALIDATE_IP)) { + throw new \InvalidArgumentException('Invalid IP address'); + } + + if ($parent instanceof Submission) { + $submission = $parent; + $parent = null; + } elseif ($parent instanceof self) { + $submission = $parent->getSubmission(); + } else { + throw new \TypeError('$parent must be Submission or Comment'); + } + + if ($submission->isLocked() && !$user->isAdmin()) { + throw new SubmissionLockedException(); + } + + if ($submission->getForum()->userIsBanned($user)) { + throw new BannedFromForumException(); + } + + $this->body = $body; + $this->user = $user; + $this->submission = $submission; + $this->parent = $parent; + $this->ip = $user->isWhitelistedOrAdmin() ? null : $ip; + $this->timestamp = new \DateTimeImmutable('@'.time()); + $this->children = new ArrayCollection(); + $this->votes = new ArrayCollection(); + $this->notifications = new ArrayCollection(); + $this->mentions = new ArrayCollection(); + $this->addVote($this->createVote(self::VOTE_UP, $user, $ip)); + $this->notify(); + + if ($parent) { + $parent->children->add($this); + } + + $submission->addComment($this); + $user->addComment($this); + } + + public function getId(): ?int { + return $this->id; + } + + public function getBody(): string { + return $this->body; + } + + public function setBody(string $body): void { + $this->body = $body; + } + + public function getTimestamp(): \DateTimeImmutable { + return $this->timestamp; + } + + public function getUser(): User { + return $this->user; + } + + public function getSubmission(): Submission { + return $this->submission; + } + + public function getParent(): ?self { + return $this->parent; + } + + /** + * Get replies, ordered by descending net score. + * + * @return Comment[] + */ + public function getChildren(): array { + $criteria = Criteria::create()->orderBy(['netScore' => 'DESC']); + + return $this->children->matching($criteria)->getValues(); + } + + /** + * @return \Generator + */ + public function getChildrenRecursive(int &$startIndex = 0): \Generator { + foreach ($this->getChildren() as $child) { + // each yielded key must be unique, lol + yield $startIndex++ => $child; + yield from $child->getChildrenRecursive($startIndex); + } + } + + public function getReplyCount(): int { + return \count($this->children); + } + + public function removeReply(self $reply): void { + $this->children->removeElement($reply); + } + + /** + * @return Collection&Selectable + */ + protected function getVotes(): Collection { + return $this->votes; + } + + public function addMention(User $mentioned): void { + if ($mentioned === $this->getUser()) { + // don't notify yourself + return; + } + + if ($mentioned->isAccountDeleted()) { + return; + } + + if (!$mentioned->getNotifyOnMentions()) { + // don't notify users who've disabled mention notifications + return; + } + + if ($mentioned->isBlocking($this->getUser())) { + // don't notify users blocking you + return; + } + + $replyingTo = ($this->getParent() ?: $this->getSubmission())->getUser(); + + if ($replyingTo === $mentioned && $replyingTo->getNotifyOnReply()) { + // don't notify users who'll get a notification for the reply anyway + return; + } + + $mentioned->sendNotification(new CommentMention($mentioned, $this)); + } + + public function createVote(int $choice, User $user, ?string $ip): Vote { + return new CommentVote($choice, $user, $ip, $this); + } + + public function addVote(Vote $vote): void { + if (!$vote instanceof CommentVote) { + throw new \InvalidArgumentException(sprintf( + '$vote must be of subtype %s, %s given', + CommentVote::class, + \get_class($vote) + )); + } + + if ($this->submission->getForum()->userIsBanned($vote->getUser())) { + throw new BannedFromForumException(); + } + + // Only process if this is a new vote (not already in collection) + if (!$this->votes->contains($vote)) { + $this->votes->add($vote); + // New vote: add its effect (+1 for upvote, -1 for downvote) + $this->netScore += $vote->getChoice(); + } + // If vote already in collection, do nothing (handled by adjustNetScore for changes) + } + + public function removeVote(Vote $vote): void { + if (!$vote instanceof CommentVote) { + throw new \InvalidArgumentException(sprintf( + '$vote must be of subtype %s, %s given', + CommentVote::class, + \get_class($vote) + )); + } + + $this->votes->removeElement($vote); + + // Reverse the vote's effect (+1 upvote removed = -1, -1 downvote removed = +1) + $this->netScore -= $vote->getChoice(); + } + + /** + * Adjust netScore by a delta value. Used for vote changes. + */ + public function adjustNetScore(int $delta): void { + $this->netScore += $delta; + } + + public function getVisibility(): string { + return $this->visibility; + } + + public function isThreadVisible(): bool { + if ($this->isVisible()) { + return true; + } + + // TODO: avoid doing this more than once for an entire comment tree + foreach ($this->getChildrenRecursive() as $child) { + if ($child->isVisible()) { + return true; + } + } + + return false; + } + + public function isVisibleToUser($user, bool $search = false): bool { + $visibility = $this->getVisibility(); + + if ($search) { + if ($visibility === self::VISIBILITY_VISIBLE) { + return true; + } else if ($visibility === self::VISIBILITY_SOFT_DELETED) { + return false; + } + } else { + if (\in_array($visibility, [ + self::VISIBILITY_VISIBLE, + self::VISIBILITY_SOFT_DELETED, + ], true)) { + return true; + } + } + + if ($user === $this->getUser()) { + return true; + } + + return $this->getSubmission()->getForum()->userIsModerator($user); + } + + public function isThreadVisibleToUser($user): bool { + $userKey = $user instanceof User ? $user->getId() : -1; + if (!isset($this->cachedUserThreadVisibilityMap[$userKey])) { + $this->cacheThreadVisibilitiesForUser($user, $userKey); + } + + return $this->cachedUserThreadVisibilityMap[$userKey][$this->getId()]; + } + + private function setCachedUserThreadVisibilityMap( + int $userKey, + \ArrayObject $threadVisibilityMap + ): void { + if (!isset($this->cachedUserThreadVisibilityMap)) { + $this->cachedUserThreadVisibilityMap = new \ArrayObject(); + } + $this->cachedUserThreadVisibilityMap[$userKey] = $threadVisibilityMap; + } + + private function cacheThreadVisibilitiesForUser($user, int $userKey): void { + $search = [new CommentTree( + null, + $this, + $this->isVisibleToUser($user, true) + )]; + $threadVisibilityMap = new \ArrayObject(); + + for ($parentTree = 0; $parentTree < count($search); $parentTree++) { + $children = $search[$parentTree]->getComment()->getChildren(); + + foreach ($children as $child) { + $search[] = new CommentTree( + $search[$parentTree], + $child, + $child->isVisibleToUser($user, true) + ); + } + } + + for ($childTree = count($search) - 1; $childTree > 0; $childTree--) { + $visible = $search[$childTree]->getVisible(); + + if ($visible) { + $search[$childTree]->getParent()->setVisible(true); + } + $threadVisibilityMap[$search[$childTree]->getComment()->getId()] = + $visible; + + $search[$childTree]->getComment() + ->setCachedUserThreadVisibilityMap($userKey, $threadVisibilityMap); + } + $threadVisibilityMap[$this->getId()] = $search[0]->getVisible(); + + $this->setCachedUserThreadVisibilityMap($userKey, $threadVisibilityMap); + } + + /** + * Delete a comment without deleting its replies. + */ + public function softDelete(): void { + $this->visibility = self::VISIBILITY_SOFT_DELETED; + $this->body = ''; + $this->userFlag = UserFlags::FLAG_NONE; + $this->mentions->clear(); + $this->submission->updateCommentCount(); + $this->submission->updateRanking(); + $this->submission->updateLastActive(); + } + + public function trash(): void { + $this->visibility = self::VISIBILITY_TRASHED; + $this->mentions->clear(); + $this->submission->updateCommentCount(); + $this->submission->updateRanking(); + $this->submission->updateLastActive(); + } + + public function restore(): void { + $this->visibility = self::VISIBILITY_VISIBLE; + $this->submission->updateCommentCount(); + $this->submission->updateRanking(); + $this->submission->updateLastActive(); + } + + public function getIp(): ?string { + return $this->ip; + } + + public function getEditedAt(): ?\DateTimeImmutable { + return $this->editedAt; + } + + public function updateEditedAt(): void { + $this->editedAt = new \DateTimeImmutable('@'.time()); + } + + public function isModerated(): bool { + return $this->moderated; + } + + public function setModerated(bool $moderated): void { + $this->moderated = $moderated; + } + + public function getUserFlag(): string { + return $this->userFlag; + } + + public function setUserFlag(string $userFlag): void { + UserFlags::checkUserFlag($userFlag); + + $this->userFlag = $userFlag; + } + + private function notify(): void { + $receiver = ($this->parent ?: $this->submission)->getUser(); + + if ( + $this->user === $receiver || + $receiver->isAccountDeleted() || + !$receiver->getNotifyOnReply() || + $receiver->isBlocking($this->user) + ) { + // don't send notifications to oneself, to a user who's disabled + // them, or to a user who's blocked the user replying + return; + } + + $receiver->sendNotification(new CommentNotification($receiver, $this)); + } + + public function getNetScore(): int { + return $this->netScore; + } + + public function onCreate(): Event { + return new CommentCreated($this); + } + + public function onUpdate($previous): Event { + \assert($previous instanceof self); + + return new CommentUpdated($previous, $this); + } + + public function onDelete(): Event { + return new Event(); + } +} diff --git a/dev/environments/docker/sites/reddit/docker_overrides/HeaderAutologinAuthenticator.php b/dev/environments/docker/sites/reddit/docker_overrides/HeaderAutologinAuthenticator.php new file mode 100644 index 0000000..d70af0e --- /dev/null +++ b/dev/environments/docker/sites/reddit/docker_overrides/HeaderAutologinAuthenticator.php @@ -0,0 +1,89 @@ +entityManager = $entityManager; + } + + public function supports(Request $request): ?bool { + return $request->headers->has(self::HEADER_NAME); + } + + public function authenticate(Request $request): Passport { + $headerValue = $request->headers->get(self::HEADER_NAME); + + if (!$headerValue) { + throw new AuthenticationException('No credentials provided in ' . self::HEADER_NAME . ' header'); + } + + // Parse username:password format + $parts = explode(':', $headerValue, 2); + if (count($parts) !== 2) { + throw new AuthenticationException('Invalid format in ' . self::HEADER_NAME . ' header. Expected username:password'); + } + + $username = $parts[0]; + $password = $parts[1]; + + if (!$username) { + throw new AuthenticationException('No username provided in ' . self::HEADER_NAME . ' header'); + } + + if (!$password) { + throw new AuthenticationException('No password provided in ' . self::HEADER_NAME . ' header'); + } + + return new Passport( + new UserBadge($username, function (string $userIdentifier): User { + $user = $this->entityManager + ->getRepository(User::class) + ->loadUserByUsername($userIdentifier); + + if (!$user) { + throw new UserNotFoundException(sprintf('User "%s" not found.', $userIdentifier)); + } + + return $user; + }), + new PasswordCredentials($password) + ); + } + + public function onAuthenticationSuccess(Request $request, TokenInterface $token, string $firewallName): ?Response { + // Continue with the request + return null; + } + + public function onAuthenticationFailure(Request $request, AuthenticationException $exception): ?Response { + return new Response( + 'Authentication failed: ' . $exception->getMessage(), + Response::HTTP_UNAUTHORIZED + ); + } +} diff --git a/dev/environments/docker/sites/reddit/docker_overrides/Submission.php b/dev/environments/docker/sites/reddit/docker_overrides/Submission.php new file mode 100644 index 0000000..7f06c32 --- /dev/null +++ b/dev/environments/docker/sites/reddit/docker_overrides/Submission.php @@ -0,0 +1,711 @@ +&Selectable + */ + private $comments; + + /** + * @ORM\Column(type="integer") + * + * @var int + */ + private $commentCount = 0; + + /** + * @ORM\Column(type="datetimetz_immutable") + * + * @var \DateTimeImmutable + */ + private $timestamp; + + /** + * @ORM\Column(type="datetimetz_immutable") + * + * @var \DateTimeImmutable + */ + private $lastActive; + + /** + * @ORM\Column(type="text", options={"default": "visible"}) + * + * @var string + */ + private $visibility = self::VISIBILITY_VISIBLE; + + /** + * @ORM\JoinColumn(nullable=false) + * @ORM\ManyToOne(targetEntity="Forum", inversedBy="submissions") + * + * @var Forum + */ + private $forum; + + /** + * @ORM\JoinColumn(nullable=false) + * @ORM\ManyToOne(targetEntity="User", inversedBy="submissions") + * + * @var User + */ + private $user; + + /** + * @ORM\OneToMany(targetEntity="SubmissionVote", mappedBy="submission", + * fetch="EXTRA_LAZY", cascade={"persist"}, orphanRemoval=true) + * + * @var Collection&Selectable + */ + private $votes; + + /** + * @ORM\OneToMany(targetEntity="SubmissionMention", mappedBy="submission", cascade={"remove"}, orphanRemoval=true) + * + * @var Collection&Selectable + */ + private $mentions; + + /** + * @ORM\ManyToOne(targetEntity="Image", cascade={"persist"}) + * + * @var Image|null + */ + private $image; + + /** + * @ORM\Column(type="inet", nullable=true) + * + * @var string|null + */ + private $ip; + + /** + * @ORM\Column(type="boolean") + * + * @var bool + */ + private $sticky = false; + + /** + * @ORM\Column(type="bigint") + * + * @var int + */ + private $ranking; + + /** + * @ORM\Column(type="datetimetz_immutable", nullable=true) + * + * @var \DateTimeImmutable|null + */ + private $editedAt; + + /** + * @ORM\Column(type="boolean", options={"default": false}) + * + * @var bool + */ + private $moderated = false; + + /** + * @ORM\Column(type="text") + * + * @var string + */ + private $userFlag = UserFlags::FLAG_NONE; + + /** + * @ORM\Column(type="boolean", options={"default": false}) + * + * @var bool + */ + private $locked = false; + + /** + * @ORM\Column(type="integer") + * + * @var int + */ + private $netScore = 0; + + /** + * @ORM\JoinTable(name="submissions_flairs", joinColumns={ + * @ORM\JoinColumn(name="submission_id", referencedColumnName="id"), + * }, inverseJoinColumns={ + * @ORM\JoinColumn(name="flair_id", referencedColumnName="id"), + * }) + * @ORM\ManyToMany(targetEntity="Flair", cascade={"persist", "remove"}) + * + * @var Collection&Selectable + */ + private $flairs; + + /** + * @ORM\Column(type="tsvector", nullable=true) + * + * @var string + */ + private $searchDoc; + + public function __construct( + string $title, + ?string $url, + ?string $body, + Forum $forum, + User $user, + ?string $ip + ) { + if ($ip !== null && !filter_var($ip, FILTER_VALIDATE_IP)) { + throw new \InvalidArgumentException("Invalid IP address '$ip'"); + } + + if ($forum->userIsBanned($user)) { + throw new BannedFromForumException(); + } + + $this->title = $title; + $this->url = $url; + $this->body = $body; + $this->forum = $forum; + $this->user = $user; + $this->ip = $user->isWhitelistedOrAdmin() ? null : $ip; + $this->timestamp = new \DateTimeImmutable('@'.time()); + $this->comments = new ArrayCollection(); + $this->votes = new ArrayCollection(); + $this->mentions = new ArrayCollection(); + $this->flairs = new ArrayCollection(); + $this->addVote($this->createVote(self::VOTE_UP, $user, $ip)); + $this->updateLastActive(); + $forum->addSubmission($this); + $user->addSubmission($this); + } + + public function getId(): ?int { + return $this->id; + } + + public function getTitle(): string { + return $this->title; + } + + public function setTitle(string $title): void { + $this->title = $title; + } + + public function getUrl(): ?string { + return $this->url; + } + + public function setUrl(?string $url): void { + $this->url = $url; + } + + public function getBody(): ?string { + return $this->body; + } + + public function setBody(?string $body): void { + $this->body = $body; + } + + public function getMediaType(): string { + return $this->mediaType; + } + + public function setMediaType(string $mediaType): void { + if (!\in_array($mediaType, self::MEDIA_TYPES, true)) { + throw new \InvalidArgumentException("Bad media type '$mediaType'"); + } + + if ($mediaType === self::MEDIA_IMAGE && $this->url !== null) { + throw new \BadMethodCallException( + 'Submission with URL cannot have image as media type' + ); + } + + $this->mediaType = $mediaType; + } + + /** + * Get all comments, ordered by ascending time of creation. + * + * @return Comment[] + */ + public function getComments(): array { + $criteria = Criteria::create() + ->where(Criteria::expr()->eq('visibility', Comment::VISIBILITY_VISIBLE)) + ->orderBy(['timestamp' => 'ASC']); + + return $this->comments->matching($criteria)->toArray(); + } + + /** + * Get top-level comments, ordered by descending net score. + * + * @return Comment[] + */ + public function getTopLevelComments(): array { + $criteria = Criteria::create() + ->where(Criteria::expr()->isNull('parent')) + ->andWhere(Criteria::expr()->eq('visibility', Comment::VISIBILITY_VISIBLE)); + + $comments = $this->comments->matching($criteria)->toArray(); + + usort($comments, static function (Comment $a, Comment $b) { + return $b->getNetScore() <=> $a->getNetScore(); + }); + + return $comments; + } + + public function hasVisibleComments(): bool { + $criteria = Criteria::create() + ->where(Criteria::expr()->eq('visibility', Comment::VISIBILITY_VISIBLE)) + ->setMaxResults(1); + + return \count($this->comments->matching($criteria)) > 0; + } + + public function addComment(Comment ...$comments): void { + foreach ($comments as $comment) { + if (!$this->comments->contains($comment)) { + $this->comments->add($comment); + } + } + + $this->updateCommentCount(); + $this->updateRanking(); + $this->updateLastActive(); + } + + public function removeComment(Comment ...$comments): void { + // hydrate the collection + $this->comments->get(-1); + + foreach ($comments as $comment) { + if ($this->comments->contains($comment)) { + $this->comments->removeElement($comment); + } + } + + $this->updateCommentCount(); + $this->updateRanking(); + $this->updateLastActive(); + } + + public function getCommentCount(): int { + return $this->commentCount; + } + + public function updateCommentCount(): void { + // hydrate the collection + $this->comments->get(-1); + + $criteria = Criteria::create() + ->where(Criteria::expr()->eq('visibility', Comment::VISIBILITY_VISIBLE)); + + $this->commentCount = \count($this->comments->matching($criteria)); + } + + public function getTimestamp(): \DateTimeImmutable { + return $this->timestamp; + } + + public function getLastActive(): \DateTimeImmutable { + return $this->lastActive; + } + + public function updateLastActive(): void { + // hydrate the collection + $this->comments->get(-1); + + $criteria = Criteria::create() + ->where(Criteria::expr()->eq('visibility', Comment::VISIBILITY_VISIBLE)) + ->orderBy(['timestamp' => 'DESC']) + ->setMaxResults(1); + + $lastComment = $this->comments->matching($criteria)->first(); + \assert($lastComment instanceof Comment || !$lastComment); + + if ($lastComment) { + $this->lastActive = $lastComment->getTimestamp(); + } else { + $this->lastActive = $this->getTimestamp(); + } + } + + public function getVisibility(): string { + return $this->visibility; + } + + public function softDelete(): void { + $this->visibility = self::VISIBILITY_SOFT_DELETED; + $this->title = ''; + $this->mediaType = self::MEDIA_URL; + $this->url = null; + $this->body = null; + $this->image = null; + $this->sticky = false; + $this->userFlag = UserFlags::FLAG_NONE; + $this->mentions->clear(); + } + + public function trash(): void { + $this->visibility = self::VISIBILITY_TRASHED; + $this->sticky = false; + $this->mentions->clear(); + } + + public function restore(): void { + if (!\in_array($this->visibility, [ + self::VISIBILITY_VISIBLE, + self::VISIBILITY_TRASHED, + ], true)) { + throw new \DomainException(sprintf( + 'Cannot restore a submission in the "%s" state', + $this->visibility, + )); + } + + $this->visibility = self::VISIBILITY_VISIBLE; + } + + public function getForum(): Forum { + return $this->forum; + } + + public function getUser(): User { + return $this->user; + } + + /** + * @return Collection&Selectable + */ + protected function getVotes(): Collection { + return $this->votes; + } + + public function createVote(int $choice, User $user, ?string $ip): Vote { + return new SubmissionVote($choice, $user, $ip, $this); + } + + public function addVote(Vote $vote): void { + if (!$vote instanceof SubmissionVote) { + throw new \InvalidArgumentException(sprintf( + '$vote must be of subtype %s, %s given', + SubmissionVote::class, + \get_class($vote) + )); + } + + if ( + $vote->getChoice() !== self::VOTE_NONE && + $this->forum->userIsBanned($vote->getUser()) + ) { + throw new BannedFromForumException(); + } + + // Only process if this is a new vote (not already in collection) + if (!$this->votes->contains($vote)) { + $this->votes->add($vote); + // New vote: add its effect (+1 for upvote, -1 for downvote) + $this->netScore += $vote->getChoice(); + $this->updateRanking(); + } + // If vote already in collection, do nothing (handled by adjustNetScore for changes) + } + + /** + * Adjust netScore by a delta value. Used for vote changes. + */ + public function adjustNetScore(int $delta): void { + $this->netScore += $delta; + $this->updateRanking(); + } + + public function removeVote(Vote $vote): void { + if (!$vote instanceof SubmissionVote) { + throw new \InvalidArgumentException(sprintf( + '$vote must be of subtype %s, %s given', + SubmissionVote::class, + \get_class($vote) + )); + } + + $this->votes->removeElement($vote); + + // Reverse the vote's effect (+1 upvote removed = -1, -1 downvote removed = +1) + $this->netScore -= $vote->getChoice(); + + $this->updateRanking(); + } + + public function addMention(User $mentioned): void { + if ($mentioned === $this->getUser()) { + // don't notify yourself + return; + } + + if ($mentioned->isAccountDeleted()) { + return; + } + + if (!$mentioned->getNotifyOnMentions()) { + // don't notify users who've disabled mention notifications + return; + } + + if ($mentioned->isBlocking($this->getUser())) { + // don't notify users blocking you + return; + } + + $mentioned->sendNotification(new SubmissionMention($mentioned, $this)); + } + + public function getImage(): ?Image { + return $this->image; + } + + public function setImage(?Image $image): void { + $this->image = $image; + } + + public function getIp(): ?string { + return $this->ip; + } + + public function isSticky(): bool { + return $this->sticky; + } + + public function setSticky(bool $sticky): void { + $this->sticky = $sticky; + } + + public function getRanking(): int { + return $this->ranking; + } + + public function updateRanking(): void { + $netScore = $this->getNetScore(); + $netScoreAdvantage = $netScore * self::NETSCORE_MULTIPLIER; + + if ($netScore > self::DOWNVOTED_CUTOFF) { + $commentAdvantage = $this->getCommentCount() * self::COMMENT_MULTIPLIER; + } else { + $commentAdvantage = $this->getCommentCount() * self::COMMENT_DOWNVOTED_MULTIPLIER; + } + + $advantage = max(min($netScoreAdvantage + $commentAdvantage, self::MAX_ADVANTAGE), -self::MAX_PENALTY); + + $this->ranking = $this->getTimestamp()->getTimestamp() + $advantage; + } + + public function getEditedAt(): ?\DateTimeImmutable { + return $this->editedAt; + } + + public function setEditedAt(?\DateTimeInterface $editedAt): void { + if ($editedAt instanceof \DateTime) { + $editedAt = \DateTimeImmutable::createFromMutable($editedAt); + } + + $this->editedAt = $editedAt; + } + + public function isModerated(): bool { + return $this->moderated; + } + + public function setModerated(bool $moderated): void { + $this->moderated = $moderated; + } + + public function getUserFlag(): string { + return $this->userFlag; + } + + public function setUserFlag(string $userFlag): void { + UserFlags::checkUserFlag($userFlag); + + $this->userFlag = $userFlag; + } + + public function isLocked(): bool { + return $this->locked; + } + + public function setLocked(bool $locked): void { + $this->locked = $locked; + } + + public function getNetScore(): int { + return $this->netScore; + } + + /** + * @return Flair[] + */ + public function getFlairs(): array { + return $this->flairs->getValues(); + } + + public function addFlair(Flair $flair): void { + if (!$this->flairs->contains($flair)) { + $this->flairs->add($flair); + } + } + + public function removeFlair(Flair $flair): void { + $this->flairs->removeElement($flair); + } + + public function onCreate(): Event { + return new SubmissionCreated($this); + } + + public function onUpdate($previous): Event { + \assert($previous instanceof self); + + return new SubmissionUpdated($previous, $this); + } + + /** + * use {@link DeleteSubmission} with negative priority instead + */ + public function onDelete(): Event { + return new Event(); + } +} diff --git a/dev/environments/docker/sites/reddit/docker_overrides/SubmissionData.php b/dev/environments/docker/sites/reddit/docker_overrides/SubmissionData.php new file mode 100644 index 0000000..f1bbd45 --- /dev/null +++ b/dev/environments/docker/sites/reddit/docker_overrides/SubmissionData.php @@ -0,0 +1,324 @@ +id = $submission->getId(); + $self->title = $submission->getTitle(); + $self->url = $submission->getUrl(); + $self->body = $submission->getBody(); + $self->mediaType = $submission->getMediaType(); + $self->commentCount = $submission->getCommentCount(); + $self->timestamp = $submission->getTimestamp(); + $self->lastActive = $submission->getLastActive(); + $self->visibility = $submission->getVisibility(); + $self->forum = $submission->getForum(); + $self->user = $submission->getUser(); + $self->netScore = $submission->getNetScore(); + $self->upvotes = $submission->getUpvotes(); + $self->downvotes = $submission->getDownvotes(); + $self->image = $submission->getImage(); + $self->sticky = $submission->isSticky(); + $self->editedAt = $submission->getEditedAt(); + $self->moderated = $submission->isModerated(); + $self->userFlag = $submission->getUserFlag(); + $self->locked = $submission->isLocked(); + + return $self; + } + + public function getId(): ?int { + return $this->id; + } + + public function getTitle(): ?string { + return $this->title; + } + + public function setTitle(?string $title): void { + $this->title = $title; + } + + public function getUrl(): ?string { + return $this->url; + } + + public function setUrl(?string $url): void { + $this->url = $url; + } + + public function getBody(): ?string { + return $this->body; + } + + public function setBody(?string $body): void { + $this->body = $body; + } + + public function getMediaType(): ?string { + return $this->mediaType; + } + + public function setMediaType(string $mediaType): void { + $this->mediaType = $mediaType; + } + + public function getCommentCount(): int { + return $this->commentCount; + } + + public function getTimestamp(): ?\DateTimeImmutable { + return $this->timestamp; + } + + public function getLastActive(): ?\DateTimeImmutable { + return $this->lastActive; + } + + public function getVisibility(): ?string { + return $this->visibility; + } + + public function getForum(): ?Forum { + return $this->forum; + } + + public function setForum(?Forum $forum): void { + $this->forum = $forum; + } + + public function getUser(): ?User { + return $this->user; + } + + public function getNetScore(): int { + return $this->netScore; + } + + public function getUpvotes(): int { + return $this->upvotes; + } + + public function getDownvotes(): int { + return $this->downvotes; + } + + public function getImage(): ?Image { + return $this->image; + } + + public function setImage(?Image $image): void { + $this->image = $image; + } + + public function isSticky(): bool { + return $this->sticky; + } + + public function setSticky(bool $sticky): void { + $this->sticky = $sticky; + } + + public function getEditedAt(): ?\DateTimeImmutable { + return $this->editedAt; + } + + public function isModerated(): bool { + return $this->moderated; + } + + public function getUserFlag(): string { + return $this->userFlag; + } + + public function setUserFlag(?string $userFlag): void { + $this->userFlag = $userFlag; + } + + public function isLocked(): bool { + return $this->locked; + } + + public function setLocked(bool $locked): void { + $this->locked = $locked; + } + + public function getMarkdownFields(): iterable { + yield 'body'; + } +} diff --git a/dev/environments/docker/sites/reddit/docker_overrides/UrlRewritingHttpClient.php b/dev/environments/docker/sites/reddit/docker_overrides/UrlRewritingHttpClient.php new file mode 100644 index 0000000..5d699c9 --- /dev/null +++ b/dev/environments/docker/sites/reddit/docker_overrides/UrlRewritingHttpClient.php @@ -0,0 +1,48 @@ +client = $client; + } + + public function request(string $method, string $url, array $options = []): ResponseInterface + { + // Rewrite any external URL (any host, any port) to internal container address + // Examples: + // http://localhost:9999/post/123 -> http://localhost/post/123 + // http://reddit.example.com/post/123 -> http://localhost/post/123 + // https://192.168.1.100:8443/post/123 -> http://localhost/post/123 + $rewrittenUrl = preg_replace('/^https?:\/\/[^\/]+\//', 'http://localhost/', $url); + + return $this->client->request($method, $rewrittenUrl, $options); + } + + public function stream($responses, float $timeout = null): ResponseStreamInterface + { + return $this->client->stream($responses, $timeout); + } + + public function withOptions(array $options): static + { + $clone = clone $this; + $clone->client = $this->client->withOptions($options); + return $clone; + } +} diff --git a/dev/environments/docker/sites/reddit/docker_overrides/Votable.php b/dev/environments/docker/sites/reddit/docker_overrides/Votable.php new file mode 100644 index 0000000..d09fcd5 --- /dev/null +++ b/dev/environments/docker/sites/reddit/docker_overrides/Votable.php @@ -0,0 +1,30 @@ +entityManager = $entityManager; + } + + public function vote(Votable $votable, User $user, int $choice, ?string $ip): void { + $vote = $votable->getUserVote($user); + + if ($vote) { + if ($choice === Votable::VOTE_NONE) { + // Retract vote + $votable->removeVote($vote); + $this->entityManager->remove($vote); + } elseif ($choice !== $vote->getChoice()) { + // Vote change: adjust netScore by the delta (e.g., +1 to -1 = -2) + $oldChoice = $vote->getChoice(); + $vote->setChoice($choice); + $votable->adjustNetScore($choice - $oldChoice); + } + // If same choice, do nothing + } elseif ($choice !== Votable::VOTE_NONE) { + // New vote: createVote calls addVote internally via constructor + $vote = $votable->createVote($choice, $user, $ip); + $this->entityManager->persist($vote); + } + + $this->entityManager->flush(); + } +} diff --git a/dev/environments/docker/sites/reddit/docker_overrides/etc/supervisor.d/env-ctrl-init.ini b/dev/environments/docker/sites/reddit/docker_overrides/etc/supervisor.d/env-ctrl-init.ini new file mode 100644 index 0000000..d3b0dda --- /dev/null +++ b/dev/environments/docker/sites/reddit/docker_overrides/etc/supervisor.d/env-ctrl-init.ini @@ -0,0 +1,11 @@ +[program:env-ctrl-init] +command=/bin/sh -c '[ "$WA_ENV_CTRL_ENABLE" != "true" ] && exit 0; exec /usr/local/bin/env-ctrl init' +autostart=true +autorestart=false +startsecs=0 +exitcodes=0 +priority=999 +stdout_logfile=/tmp/env-ctrl-init.log +stdout_logfile_maxbytes=10MB +stderr_logfile=/tmp/env-ctrl-init.log +stderr_logfile_maxbytes=10MB diff --git a/dev/environments/docker/sites/reddit/docker_overrides/etc/supervisor.d/env-ctrl.ini b/dev/environments/docker/sites/reddit/docker_overrides/etc/supervisor.d/env-ctrl.ini new file mode 100644 index 0000000..0517819 --- /dev/null +++ b/dev/environments/docker/sites/reddit/docker_overrides/etc/supervisor.d/env-ctrl.ini @@ -0,0 +1,9 @@ +[program:env-ctrl] +command=/bin/sh -c '[ "$WA_ENV_CTRL_ENABLE" != "true" ] && exit 0; exec /usr/local/bin/env-ctrl serve --port %(ENV_WA_ENV_CTRL_PORT)s' +autostart=true +autorestart=unexpected +startsecs=0 +stdout_logfile=/tmp/env-ctrl.log +stdout_logfile_maxbytes=10MB +stderr_logfile=/tmp/env-ctrl.log +stderr_logfile_maxbytes=10MB diff --git a/dev/environments/docker/sites/reddit/docker_overrides/http_client.yaml b/dev/environments/docker/sites/reddit/docker_overrides/http_client.yaml new file mode 100644 index 0000000..8a42718 --- /dev/null +++ b/dev/environments/docker/sites/reddit/docker_overrides/http_client.yaml @@ -0,0 +1,33 @@ +parameters: + postmill.http_client.default_user_agent: "Postmill/%env(APP_VERSION)% (https://postmill.xyz)" + postmill.http_client.external_client_options: + headers: + User-Agent: "%postmill.http_client.default_user_agent%" + base_uri: null + +services: + _defaults: + public: false + + Symfony\Contracts\HttpClient\HttpClientInterface $externalClient: "@postmill.http_client.external_client" + + postmill.http_client.default: + class: Symfony\Contracts\HttpClient\HttpClientInterface + factory: ['@Symfony\Contracts\HttpClient\HttpClientInterface', withOptions] + arguments: + - "%postmill.http_client.external_client_options%" + + # Decorator to rewrite any external URLs to internal localhost address + postmill.http_client.url_rewriter: + class: App\HttpClient\UrlRewritingHttpClient + arguments: ["@.inner"] + decorates: postmill.http_client.default + decoration_priority: 10 + + # Use the default client directly without NoPrivateNetworkHttpClient wrapper + postmill.http_client.external_client: + alias: postmill.http_client.default + + postmill.http_client.external_client.psr18: + class: Symfony\Component\HttpClient\Psr18Client + arguments: ["@postmill.http_client.external_client"] diff --git a/dev/environments/docker/sites/reddit/docker_overrides/security.yaml b/dev/environments/docker/sites/reddit/docker_overrides/security.yaml new file mode 100644 index 0000000..a9df601 --- /dev/null +++ b/dev/environments/docker/sites/reddit/docker_overrides/security.yaml @@ -0,0 +1,66 @@ +services: + _defaults: + autowire: true + public: false + + postmill.security.form_login_authentication_failure_handler: + stack: + - App\Security\FormLoginAuthenticationFailureHandler: + - Symfony\Component\Security\Http\Authentication\DefaultAuthenticationFailureHandler: + + # Header-based authenticator for auto-login + App\Security\HeaderAutologinAuthenticator: + arguments: + - "@doctrine.orm.entity_manager" + +security: + enable_authenticator_manager: true + + password_hashers: + App\DataObject\UserData: native + App\Entity\User: native + + providers: + user_provider: + entity: + class: App\Entity\User + + firewalls: + dev: + pattern: ^/(_(profiler|wdt)|css|images|js)/ + security: false + + main: + entry_point: form_login + + # Header-based auth (X-Postmill-Auto-Login header) + custom_authenticators: + - App\Security\HeaderAutologinAuthenticator + + form_login: + enable_csrf: true + remember_me: true + failure_handler: postmill.security.form_login_authentication_failure_handler + + login_link: + check_route: login_url + signature_properties: ["id"] + + logout: + csrf_parameter: token + csrf_token_generator: security.csrf.token_manager + delete_cookies: [PHPSESSID] + path: log_out + + remember_me: + secret: "%env(APP_SECRET)%" + + user_checker: App\Security\UserChecker + + login_throttling: + max_attempts: 5 + limiter: App\Security\RateLimitExemptLimiter + + role_hierarchy: + ROLE_ADMIN: [ROLE_WHITELISTED, ROLE_USER] + ROLE_WHITELISTED: [ROLE_USER] diff --git a/dev/environments/docker/sites/reddit/scripts/00_apply_patches.sh b/dev/environments/docker/sites/reddit/scripts/00_apply_patches.sh new file mode 100755 index 0000000..9db8fd1 --- /dev/null +++ b/dev/environments/docker/sites/reddit/scripts/00_apply_patches.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# +# Bootstrap and apply patches for Reddit (Postmill) container +# Sets up env-ctrl, copies entrypoint, and applies patches +# +set -euo pipefail + +WA_ENV_CTRL_ROOT="${WA_ENV_CTRL_ROOT:-/opt}" +OVERRIDES_DIR="/build-site/docker_overrides" + +echo "========================================" +echo "[bootstrap] Setting up env-ctrl..." +echo "========================================" + +echo "[bootstrap] Copying env-ctrl package to ${WA_ENV_CTRL_ROOT}/environment_control" +cp -r "${BUILD_ENV_CTRL_SRC}" "${WA_ENV_CTRL_ROOT}/environment_control" + +echo "[bootstrap] Creating env-ctrl wrapper at /usr/local/bin/env-ctrl" +cat >/usr/local/bin/env-ctrl </dev/null | cut -f1) +echo "[optimize] Initial size: ${initial_size}MB" + +echo "[optimize] Files: ${TOTAL_GIFS} GIFs, ${TOTAL_JPGS} JPEGs, ${TOTAL_PNGS} PNGs" +echo "[optimize] Using ${WORKERS} parallel workers" + +# Create helper script for parallel +cat > /tmp/optimize_image.sh << 'EOF' +#!/bin/sh +file="$1" +ext="${file##*.}" +ext_lower=$(echo "$ext" | tr '[:upper:]' '[:lower:]') + +case "$ext_lower" in + jpg|jpeg) + mogrify -verbose -resize "400x400>" "$file" + jpegoptim -v --strip-all --max=30 "$file" + ;; + png) + mogrify -verbose -resize "400x400>" "$file" + pngquant --verbose --force --quality=20-40 --speed 1 --ext .png "$file" || true + ;; + gif) + gifsicle -b -V -O3 --lossy=100 --colors 128 --resize-fit 400x400 "$file" || true + ;; +esac +EOF +chmod +x /tmp/optimize_image.sh + +# Optimize JPEGs: resize with mogrify, then compress with jpegoptim +echo "" +echo "[optimize] Optimizing JPEGs (resize 400x400, quality=30)..." +find "$IMAGE_DIR" -type f \( -iname '*.jpg' -o -iname '*.jpeg' \) -print0 2>/dev/null | \ + parallel -0 -j "$WORKERS" /tmp/optimize_image.sh +echo "[optimize] JPEG optimization complete" + +# Optimize PNGs: resize with mogrify, then compress with pngquant +echo "" +echo "[optimize] Optimizing PNGs (resize 400x400, quality=20-40)..." +find "$IMAGE_DIR" -type f -iname '*.png' -print0 2>/dev/null | \ + parallel -0 -j "$WORKERS" /tmp/optimize_image.sh +echo "[optimize] PNG optimization complete" + +# Optimize GIFs with gifsicle (resize + aggressive compression) +echo "" +echo "[optimize] Optimizing GIFs (resize 400x400, lossy=100, colors=128)..." +find "$IMAGE_DIR" -type f -iname '*.gif' -print0 2>/dev/null | \ + parallel -0 -j "$WORKERS" /tmp/optimize_image.sh +echo "[optimize] GIF optimization complete" + +# Cleanup +echo "" +echo "[optimize] Removing optimization tools..." +apk del gifsicle jpegoptim pngquant parallel 2>/dev/null || true +rm -rf /var/cache/apk/* + + +final_size=$(du -sm "$IMAGE_DIR" 2>/dev/null | cut -f1) +saved=$((initial_size - final_size)) + +echo "" +echo "========================================" +echo "[optimize] Image optimization complete" +echo "[optimize] Before: ${initial_size}MB" +echo "[optimize] After: ${final_size}MB" +echo "[optimize] Saved: ${saved}MB" +echo "========================================" diff --git a/dev/environments/docker/sites/reddit/scripts/60_cleanup.sh b/dev/environments/docker/sites/reddit/scripts/60_cleanup.sh new file mode 100755 index 0000000..0094e57 --- /dev/null +++ b/dev/environments/docker/sites/reddit/scripts/60_cleanup.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# +# Cleanup script for Reddit (Postmill) container +# Removes logs, caches, and temporary files to reduce image size +# +set -euo pipefail +set -x # Echo all commands + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Skipping cleanup" + exit 0 +fi + +echo "========================================" +echo "[cleanup] Starting cleanup..." +echo "========================================" + +# ----------------------------------------- +# PostgreSQL WAL reset (do first) +# ----------------------------------------- + +echo "========================================" +echo "[cleanup] Resetting PostgreSQL WAL files..." +echo "========================================" + +PG_DATA="/usr/local/pgsql/data" +PG_CTL="/usr/bin/pg_ctl" +PG_RESETWAL="/usr/bin/pg_resetwal" + +# Database credentials +DB_HOST="localhost" +DB_USER="postmill" +DB_PASS="postmill" +DB_NAME="postmill" + +# Superuser credentials (for CHECKPOINT) +DB_SUPERUSER="postgres" +DB_SUPERPASS="postgres" + +echo "[cleanup] WAL size before: $(du -sh ${PG_DATA}/pg_wal/ 2>/dev/null | cut -f1)" + +# Start PostgreSQL to run CHECKPOINT +echo "[cleanup] Starting PostgreSQL..." +su - postgres -s /bin/sh -c "${PG_CTL} start -D ${PG_DATA} -w -t 60" + +# Set superuser password (base image may not have it configured) +echo "[cleanup] Setting superuser password..." +su - postgres -s /bin/sh -c "psql -c \"ALTER USER ${DB_SUPERUSER} WITH PASSWORD '${DB_SUPERPASS}';\"" + +# Run CHECKPOINT to flush all data to disk (requires superuser) +echo "[cleanup] Running CHECKPOINT..." +PGPASSWORD="${DB_SUPERPASS}" psql -h "${DB_HOST}" -U "${DB_SUPERUSER}" -d postgres -c 'CHECKPOINT;' + +# Stop PostgreSQL cleanly +echo "[cleanup] Stopping PostgreSQL..." +su - postgres -s /bin/sh -c "${PG_CTL} stop -D ${PG_DATA} -m fast -w -t 60" + +# Reset WAL files +echo "[cleanup] Running pg_resetwal..." +su - postgres -s /bin/sh -c "${PG_RESETWAL} -f ${PG_DATA}" + +echo "[cleanup] WAL size after: $(du -sh ${PG_DATA}/pg_wal/ 2>/dev/null | cut -f1)" + +# ----------------------------------------- +# Application cleanup +# ----------------------------------------- + +echo "[cleanup] Removing Symfony caches and logs..." +rm -rf /var/www/html/var/cache/* +rm -rf /var/www/html/var/log/* +rm -rf /var/www/html/var/sessions/* + +echo "[cleanup] Removing node_modules (not needed at runtime)..." +rm -rf /var/www/html/node_modules + +echo "[cleanup] Removing Elasticsearch logs..." +rm -rf /usr/share/java/elasticsearch/logs/* + +echo "[cleanup] Removing system logs..." +rm -rf /var/log/*.log +rm -rf /var/log/*/*.log + +echo "[cleanup] Removing yarn cache..." +rm -rf /usr/local/share/.cache/yarn + +echo "[cleanup] Removing Composer cache..." +rm -rf /root/.composer/cache/* + +echo "[cleanup] Removing Alpine package cache..." +rm -rf /var/cache/apk/* + +echo "[cleanup] Removing stale PID files..." +rm -f /run/nginx.pid +rm -f /run/mysqld/*.pid +rm -f /supervisord.pid +rm -f /usr/local/pgsql/data/postmaster.pid + +echo "[cleanup] Removing shell history..." +rm -f /root/.bash_history + +# ----------------------------------------- +# Shutdown services +# ----------------------------------------- + +echo "[cleanup] Shutting down services..." +if pgrep -x supervisord > /dev/null; then + echo "[cleanup] Supervisord is running, stopping all services..." + supervisorctl stop all 2>/dev/null || true + pkill supervisord 2>/dev/null || true + sleep 2 +else + echo "[cleanup] Supervisord not running, skipping service shutdown" +fi + +# ----------------------------------------- +# Final cleanup +# ----------------------------------------- + +# Remove ImageMagick temporary files (can be 25GB+) +echo "[cleanup] Removing ImageMagick temp files..." +rm -rf /tmp/magick-* 2>/dev/null || true + +# General temp cleanup +echo "[cleanup] Removing general temp files..." +rm -rf /tmp/* 2>/dev/null || true +rm -rf /var/cache/* 2>/dev/null || true +rm -rf /var/log/* 2>/dev/null || true + +# Recreate required nginx directories (removed by cleanup above) +echo "[cleanup] Recreating nginx log directories..." +mkdir -p /var/log/nginx /var/lib/nginx/logs + +# Remove Python cache files +echo "[cleanup] Removing Python cache files..." +find / -type d -name '__pycache__' -exec rm -rf {} + 2>/dev/null || true +find / -type f -name '*.pyc' -delete 2>/dev/null || true + +echo "========================================" +echo "[cleanup] Done" +echo "========================================" diff --git a/dev/environments/docker/sites/reddit/scripts/90_verify.sh b/dev/environments/docker/sites/reddit/scripts/90_verify.sh new file mode 100755 index 0000000..b0a9b2d --- /dev/null +++ b/dev/environments/docker/sites/reddit/scripts/90_verify.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# +# Verify Reddit (Postmill) database integrity after cleanup +# Starts PostgreSQL, runs checks, then stops it +# +set -euo pipefail + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Skipping verification" + exit 0 +fi + +echo "========================================" +echo "[verify] Starting database verification..." +echo "========================================" + +PG_DATA="/usr/local/pgsql/data" +PG_CTL="/usr/bin/pg_ctl" + +# Database credentials +DB_HOST="localhost" +DB_USER="postmill" +DB_PASS="postmill" +DB_NAME="postmill" + +# psql command with credentials +run_psql() { + PGPASSWORD="${DB_PASS}" psql -h "${DB_HOST}" -U "${DB_USER}" -d "${DB_NAME}" -t -A "$@" +} + +# Start PostgreSQL +echo "[verify] Starting PostgreSQL..." +su - postgres -c "${PG_CTL} start -D ${PG_DATA} -w -t 60" + +# Run verification queries +echo "" +echo "=== TABLE ROW COUNTS ===" + +verify_count() { + local table="$1" + local min_expected="$2" + local count + count=$(run_psql -c "SELECT COUNT(*) FROM ${table};" 2>/dev/null || echo "ERROR") + + if [ "$count" = "ERROR" ]; then + echo " ${table}: ERROR - query failed" + return 1 + elif [ "$count" -lt "$min_expected" ]; then + echo " ${table}: ${count} (WARN: expected >= ${min_expected})" + return 1 + else + echo " ${table}: ${count} โœ“" + return 0 + fi +} + +ERRORS=0 + +# Core tables with minimum expected counts +verify_count "users" 600000 || ((ERRORS++)) +verify_count "forums" 90 || ((ERRORS++)) +verify_count "submissions" 120000 || ((ERRORS++)) +verify_count "comments" 2500000 || ((ERRORS++)) +verify_count "images" 30000 || ((ERRORS++)) + +echo "" +echo "=== DATABASE SIZE ===" +DB_SIZE=$(run_psql -c "SELECT pg_size_pretty(pg_database_size('${DB_NAME}'));" 2>/dev/null) +echo " Total: ${DB_SIZE}" + +echo "" +echo "=== SAMPLE DATA CHECK ===" +# Verify admin user exists +ADMIN_USER=$(run_psql -c "SELECT username FROM users WHERE admin = true ORDER BY id LIMIT 1;" 2>/dev/null) +if [ -n "$ADMIN_USER" ]; then + echo " Admin user: ${ADMIN_USER} โœ“" +else + echo " Admin user: MISSING" + ((ERRORS++)) +fi + +# Verify sample forum exists +FORUM=$(run_psql -c "SELECT name FROM forums ORDER BY id LIMIT 1;" 2>/dev/null) +if [ -n "$FORUM" ]; then + echo " Sample forum: ${FORUM} โœ“" +else + echo " Sample forum: MISSING" + ((ERRORS++)) +fi + +# Stop PostgreSQL +echo "" +echo "[verify] Stopping PostgreSQL..." +su - postgres -c "${PG_CTL} stop -D ${PG_DATA} -m fast -w -t 60" + +echo "" +echo "========================================" +if [ "$ERRORS" -gt 0 ]; then + echo "[verify] FAILED - ${ERRORS} error(s) found" + echo "========================================" + exit 1 +else + echo "[verify] PASSED - All checks OK" + echo "========================================" +fi diff --git a/dev/environments/docker/sites/shopping/Dockerfile b/dev/environments/docker/sites/shopping/Dockerfile new file mode 100644 index 0000000..3ec112f --- /dev/null +++ b/dev/environments/docker/sites/shopping/Dockerfile @@ -0,0 +1,51 @@ +# Shopping Environment - Final Image +# +# Base image created by: ./create_optimized_image.sh +# This Dockerfile adds env-ctrl on top of the optimized base. +# +# Note: ENV variables from the original image are preserved by docker-squash, +# so we only need to add the env-ctrl configuration. +# +# Build (from repo root): +# docker build -t shopping:latest -f dev/environments/docker/sites/shopping/Dockerfile . +# +# Run: +# docker run -d -p 7770:80 -p 8877:8877 shopping:latest + +ARG BASE_IMAGE=am1n3e/webarena-verified-shopping-base:latest +FROM ${BASE_IMAGE} +ARG ENV_CTRL_PORT=8877 + +WORKDIR /var/www/html + +# Environment control configuration +ENV WA_ENV_CTRL_TYPE=shopping +ENV WA_ENV_CTRL_AVAILABLE=true +ENV WA_ENV_CTRL_ENABLE=true +ENV WA_ENV_CTRL_PORT=${ENV_CTRL_PORT} +ENV WA_ENV_CTRL_EXTERNAL_SITE_URL=http://localhost:7770 +ENV WA_CONTAINER_PORT=80 + +# Resource optimization settings (applied by custom entrypoint) +ENV NGINX_WORKER_PROCESSES=2 +ENV ES_JAVA_OPTS="-Xms512m -Xmx512m" +ENV MYSQL_INNODB_BUFFER_POOL_SIZE=256M +ENV MYSQL_MAX_CONNECTIONS=20 + +# Copy environment_control package and env-ctrl CLI +COPY packages/environment_control/environment_control /usr/local/environment_control +COPY --chmod=755 packages/environment_control/environment_control/env-ctrl /usr/local/bin/env-ctrl + +# Copy env-ctrl supervisor configs (server + init) +COPY dev/environments/docker/sites/shopping/docker_overrides/etc/supervisor.d/env-ctrl.ini /etc/supervisor.d/env-ctrl.ini +COPY dev/environments/docker/sites/shopping/docker_overrides/etc/supervisor.d/env-ctrl-init.ini /etc/supervisor.d/env-ctrl-init.ini + +# Copy custom entrypoint that applies resource optimizations +COPY --chmod=755 dev/environments/docker/sites/shopping/docker_overrides/entrypoint.sh /custom-entrypoint.sh + +# Use custom entrypoint (calls original entrypoint after applying optimizations) +ENTRYPOINT ["/custom-entrypoint.sh"] +CMD ["supervisord", "-n", "-j", "/supervisord.pid"] + +# Expose ports (nginx: 80, env-ctrl) +EXPOSE 80 ${ENV_CTRL_PORT} diff --git a/dev/environments/docker/sites/shopping/README.md b/dev/environments/docker/sites/shopping/README.md new file mode 100644 index 0000000..a5be3f8 --- /dev/null +++ b/dev/environments/docker/sites/shopping/README.md @@ -0,0 +1,20 @@ +# Shopping (Magento Storefront) + +Port: 7770 + +## Fixes + +### Header Authentication + +**Problem:** UI login via Playwright is slow and requires maintaining test credentials. + +**Fix:** Magento plugin intercepts frontend requests and authenticates customers via `X-M2-Customer-Auto-Login: email:password` header. + +**Files:** `CustomerAutoLogin/Plugin/CustomerAutoLoginPlugin.php`, `CustomerAutoLogin/etc/di.xml`, `CustomerAutoLogin/etc/module.xml`, `CustomerAutoLogin/registration.php` + +**Usage:** +```python +context = await browser.new_context( + extra_http_headers={"X-M2-Customer-Auto-Login": "emma.lopez@gmail.com:Password.123"} +) +``` diff --git a/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/Plugin/CustomerAutoLoginPlugin.php b/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/Plugin/CustomerAutoLoginPlugin.php new file mode 100644 index 0000000..6e3cffa --- /dev/null +++ b/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/Plugin/CustomerAutoLoginPlugin.php @@ -0,0 +1,126 @@ +customerSession = $customerSession; + $this->accountManagement = $accountManagement; + $this->appState = $appState; + $this->logger = $logger; + } + + /** + * Before front controller dispatch, check for auto-login header. + * + * @param FrontControllerInterface $subject + * @param RequestInterface $request + * @return void + */ + public function beforeDispatch( + FrontControllerInterface $subject, + RequestInterface $request + ) { + // Only process frontend area requests + try { + $areaCode = $this->appState->getAreaCode(); + if ($areaCode !== \Magento\Framework\App\Area::AREA_FRONTEND) { + return; + } + } catch (\Exception $e) { + // Area not set yet, skip + return; + } + + // Check if customer is already logged in + if ($this->customerSession->isLoggedIn()) { + return; + } + + // Check for auto-login header + $credentials = $request->getHeader(self::AUTO_LOGIN_HEADER); + if (!$credentials) { + return; + } + + // Parse email:password format + $parts = explode(':', $credentials, 2); + if (count($parts) !== 2) { + $this->logger->warning('Customer auto-login header must be in format email:password'); + return; + } + + list($email, $password) = $parts; + + // Validate both email and password are present + if (empty($email) || empty($password)) { + $this->logger->warning('Customer auto-login email and password cannot be empty'); + return; + } + + try { + // Authenticate customer using Magento's official API + $customer = $this->accountManagement->authenticate($email, $password); + + // Login the customer + $this->customerSession->setCustomerDataAsLoggedIn($customer); + + $this->logger->info('Customer auto-login successful for: ' . $email); + + } catch (\Magento\Framework\Exception\InvalidEmailOrPasswordException $e) { + $this->logger->warning('Customer auto-login failed - invalid credentials for: ' . $email); + } catch (\Exception $e) { + $this->logger->error('Customer auto-login failed: ' . $e->getMessage()); + } + } +} diff --git a/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/etc/di.xml b/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/etc/di.xml new file mode 100644 index 0000000..8d6a6fd --- /dev/null +++ b/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/etc/di.xml @@ -0,0 +1,12 @@ + + + + + + + + + diff --git a/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/etc/module.xml b/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/etc/module.xml new file mode 100644 index 0000000..2460e6f --- /dev/null +++ b/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/etc/module.xml @@ -0,0 +1,9 @@ + + + + + + + + diff --git a/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/registration.php b/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/registration.php new file mode 100644 index 0000000..db8efd0 --- /dev/null +++ b/dev/environments/docker/sites/shopping/docker_overrides/CustomerAutoLogin/registration.php @@ -0,0 +1,9 @@ + /etc/my.cnf.d/optimize.cnf << EOF +[mysqld] +innodb_buffer_pool_size = ${MYSQL_INNODB_BUFFER_POOL_SIZE:-256M} +max_connections = ${MYSQL_MAX_CONNECTIONS:-20} +EOF + echo "[entrypoint] MySQL buffer pool: ${MYSQL_INNODB_BUFFER_POOL_SIZE:-256M}, max connections: ${MYSQL_MAX_CONNECTIONS:-20}" +fi + +# Apply Elasticsearch heap optimization +if [ -n "$ES_JAVA_OPTS" ]; then + # Update supervisor config to pass ES_JAVA_OPTS + if grep -q "ES_JAVA_OPTS" /etc/supervisor.d/elasticsearch.ini; then + echo "[entrypoint] ES_JAVA_OPTS already configured" + else + sed -i "s|command=su elastico -c \"ES_JAVA_HOME=/usr elasticsearch\"|command=su elastico -c \"ES_JAVA_HOME=/usr ES_JAVA_OPTS='$ES_JAVA_OPTS' elasticsearch\"|" /etc/supervisor.d/elasticsearch.ini + echo "[entrypoint] Elasticsearch heap set to: $ES_JAVA_OPTS" + fi +fi + +# Call original entrypoint +exec /docker-entrypoint.sh "$@" diff --git a/dev/environments/docker/sites/shopping/docker_overrides/etc/supervisor.d/env-ctrl-init.ini b/dev/environments/docker/sites/shopping/docker_overrides/etc/supervisor.d/env-ctrl-init.ini new file mode 100644 index 0000000..d3b0dda --- /dev/null +++ b/dev/environments/docker/sites/shopping/docker_overrides/etc/supervisor.d/env-ctrl-init.ini @@ -0,0 +1,11 @@ +[program:env-ctrl-init] +command=/bin/sh -c '[ "$WA_ENV_CTRL_ENABLE" != "true" ] && exit 0; exec /usr/local/bin/env-ctrl init' +autostart=true +autorestart=false +startsecs=0 +exitcodes=0 +priority=999 +stdout_logfile=/tmp/env-ctrl-init.log +stdout_logfile_maxbytes=10MB +stderr_logfile=/tmp/env-ctrl-init.log +stderr_logfile_maxbytes=10MB diff --git a/dev/environments/docker/sites/shopping/docker_overrides/etc/supervisor.d/env-ctrl.ini b/dev/environments/docker/sites/shopping/docker_overrides/etc/supervisor.d/env-ctrl.ini new file mode 100644 index 0000000..0517819 --- /dev/null +++ b/dev/environments/docker/sites/shopping/docker_overrides/etc/supervisor.d/env-ctrl.ini @@ -0,0 +1,9 @@ +[program:env-ctrl] +command=/bin/sh -c '[ "$WA_ENV_CTRL_ENABLE" != "true" ] && exit 0; exec /usr/local/bin/env-ctrl serve --port %(ENV_WA_ENV_CTRL_PORT)s' +autostart=true +autorestart=unexpected +startsecs=0 +stdout_logfile=/tmp/env-ctrl.log +stdout_logfile_maxbytes=10MB +stderr_logfile=/tmp/env-ctrl.log +stderr_logfile_maxbytes=10MB diff --git a/dev/environments/docker/sites/shopping/scripts/00_apply_patches.sh b/dev/environments/docker/sites/shopping/scripts/00_apply_patches.sh new file mode 100755 index 0000000..b36901a --- /dev/null +++ b/dev/environments/docker/sites/shopping/scripts/00_apply_patches.sh @@ -0,0 +1,76 @@ +#!/bin/bash +# +# Bootstrap and apply patches for Shopping (Magento) container +# Sets up env-ctrl, copies entrypoint, and applies patches +# +set -euo pipefail + +WA_ENV_CTRL_ROOT="${WA_ENV_CTRL_ROOT:-/opt}" +OVERRIDES_DIR="/build-site/docker_overrides" +AUTOLOGIN_DIR="/var/www/magento2/app/code/WebArena/CustomerAutoLogin" + +echo "========================================" +echo "[bootstrap] Setting up env-ctrl..." +echo "========================================" + +echo "[bootstrap] Copying env-ctrl package to ${WA_ENV_CTRL_ROOT}/environment_control" +cp -r "${BUILD_ENV_CTRL_SRC}" "${WA_ENV_CTRL_ROOT}/environment_control" + +echo "[bootstrap] Creating env-ctrl wrapper at /usr/local/bin/env-ctrl" +cat > /usr/local/bin/env-ctrl << EOF +#!/bin/sh +export PYTHONPATH="${WA_ENV_CTRL_ROOT}" +exec python3 -m environment_control.cli "\$@" +EOF +chmod +x /usr/local/bin/env-ctrl + +echo "========================================" +echo "[setup] Copying entrypoint..." +echo "========================================" + +echo "[setup] Copying /build-site/entrypoint.sh to /entrypoint.sh" +cp /build-site/entrypoint.sh /entrypoint.sh +chmod +x /entrypoint.sh + +echo "========================================" +echo "[setup] Applying patches..." +echo "========================================" + +echo "[patch] Creating CustomerAutoLogin module directory structure at ${AUTOLOGIN_DIR}" +mkdir -p "${AUTOLOGIN_DIR}/etc" "${AUTOLOGIN_DIR}/Plugin" + +echo "[patch] Copying registration.php to ${AUTOLOGIN_DIR}/registration.php" +cp "${OVERRIDES_DIR}/CustomerAutoLogin/registration.php" "${AUTOLOGIN_DIR}/registration.php" + +echo "[patch] Copying module.xml to ${AUTOLOGIN_DIR}/etc/module.xml" +cp "${OVERRIDES_DIR}/CustomerAutoLogin/etc/module.xml" "${AUTOLOGIN_DIR}/etc/module.xml" + +echo "[patch] Copying di.xml to ${AUTOLOGIN_DIR}/etc/di.xml" +cp "${OVERRIDES_DIR}/CustomerAutoLogin/etc/di.xml" "${AUTOLOGIN_DIR}/etc/di.xml" + +echo "[patch] Copying CustomerAutoLoginPlugin.php to ${AUTOLOGIN_DIR}/Plugin/CustomerAutoLoginPlugin.php" +cp "${OVERRIDES_DIR}/CustomerAutoLogin/Plugin/CustomerAutoLoginPlugin.php" "${AUTOLOGIN_DIR}/Plugin/CustomerAutoLoginPlugin.php" + +echo "========================================" +echo "[setup] Fixing ownership..." +echo "========================================" + +echo "[setup] Setting ownership to www-data:www-data for ${AUTOLOGIN_DIR}" +chown -R www-data:www-data "${AUTOLOGIN_DIR}" + +echo "========================================" +echo "[setup] Enabling Magento module..." +echo "========================================" + +echo "[magento] Running: php /var/www/magento2/bin/magento module:enable WebArena_CustomerAutoLogin" +php /var/www/magento2/bin/magento module:enable WebArena_CustomerAutoLogin + +echo "[magento] Running: php /var/www/magento2/bin/magento setup:di:compile" +php /var/www/magento2/bin/magento setup:di:compile + +echo "[magento] Running: php /var/www/magento2/bin/magento cache:flush" +php /var/www/magento2/bin/magento cache:flush + +echo "========================================" +echo "[setup] Done" +echo "========================================" diff --git a/dev/environments/docker/sites/shopping/scripts/20_optimize.sh b/dev/environments/docker/sites/shopping/scripts/20_optimize.sh new file mode 100755 index 0000000..e636892 --- /dev/null +++ b/dev/environments/docker/sites/shopping/scripts/20_optimize.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Image optimization script for Shopping (Magento) container +# Resizes and compresses product catalog images using mogrify +# +set -euo pipefail + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" +IMAGE_DIR="/var/www/magento2/pub/media/catalog/product" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Would optimize JPEG images in $IMAGE_DIR" + exit 0 +fi +IMAGE_MAX_SIZE="400x400>" +JPEG_QUALITY=60 +THREADS=$(nproc 2>/dev/null || echo 4) + +echo "========================================" +echo "[optimize] Starting image optimization..." +echo "[optimize] Directory: $IMAGE_DIR" +echo "[optimize] Max size: $IMAGE_MAX_SIZE" +echo "[optimize] Quality: $JPEG_QUALITY" +echo "[optimize] Threads: $THREADS" +echo "========================================" + +if ! command -v parallel &> /dev/null; then + echo "[optimize] Installing GNU parallel..." + apk add --no-cache parallel || true +fi + +echo "[optimize] Running mogrify..." +find "$IMAGE_DIR" -type f \( -iname '*.jpg' -o -iname '*.jpeg' \) -print0 2>/dev/null | \ + parallel -0 -n 50 -j "$THREADS" "mogrify -resize '$IMAGE_MAX_SIZE' -quality $JPEG_QUALITY -strip {} && printf '.'" 2>/dev/null || true +echo + +echo "[optimize] Removing product image cache..." +rm -rf /var/www/magento2/pub/media/catalog/product/cache +install -d -o www-data -g www-data -m 2775 /var/www/magento2/pub/media/catalog/product/cache + +echo "========================================" +echo "[optimize] Done" +echo "========================================" diff --git a/dev/environments/docker/sites/shopping/scripts/60_cleanup.sh b/dev/environments/docker/sites/shopping/scripts/60_cleanup.sh new file mode 100755 index 0000000..d182f7e --- /dev/null +++ b/dev/environments/docker/sites/shopping/scripts/60_cleanup.sh @@ -0,0 +1,94 @@ +#!/bin/bash +# +# Cleanup script for Shopping (Magento) container +# Removes logs, caches, and temporary files to reduce image size +# +set -euo pipefail +set -x # Echo all commands + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Skipping cleanup" + exit 0 +fi + +echo "========================================" +echo "[cleanup] Starting cleanup..." +echo "========================================" + +# ----------------------------------------- +# Application cleanup +# ----------------------------------------- + +echo "[cleanup] Removing Elasticsearch logs..." +rm -rf /usr/share/java/elasticsearch/logs/* + +echo "[cleanup] Removing Magento logs and caches..." +rm -rf /var/www/magento2/var/log/* +rm -rf /var/www/magento2/var/cache/* +rm -rf /var/www/magento2/var/page_cache/* +rm -rf /var/www/magento2/var/session/* +rm -rf /var/www/magento2/var/view_preprocessed/* +rm -rf /var/www/magento2/generated/* +rm -rf /var/www/magento2/dev/tests + +echo "[cleanup] Removing Composer cache..." +rm -rf /root/.composer/cache/* +rm -rf /var/www/.composer/cache/* + +echo "[cleanup] Removing apt cache..." +rm -rf /var/cache/apt/archives/* +rm -rf /var/lib/apt/lists/* + +echo "[cleanup] Removing service logs..." +rm -rf /var/log/mysql/* +rm -rf /var/log/nginx/* +rm -rf /var/log/*.log +rm -rf /var/log/*/*.log + +echo "[cleanup] Removing temp files..." +rm -rf /tmp/*.log +rm -rf /tmp/magento* +rm -rf /tmp/elasticsearch-* +rm -rf /tmp/*.jpg +rm -rf /tmp/sess_* +rm -rf /tmp/hsperfdata_* + +echo "[cleanup] Removing stale PID files..." +rm -f /var/lib/mysql/*.pid +rm -f /run/*.pid +rm -f /supervisord.pid + +echo "[cleanup] Removing MySQL recovery logs..." +rm -f /var/lib/mysql/ddl_recovery*.log + +echo "[cleanup] Removing Alpine package cache..." +rm -rf /var/cache/apk/* + +# ----------------------------------------- +# Shutdown services +# ----------------------------------------- + +echo "[cleanup] Shutting down services..." +if pgrep -x supervisord > /dev/null; then + echo "[cleanup] Supervisord is running, stopping all services..." + supervisorctl stop all 2>/dev/null || true + pkill supervisord 2>/dev/null || true + sleep 2 +else + echo "[cleanup] Supervisord not running, skipping service shutdown" +fi + +# ----------------------------------------- +# Final cleanup +# ----------------------------------------- + +echo "[cleanup] Removing general temp files..." +rm -rf /tmp/* 2>/dev/null || true +rm -rf /var/cache/* 2>/dev/null || true +rm -rf /var/log/* 2>/dev/null || true + +echo "========================================" +echo "[cleanup] Done" +echo "========================================" diff --git a/dev/environments/docker/sites/shopping/scripts/90_verify.sh b/dev/environments/docker/sites/shopping/scripts/90_verify.sh new file mode 100755 index 0000000..0fd1549 --- /dev/null +++ b/dev/environments/docker/sites/shopping/scripts/90_verify.sh @@ -0,0 +1,99 @@ +#!/bin/bash +# +# Verify Shopping (Magento) database integrity after cleanup +# Starts MySQL, runs checks, then stops it +# +set -euo pipefail + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Skipping verification" + exit 0 +fi + +echo "========================================" +echo "[verify] Starting database verification..." +echo "========================================" + +MYSQL_DATA="/var/lib/mysql" +MYSQL="mysql -u magentouser -pMyPassword -N -B magentodb" + +# Start MySQL +echo "[verify] Starting MySQL..." +mysqld_safe --skip-grant-tables & +sleep 10 + +# Run verification queries +echo "" +echo "=== TABLE ROW COUNTS ===" + +verify_count() { + local table="$1" + local min_expected="$2" + local count + count=$($MYSQL -e "SELECT COUNT(*) FROM ${table};" 2>/dev/null || echo "ERROR") + + if [ "$count" = "ERROR" ]; then + echo " ${table}: ERROR - query failed" + return 1 + elif [ "$count" -lt "$min_expected" ]; then + echo " ${table}: ${count} (WARN: expected >= ${min_expected})" + return 1 + else + echo " ${table}: ${count} โœ“" + return 0 + fi +} + +ERRORS=0 + +# Core tables with minimum expected counts +verify_count "customer_entity" 20 || ((ERRORS++)) +verify_count "catalog_product_entity" 100000 || ((ERRORS++)) +verify_count "catalog_category_entity" 300 || ((ERRORS++)) +verify_count "sales_order" 150 || ((ERRORS++)) +verify_count "review" 300000 || ((ERRORS++)) +verify_count "admin_user" 1 || ((ERRORS++)) + +echo "" +echo "=== DATABASE SIZE ===" +DB_SIZE=$($MYSQL -e "SELECT ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) FROM information_schema.tables WHERE table_schema = 'magentodb';" 2>/dev/null) +echo " Total: ${DB_SIZE} MB" + +echo "" +echo "=== SAMPLE DATA CHECK ===" +# Verify admin user exists +ADMIN_USER=$($MYSQL -e "SELECT username FROM admin_user WHERE user_id = 1;" 2>/dev/null) +if [ "$ADMIN_USER" = "admin" ]; then + echo " Admin user: exists โœ“" +else + echo " Admin user: MISSING" + ((ERRORS++)) +fi + +# Verify sample customer exists +CUSTOMER=$($MYSQL -e "SELECT email FROM customer_entity ORDER BY entity_id LIMIT 1;" 2>/dev/null) +if [ -n "$CUSTOMER" ]; then + echo " Sample customer: ${CUSTOMER} โœ“" +else + echo " Sample customer: MISSING" + ((ERRORS++)) +fi + +# Stop MySQL +echo "" +echo "[verify] Stopping MySQL..." +mysqladmin shutdown 2>/dev/null || killall mysqld 2>/dev/null || true +sleep 5 + +echo "" +echo "========================================" +if [ "$ERRORS" -gt 0 ]; then + echo "[verify] FAILED - ${ERRORS} error(s) found" + echo "========================================" + exit 1 +else + echo "[verify] PASSED - All checks OK" + echo "========================================" +fi diff --git a/dev/environments/docker/sites/shopping_admin/Dockerfile b/dev/environments/docker/sites/shopping_admin/Dockerfile new file mode 100644 index 0000000..63f43d0 --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/Dockerfile @@ -0,0 +1,48 @@ +# Shopping Admin Environment - Final Image +# +# Base image created by: ./create_optimized_image.sh +# This Dockerfile adds env-ctrl on top of the optimized base. +# +# Build (from repo root): +# docker build -t shopping_admin:latest -f dev/environments/docker/sites/shopping_admin/Dockerfile . +# +# Run: +# docker run -d -p 7780:80 -p 8877:8877 shopping_admin:latest + +ARG BASE_IMAGE=am1n3e/webarena-verified-shopping-admin-base:latest +FROM ${BASE_IMAGE} +ARG ENV_CTRL_PORT=8877 + +WORKDIR /var/www/html + +# Environment control configuration +ENV WA_ENV_CTRL_TYPE=shopping_admin +ENV WA_ENV_CTRL_AVAILABLE=true +ENV WA_ENV_CTRL_ENABLE=true +ENV WA_ENV_CTRL_PORT=${ENV_CTRL_PORT} +ENV WA_ENV_CTRL_EXTERNAL_SITE_URL=http://localhost:7780 +ENV WA_CONTAINER_PORT=80 + +# Resource optimization settings (applied by custom entrypoint) +ENV NGINX_WORKER_PROCESSES=2 +ENV ES_JAVA_OPTS="-Xms512m -Xmx512m" +ENV MYSQL_INNODB_BUFFER_POOL_SIZE=256M +ENV MYSQL_MAX_CONNECTIONS=20 + +# Copy environment_control package and env-ctrl CLI +COPY packages/environment_control/environment_control /usr/local/environment_control +COPY --chmod=755 packages/environment_control/environment_control/env-ctrl /usr/local/bin/env-ctrl + +# Copy env-ctrl supervisor configs +COPY dev/environments/docker/sites/shopping_admin/docker_overrides/etc/supervisor.d/env-ctrl.ini /etc/supervisor.d/env-ctrl.ini +COPY dev/environments/docker/sites/shopping_admin/docker_overrides/etc/supervisor.d/env-ctrl-init.ini /etc/supervisor.d/env-ctrl-init.ini + +# Copy custom entrypoint that applies resource optimizations +COPY --chmod=755 dev/environments/docker/sites/shopping_admin/docker_overrides/entrypoint.sh /custom-entrypoint.sh + +# Expose ports (nginx: 80, env-ctrl) +EXPOSE 80 ${ENV_CTRL_PORT} + +# Use custom entrypoint (calls original entrypoint after applying optimizations) +ENTRYPOINT ["/custom-entrypoint.sh"] +CMD ["supervisord", "-n", "-j", "/supervisord.pid"] diff --git a/dev/environments/docker/sites/shopping_admin/README.md b/dev/environments/docker/sites/shopping_admin/README.md new file mode 100644 index 0000000..cc133bc --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/README.md @@ -0,0 +1,28 @@ +# Shopping Admin (Magento) + +Port: 7780 + +## Fixes + +### Header Authentication + +**Problem:** UI login via Playwright is slow and requires maintaining test credentials. + +**Fix:** Magento plugin intercepts admin requests and authenticates via `X-M2-Admin-Auto-Login: username:password` header. + +**Files:** `AutoLoginPlugin.php`, `di.xml`, `module.xml`, `registration.php` + +**Usage:** +```python +context = await browser.new_context( + extra_http_headers={"X-M2-Admin-Auto-Login": "admin:admin1234"} +) +``` + +### Mass Action Protection + +**Problem:** Bulk delete actions in admin panel can accidentally destroy test data. + +**Fix:** Plugins disable mass delete actions for products and reviews. + +**Files:** `DisableProductMassActionsPlugin.php`, `DisableReviewMassActionsPlugin.php` diff --git a/dev/environments/docker/sites/shopping_admin/docker_overrides/AutoLoginPlugin.php b/dev/environments/docker/sites/shopping_admin/docker_overrides/AutoLoginPlugin.php new file mode 100644 index 0000000..f1ea6ab --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/docker_overrides/AutoLoginPlugin.php @@ -0,0 +1,125 @@ +auth = $auth; + $this->appState = $appState; + $this->logger = $logger; + } + + /** + * Before front controller dispatch, check for auto-login header. + * + * @param FrontControllerInterface $subject + * @param RequestInterface $request + * @return void + */ + public function beforeDispatch( + FrontControllerInterface $subject, + RequestInterface $request + ) { + // Only process admin area requests + try { + $areaCode = $this->appState->getAreaCode(); + if ($areaCode !== \Magento\Framework\App\Area::AREA_ADMINHTML) { + return; + } + } catch (\Exception $e) { + // Area not set yet, skip + return; + } + + // Check if user is already logged in + if ($this->auth->isLoggedIn()) { + return; + } + + // Check for auto-login header + $credentials = $request->getHeader(self::AUTO_LOGIN_HEADER); + if (!$credentials) { + return; + } + + // Parse username:password format + $parts = explode(':', $credentials, 2); + if (count($parts) !== 2) { + $this->logger->warning('Auto-login header must be in format username:password'); + return; + } + + list($username, $password) = $parts; + + // Validate both username and password are present + if (empty($username) || empty($password)) { + $this->logger->warning('Auto-login username and password cannot be empty'); + return; + } + + try { + // Use official Magento Auth API to login + // This properly handles all validation, events, and session setup + $this->auth->login($username, $password); + + // Log the successful auto-login + $this->logger->info('Auto-login successful for user: ' . $username); + + } catch (\Magento\Framework\Exception\AuthenticationException $e) { + // Authentication failed (invalid username/password, inactive user, etc.) + $this->logger->warning('Auto-login authentication failed: ' . $e->getMessage()); + } catch (\Exception $e) { + // Other errors (ReCaptcha, TFA, etc.) + $this->logger->error('Auto-login failed: ' . $e->getMessage()); + } + } +} diff --git a/dev/environments/docker/sites/shopping_admin/docker_overrides/DisableProductMassActionsPlugin.php b/dev/environments/docker/sites/shopping_admin/docker_overrides/DisableProductMassActionsPlugin.php new file mode 100644 index 0000000..b435185 --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/docker_overrides/DisableProductMassActionsPlugin.php @@ -0,0 +1,39 @@ +removeItem($actionId); + } + } + return $result; + } +} diff --git a/dev/environments/docker/sites/shopping_admin/docker_overrides/di.xml b/dev/environments/docker/sites/shopping_admin/docker_overrides/di.xml new file mode 100644 index 0000000..1b616e0 --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/docker_overrides/di.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + diff --git a/dev/environments/docker/sites/shopping_admin/docker_overrides/entrypoint.sh b/dev/environments/docker/sites/shopping_admin/docker_overrides/entrypoint.sh new file mode 100644 index 0000000..b2396c1 --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/docker_overrides/entrypoint.sh @@ -0,0 +1,37 @@ +#!/bin/sh +# +# Custom entrypoint for Shopping Admin container +# Applies resource optimizations from environment variables, then calls original entrypoint +# + +set -e + +# Apply Nginx worker optimization +if [ -n "$NGINX_WORKER_PROCESSES" ]; then + sed -i "s/worker_processes.*/worker_processes $NGINX_WORKER_PROCESSES;/" /etc/nginx/nginx.conf + echo "[entrypoint] Nginx workers set to: $NGINX_WORKER_PROCESSES" +fi + +# Apply MySQL optimizations +if [ -n "$MYSQL_INNODB_BUFFER_POOL_SIZE" ] || [ -n "$MYSQL_MAX_CONNECTIONS" ]; then + cat > /etc/my.cnf.d/optimize.cnf << EOF +[mysqld] +innodb_buffer_pool_size = ${MYSQL_INNODB_BUFFER_POOL_SIZE:-256M} +max_connections = ${MYSQL_MAX_CONNECTIONS:-20} +EOF + echo "[entrypoint] MySQL buffer pool: ${MYSQL_INNODB_BUFFER_POOL_SIZE:-256M}, max connections: ${MYSQL_MAX_CONNECTIONS:-20}" +fi + +# Apply Elasticsearch heap optimization +if [ -n "$ES_JAVA_OPTS" ]; then + # Update supervisor config to pass ES_JAVA_OPTS + if grep -q "ES_JAVA_OPTS" /etc/supervisor.d/elasticsearch.ini; then + echo "[entrypoint] ES_JAVA_OPTS already configured" + else + sed -i "s|command=su elastico -c \"ES_JAVA_HOME=/usr elasticsearch\"|command=su elastico -c \"ES_JAVA_HOME=/usr ES_JAVA_OPTS='$ES_JAVA_OPTS' elasticsearch\"|" /etc/supervisor.d/elasticsearch.ini + echo "[entrypoint] Elasticsearch heap set to: $ES_JAVA_OPTS" + fi +fi + +# Call original entrypoint +exec /docker-entrypoint.sh "$@" diff --git a/dev/environments/docker/sites/shopping_admin/docker_overrides/etc/supervisor.d/env-ctrl-init.ini b/dev/environments/docker/sites/shopping_admin/docker_overrides/etc/supervisor.d/env-ctrl-init.ini new file mode 100644 index 0000000..d3b0dda --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/docker_overrides/etc/supervisor.d/env-ctrl-init.ini @@ -0,0 +1,11 @@ +[program:env-ctrl-init] +command=/bin/sh -c '[ "$WA_ENV_CTRL_ENABLE" != "true" ] && exit 0; exec /usr/local/bin/env-ctrl init' +autostart=true +autorestart=false +startsecs=0 +exitcodes=0 +priority=999 +stdout_logfile=/tmp/env-ctrl-init.log +stdout_logfile_maxbytes=10MB +stderr_logfile=/tmp/env-ctrl-init.log +stderr_logfile_maxbytes=10MB diff --git a/dev/environments/docker/sites/shopping_admin/docker_overrides/etc/supervisor.d/env-ctrl.ini b/dev/environments/docker/sites/shopping_admin/docker_overrides/etc/supervisor.d/env-ctrl.ini new file mode 100644 index 0000000..0517819 --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/docker_overrides/etc/supervisor.d/env-ctrl.ini @@ -0,0 +1,9 @@ +[program:env-ctrl] +command=/bin/sh -c '[ "$WA_ENV_CTRL_ENABLE" != "true" ] && exit 0; exec /usr/local/bin/env-ctrl serve --port %(ENV_WA_ENV_CTRL_PORT)s' +autostart=true +autorestart=unexpected +startsecs=0 +stdout_logfile=/tmp/env-ctrl.log +stdout_logfile_maxbytes=10MB +stderr_logfile=/tmp/env-ctrl.log +stderr_logfile_maxbytes=10MB diff --git a/dev/environments/docker/sites/shopping_admin/docker_overrides/module.xml b/dev/environments/docker/sites/shopping_admin/docker_overrides/module.xml new file mode 100644 index 0000000..e67f2bb --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/docker_overrides/module.xml @@ -0,0 +1,12 @@ + + + + + + + + + + + diff --git a/dev/environments/docker/sites/shopping_admin/docker_overrides/registration.php b/dev/environments/docker/sites/shopping_admin/docker_overrides/registration.php new file mode 100644 index 0000000..eda5c4b --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/docker_overrides/registration.php @@ -0,0 +1,9 @@ + /usr/local/bin/env-ctrl << EOF +#!/bin/sh +export PYTHONPATH="${WA_ENV_CTRL_ROOT}" +exec python3 -m environment_control.cli "\$@" +EOF +chmod +x /usr/local/bin/env-ctrl + +echo "========================================" +echo "[setup] Copying entrypoint..." +echo "========================================" + +echo "[setup] Copying /build-site/entrypoint.sh to /entrypoint.sh" +cp /build-site/entrypoint.sh /entrypoint.sh +chmod +x /entrypoint.sh + +echo "========================================" +echo "[setup] Applying patches..." +echo "========================================" + +echo "[patch] Creating AutoLogin module directory structure at ${AUTOLOGIN_DIR}" +mkdir -p "${AUTOLOGIN_DIR}/etc" "${AUTOLOGIN_DIR}/Plugin" + +echo "[patch] Copying registration.php to ${AUTOLOGIN_DIR}/registration.php" +cp "${OVERRIDES_DIR}/registration.php" "${AUTOLOGIN_DIR}/registration.php" + +echo "[patch] Copying module.xml to ${AUTOLOGIN_DIR}/etc/module.xml" +cp "${OVERRIDES_DIR}/module.xml" "${AUTOLOGIN_DIR}/etc/module.xml" + +echo "[patch] Copying di.xml to ${AUTOLOGIN_DIR}/etc/di.xml" +cp "${OVERRIDES_DIR}/di.xml" "${AUTOLOGIN_DIR}/etc/di.xml" + +echo "[patch] Copying AutoLoginPlugin.php to ${AUTOLOGIN_DIR}/Plugin/AutoLoginPlugin.php" +cp "${OVERRIDES_DIR}/AutoLoginPlugin.php" "${AUTOLOGIN_DIR}/Plugin/AutoLoginPlugin.php" + +echo "========================================" +echo "[setup] Fixing ownership..." +echo "========================================" + +echo "[setup] Setting ownership to www-data:www-data for ${AUTOLOGIN_DIR}" +chown -R www-data:www-data "${AUTOLOGIN_DIR}" + +echo "========================================" +echo "[setup] Enabling Magento module..." +echo "========================================" + +echo "[magento] Running: php /var/www/magento2/bin/magento module:enable WebArena_AutoLogin" +php /var/www/magento2/bin/magento module:enable WebArena_AutoLogin + +echo "[magento] Running: php /var/www/magento2/bin/magento setup:di:compile" +php /var/www/magento2/bin/magento setup:di:compile + +echo "[magento] Running: php /var/www/magento2/bin/magento cache:flush" +php /var/www/magento2/bin/magento cache:flush + +echo "========================================" +echo "[setup] Done" +echo "========================================" diff --git a/dev/environments/docker/sites/shopping_admin/scripts/10_post_patch.sh b/dev/environments/docker/sites/shopping_admin/scripts/10_post_patch.sh new file mode 100755 index 0000000..3bd2cc7 --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/scripts/10_post_patch.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# +# Post-patch script for Shopping Admin (Magento) container +# Starts services, configures base URL, compiles Magento module +# +set -euo pipefail +set -x + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" + +echo "========================================" +echo "[services] Starting services and waiting for ready..." +echo "========================================" + +env-ctrl start --wait --timeout "${BUILD_TIMEOUT:-300}" + +echo "[services] Configuring base URL: ${BUILD_BASE_URL}" +env-ctrl setup init --base-url "${BUILD_BASE_URL}" + +echo "========================================" +echo "[magento] Compiling Magento module..." +echo "========================================" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Skipping Magento compilation" +else + php /var/www/magento2/bin/magento module:enable WebArena_AutoLogin + php /var/www/magento2/bin/magento setup:di:compile + php /var/www/magento2/bin/magento cache:flush +fi + +echo "[magento] Stopping services..." +env-ctrl stop + +echo "========================================" +echo "[magento] Done" +echo "========================================" diff --git a/dev/environments/docker/sites/shopping_admin/scripts/30_optimize.sh b/dev/environments/docker/sites/shopping_admin/scripts/30_optimize.sh new file mode 100755 index 0000000..86be70a --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/scripts/30_optimize.sh @@ -0,0 +1,43 @@ +#!/bin/bash +# +# Image optimization script for Shopping Admin (Magento) container +# Resizes and compresses product catalog images using mogrify +# +set -euo pipefail + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" +IMAGE_DIR="/var/www/magento2/pub/media/catalog/product" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Would optimize JPEG images in $IMAGE_DIR" + exit 0 +fi +IMAGE_MAX_SIZE="400x400>" +JPEG_QUALITY=60 +THREADS=$(nproc 2>/dev/null || echo 4) + +echo "========================================" +echo "[optimize] Starting image optimization..." +echo "[optimize] Directory: $IMAGE_DIR" +echo "[optimize] Max size: $IMAGE_MAX_SIZE" +echo "[optimize] Quality: $JPEG_QUALITY" +echo "[optimize] Threads: $THREADS" +echo "========================================" + +echo "[optimize] Removing product image cache..." +rm -rf /var/www/magento2/pub/media/catalog/product/cache +install -d -o www-data -g www-data -m 2775 /var/www/magento2/pub/media/catalog/product/cache + +if ! command -v parallel &> /dev/null; then + echo "[optimize] Installing GNU parallel..." + apk add --no-cache parallel || true +fi + +echo "[optimize] Running mogrify..." +find "$IMAGE_DIR" -type f \( -iname '*.jpg' -o -iname '*.jpeg' \) -print0 2>/dev/null | \ + parallel -0 -n 50 -j "$THREADS" "mogrify -resize '$IMAGE_MAX_SIZE' -quality $JPEG_QUALITY -strip {} && printf '.'" 2>/dev/null || true +echo + +echo "========================================" +echo "[optimize] Done" +echo "========================================" diff --git a/dev/environments/docker/sites/shopping_admin/scripts/60_cleanup.sh b/dev/environments/docker/sites/shopping_admin/scripts/60_cleanup.sh new file mode 100755 index 0000000..cd945cb --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/scripts/60_cleanup.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# +# Cleanup script for Shopping Admin (Magento) container +# Removes logs, caches, and temporary files to reduce image size +# +set -euo pipefail +set -x # Echo all commands + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Skipping cleanup" + exit 0 +fi + +echo "========================================" +echo "[cleanup] Starting cleanup..." +echo "========================================" + +# ----------------------------------------- +# Application cleanup +# ----------------------------------------- + +echo "[cleanup] Removing Elasticsearch logs..." +rm -rf /usr/share/java/elasticsearch/logs/* + +echo "[cleanup] Removing Magento logs and caches..." +rm -rf /var/www/magento2/var/log/* +rm -rf /var/www/magento2/var/cache/* +rm -rf /var/www/magento2/var/page_cache/* +rm -rf /var/www/magento2/var/session/* +rm -rf /var/www/magento2/var/view_preprocessed/* +rm -rf /var/www/magento2/generated/* +rm -rf /var/www/magento2/dev/tests + +echo "[cleanup] Removing Composer cache..." +rm -rf /root/.composer/cache/* +rm -rf /var/www/.composer/cache/* + +echo "[cleanup] Removing apt cache..." +rm -rf /var/cache/apt/archives/* +rm -rf /var/lib/apt/lists/* + +echo "[cleanup] Removing service logs..." +rm -rf /var/log/mysql/* +rm -rf /var/log/nginx/* +rm -rf /var/log/*.log +rm -rf /var/log/*/*.log + +echo "[cleanup] Removing temp files..." +rm -rf /tmp/*.log +rm -rf /tmp/magento* +rm -rf /tmp/elasticsearch-* +rm -rf /tmp/*.jpg +rm -rf /tmp/sess_* +rm -rf /tmp/hsperfdata_* + +echo "[cleanup] Removing stale PID files..." +rm -f /var/lib/mysql/*.pid +rm -f /run/*.pid +rm -f /supervisord.pid + +echo "[cleanup] Removing MySQL recovery logs..." +rm -f /var/lib/mysql/ddl_recovery*.log + +echo "[cleanup] Removing Alpine package cache..." +rm -rf /var/cache/apk/* + +# ----------------------------------------- +# Shutdown services +# ----------------------------------------- + +echo "[cleanup] Shutting down services..." +if pgrep -x supervisord > /dev/null; then + echo "[cleanup] Supervisord is running, stopping all services..." + supervisorctl stop all 2>/dev/null || true + pkill supervisord 2>/dev/null || true + sleep 2 +else + echo "[cleanup] Supervisord not running, skipping service shutdown" +fi + +# ----------------------------------------- +# Final cleanup +# ----------------------------------------- + +echo "[cleanup] Removing Elasticsearch temp dirs..." +rm -rf /tmp/elasticsearch-* 2>/dev/null || true + +echo "[cleanup] Removing general temp files..." +rm -rf /tmp/* 2>/dev/null || true +rm -rf /var/cache/* 2>/dev/null || true +rm -rf /var/log/* 2>/dev/null || true + +echo "========================================" +echo "[cleanup] Done" +echo "========================================" diff --git a/dev/environments/docker/sites/shopping_admin/scripts/90_verify.sh b/dev/environments/docker/sites/shopping_admin/scripts/90_verify.sh new file mode 100755 index 0000000..7edfce1 --- /dev/null +++ b/dev/environments/docker/sites/shopping_admin/scripts/90_verify.sh @@ -0,0 +1,101 @@ +#!/bin/bash +# +# Verify Shopping Admin (Magento) database integrity after cleanup +# Starts MySQL, runs checks, then stops it +# +set -euo pipefail + +BUILD_DRY_RUN="${BUILD_DRY_RUN:-false}" + +if [ "$BUILD_DRY_RUN" = "true" ]; then + echo "[dry-run] Skipping verification" + exit 0 +fi + +echo "========================================" +echo "[verify] Starting database verification..." +echo "========================================" + +MYSQL_DATA="/var/lib/mysql" +MYSQL="mysql -u magentouser -pMyPassword -N -B magentodb" + +# Start MySQL +echo "[verify] Starting MySQL..." +mysqld_safe --skip-grant-tables & +sleep 10 + +# Run verification queries +echo "" +echo "=== TABLE ROW COUNTS ===" + +verify_count() { + local table="$1" + local min_expected="$2" + local count + count=$($MYSQL -e "SELECT COUNT(*) FROM ${table};" 2>/dev/null || echo "ERROR") + + if [ "$count" = "ERROR" ]; then + echo " ${table}: ERROR - query failed" + return 1 + elif [ "$count" -lt "$min_expected" ]; then + echo " ${table}: ${count} (WARN: expected >= ${min_expected})" + return 1 + else + echo " ${table}: ${count} โœ“" + return 0 + fi +} + +ERRORS=0 + +# Core tables with minimum expected counts +verify_count "customer_entity" 60 || ((ERRORS++)) +verify_count "catalog_product_entity" 2000 || ((ERRORS++)) +verify_count "catalog_category_entity" 35 || ((ERRORS++)) +verify_count "sales_order" 300 || ((ERRORS++)) +verify_count "review" 300 || ((ERRORS++)) +verify_count "admin_user" 1 || ((ERRORS++)) +verify_count "cms_page" 5 || ((ERRORS++)) +verify_count "cms_block" 15 || ((ERRORS++)) + +echo "" +echo "=== DATABASE SIZE ===" +DB_SIZE=$($MYSQL -e "SELECT ROUND(SUM(data_length + index_length) / 1024 / 1024, 2) FROM information_schema.tables WHERE table_schema = 'magentodb';" 2>/dev/null) +echo " Total: ${DB_SIZE} MB" + +echo "" +echo "=== SAMPLE DATA CHECK ===" +# Verify admin user exists +ADMIN_USER=$($MYSQL -e "SELECT username FROM admin_user WHERE user_id = 1;" 2>/dev/null) +if [ "$ADMIN_USER" = "admin" ]; then + echo " Admin user: exists โœ“" +else + echo " Admin user: MISSING" + ((ERRORS++)) +fi + +# Verify sample product exists +PRODUCT=$($MYSQL -e "SELECT sku FROM catalog_product_entity ORDER BY entity_id LIMIT 1;" 2>/dev/null) +if [ -n "$PRODUCT" ]; then + echo " Sample product: ${PRODUCT} โœ“" +else + echo " Sample product: MISSING" + ((ERRORS++)) +fi + +# Stop MySQL +echo "" +echo "[verify] Stopping MySQL..." +mysqladmin shutdown 2>/dev/null || killall mysqld 2>/dev/null || true +sleep 5 + +echo "" +echo "========================================" +if [ "$ERRORS" -gt 0 ]; then + echo "[verify] FAILED - ${ERRORS} error(s) found" + echo "========================================" + exit 1 +else + echo "[verify] PASSED - All checks OK" + echo "========================================" +fi diff --git a/dev/environments/docker/sites/wikipedia/Dockerfile b/dev/environments/docker/sites/wikipedia/Dockerfile new file mode 100644 index 0000000..dab97ad --- /dev/null +++ b/dev/environments/docker/sites/wikipedia/Dockerfile @@ -0,0 +1,39 @@ +FROM ghcr.io/kiwix/kiwix-serve:3.3.0 + +ARG ENV_CTRL_PORT=8877 + +# Install Python and supervisor +RUN apk add --no-cache python3 supervisor + +# Copy environment_control package and env-ctrl CLI +COPY packages/environment_control/environment_control /usr/local/environment_control +COPY --chmod=755 packages/environment_control/environment_control/env-ctrl /usr/local/bin/env-ctrl + +# Copy redirect index.html +COPY dev/environments/docker/sites/wikipedia/index.html /srv/index.html + +# Copy supervisord config +COPY dev/environments/docker/sites/wikipedia/supervisord.conf /etc/supervisord.conf + +# Set environment type for env-ctrl +ENV WA_ENV_CTRL_TYPE=wikipedia + +# Default ZIM file path (can be overridden) +ENV WA_ENV_CTRL_ZIM_FILE=/data/wikipedia_en_all_maxi_2022-05.zim + +# Default ports (can be overridden) +ENV WA_CONTAINER_PORT=8080 +ENV WA_ENV_CTRL_KIWIX_PORT=${WA_CONTAINER_PORT} +ENV WA_ENV_CTRL_PORT=${ENV_CTRL_PORT} + +# Site URL for the dashboard link (can be overridden) +ENV WA_ENV_CTRL_EXTERNAL_SITE_URL=http://localhost:8888 + +# Expose ports (kiwix: 8080, env-ctrl) +EXPOSE 8080 ${ENV_CTRL_PORT} + +# Copy entrypoint script +COPY dev/environments/docker/sites/wikipedia/entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] diff --git a/dev/environments/docker/sites/wikipedia/entrypoint.sh b/dev/environments/docker/sites/wikipedia/entrypoint.sh new file mode 100644 index 0000000..910d573 --- /dev/null +++ b/dev/environments/docker/sites/wikipedia/entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/sh +set -e + +# Set ZIM file path from argument or use default +export WA_ENV_CTRL_ZIM_FILE="${1:-${WA_ENV_CTRL_ZIM_FILE:-/data/wikipedia_en_all_maxi_2022-05.zim}}" + +echo "Starting Wikipedia container with ZIM file: $WA_ENV_CTRL_ZIM_FILE" + +# Start supervisord (manages kiwix-serve and env-ctrl) +exec /usr/bin/supervisord -c /etc/supervisord.conf diff --git a/dev/environments/docker/sites/wikipedia/index.html b/dev/environments/docker/sites/wikipedia/index.html new file mode 100644 index 0000000..d2ddf57 --- /dev/null +++ b/dev/environments/docker/sites/wikipedia/index.html @@ -0,0 +1,10 @@ + + + + + + + +

Redirecting to Wikipedia...

+ + diff --git a/dev/environments/docker/sites/wikipedia/scripts/docker_setup.py b/dev/environments/docker/sites/wikipedia/scripts/docker_setup.py new file mode 100644 index 0000000..3cef2f5 --- /dev/null +++ b/dev/environments/docker/sites/wikipedia/scripts/docker_setup.py @@ -0,0 +1,92 @@ +"""Docker setup for wikipedia site. + +Sets up Docker volume for Wikipedia containing the ZIM file. +Uses a single container for download and copy operations. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from dev.environments.docker.utils import docker_setup_helpers +from dev.environments.settings import get_settings +from dev.utils import logging_utils +from webarena_verified.types.task import WebArenaSite + +if TYPE_CHECKING: + from invoke.context import Context + +SITE = WebArenaSite.WIKIPEDIA +VOLUME_SUFFIX = "data" +SCRIPT_PATH = Path(__file__).parent / "setup_volumes.sh" + + +def _print_dry_run_info(ctx: Context, data_urls: tuple[str, ...], data_dir: Path, vol_name: str) -> None: + """Print what would happen in dry-run mode, checking actual state.""" + # Check which files exist + logging_utils.print_info("Downloads:") + for url in data_urls: + filename = url.rsplit("/", 1)[-1] + file_path = data_dir / filename + if file_path.exists(): + size_gb = file_path.stat().st_size / (1024 * 1024 * 1024) + logging_utils.print_info(f" [SKIP] {filename} ({size_gb:.1f}GB exists)") + else: + logging_utils.print_info(f" [DOWNLOAD] {filename}") + logging_utils.print_info(f" {url}") + + # Check volume state + logging_utils.print_info("") + logging_utils.print_info("Volumes:") + if not docker_setup_helpers.volume_exists(ctx, vol_name): + logging_utils.print_info(f" [CREATE + COPY] {vol_name}") + elif docker_setup_helpers.volume_is_empty(ctx, vol_name): + logging_utils.print_info(f" [COPY] {vol_name}") + else: + logging_utils.print_info(f" [SKIP] {vol_name} (has data)") + + +def setup(ctx: Context, data_dir: Path, dry_run: bool = False, keep_downloads: bool = False) -> None: + """Set up wikipedia Docker volume from ZIM file. + + Uses a single container to download missing files and copy to empty volume. + + Args: + ctx: Invoke context for running commands. + data_dir: Directory to store/find data files. + dry_run: Show what would be done without doing it. + keep_downloads: Not used (files are kept in data_dir for reuse). + """ + settings = get_settings() + volume_prefix = settings.volume_prefix + data_urls = settings.wikipedia.data_urls + vol_name = docker_setup_helpers.get_volume_name(volume_prefix, SITE.value, VOLUME_SUFFIX) + + if dry_run: + _print_dry_run_info(ctx, data_urls, data_dir, vol_name) + logging_utils.print_info("") + logging_utils.print_info("Dry run complete - no changes made") + return + + # Ensure data directory exists + data_dir.mkdir(parents=True, exist_ok=True) + + # Create volume first (outside container) + docker_setup_helpers.create_volume(ctx, vol_name) + + # Run setup script in container + urls_str = " ".join(data_urls) + cmd = ( + f"docker run --rm " + f'-e DATA_URLS="{urls_str}" ' + f"-v {data_dir}:/data " + f"-v {SCRIPT_PATH}:/setup.sh:ro " + f"-v {vol_name}:/volume " + f"alpine sh /setup.sh" + ) + + logging_utils.print_info("Running setup in container...") + ctx.run(cmd, pty=True) + + logging_utils.print_success("Setup complete!") diff --git a/dev/environments/docker/sites/wikipedia/scripts/setup_volumes.sh b/dev/environments/docker/sites/wikipedia/scripts/setup_volumes.sh new file mode 100755 index 0000000..b68216f --- /dev/null +++ b/dev/environments/docker/sites/wikipedia/scripts/setup_volumes.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Setup script for wikipedia site volume +# Downloads ZIM file (if missing) and copies to volume (if empty) +# +# Environment variables: +# DATA_URLS - space-separated list of file URLs to download +# +# Volume mounts expected: +# /data - directory for downloaded files +# /volume - target volume for ZIM files + +set -e + +echo "=== Wikipedia Volume Setup ===" + +# Install aria2c for fast parallel downloads +echo "Installing aria2c..." +apk add --no-cache aria2 + +# Download files if not present +echo "" +echo "=== Downloading files ===" +for url in $DATA_URLS; do + filename=$(basename "$url") + if [ -f "/data/$filename" ]; then + echo "SKIP: $filename already exists" + else + echo "Downloading $filename..." + aria2c -x 16 -s 16 --file-allocation=none -d /data -o "$filename" "$url" + fi +done + +# Copy to volume if empty +echo "" +echo "=== Copying to volume ===" +if [ -n "$(ls -A /volume 2>/dev/null)" ]; then + echo "SKIP: volume is not empty" +else + echo "Copying ZIM files to volume..." + cp /data/*.zim /volume/ 2>/dev/null || echo "No ZIM files to copy" +fi + +echo "" +echo "=== Setup complete ===" diff --git a/dev/environments/docker/sites/wikipedia/supervisord.conf b/dev/environments/docker/sites/wikipedia/supervisord.conf new file mode 100644 index 0000000..d23d074 --- /dev/null +++ b/dev/environments/docker/sites/wikipedia/supervisord.conf @@ -0,0 +1,32 @@ +[supervisord] +nodaemon=true +logfile=/tmp/supervisord.log +pidfile=/run/supervisord.pid +user=root + +[unix_http_server] +file=/run/supervisord.sock + +[supervisorctl] +serverurl=unix:///run/supervisord.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[program:kiwix-serve] +command=/usr/local/bin/kiwix-serve --port=%(ENV_WA_ENV_CTRL_KIWIX_PORT)s --customIndex=/srv/index.html %(ENV_WA_ENV_CTRL_ZIM_FILE)s +autostart=true +autorestart=true +stdout_logfile=/tmp/supervisord.log +stdout_logfile_maxbytes=10MB +stderr_logfile=/tmp/supervisord.log +stderr_logfile_maxbytes=10MB + +[program:env-ctrl] +command=/usr/local/bin/env-ctrl serve --port %(ENV_WA_ENV_CTRL_PORT)s +autostart=true +autorestart=true +stdout_logfile=/tmp/env-ctrl.log +stdout_logfile_maxbytes=10MB +stderr_logfile=/tmp/env-ctrl.log +stderr_logfile_maxbytes=10MB diff --git a/dev/environments/docker/tasks.py b/dev/environments/docker/tasks.py new file mode 100644 index 0000000..e85e4ba --- /dev/null +++ b/dev/environments/docker/tasks.py @@ -0,0 +1,507 @@ +"""Docker container and image management tasks. + +Provides envs.docker.* namespace for container lifecycle, image management, +and optimization operations. +""" + +from __future__ import annotations + +import importlib +from pathlib import Path +from typing import TYPE_CHECKING + +from invoke import Collection, task + +if TYPE_CHECKING: + from invoke.context import Context + +from dev.environments.docker.utils import SEMVER_PATTERN, containers, downloads, sites +from dev.environments.docker.utils.create_base_img import create_base_img +from dev.utils import logging_utils +from dev.utils.network_utils import find_free_port +from dev.utils.path_utils import get_repo_root +from webarena_verified.environments.env_ctrl_client import EnvCtrlDockerClient + +# ============================================================================= +# Task definitions +# ============================================================================= + + +@task +@logging_utils.with_banner() +def start( + ctx: Context, + site: str, + port: int | None = None, + hostname: str = "localhost", + image: str | None = None, + idle: bool = False, + original: bool = False, +) -> None: + """Start container for a site. + + Args: + site: Site name (e.g., shopping-admin, wikipedia). + port: Port to expose (default: finds a free port). + hostname: Hostname for base URL (default: localhost). + image: Image name to use (default: from settings). + idle: Start with tail -f /dev/null for exploration. + original: Use the original unoptimized image. + """ + settings = sites._get_site_settings(site) + docker_image = settings.original_docker_img if original else settings.docker_img + effective_image = image or docker_image + effective_port = port if port is not None else find_free_port() + container_name = sites._get_container_name(site) + + containers._container_remove(ctx, container_name) + containers._docker_run(ctx, settings, effective_image, container_name, effective_port, idle=idle) + + if idle: + logging_utils.print_info("Container started in idle mode (tail -f /dev/null).") + else: + containers._wait_and_configure(container_name, effective_port, hostname=hostname) + + logging_utils.print_success("Container started!", URL=f"http://{hostname}:{effective_port}") + logging_utils.print_list( + [ + "Useful commands:", + f"inv envs.docker.stop --site={site} - Stop and remove container", + ] + ) + + +@task +@logging_utils.with_banner() +def stop(ctx: Context, site: str) -> None: + """Stop and remove container for a site. + + Args: + site: Site name (e.g., shopping-admin). + """ + container = sites._get_container_name(site) + + if not containers._container_exists(ctx, container): + logging_utils.print_info(f"Container '{container}' does not exist.") + return + + with logging_utils.StepContext.create(f"docker rm -f {container}", desc="Removing container"): + ctx.run(f"docker rm -f {container}", hide=True) + + logging_utils.print_success("Container removed") + + +@task +@logging_utils.with_banner() +def check(ctx: Context, site: str) -> None: + """Check if container is healthy. + + Args: + site: Site name (e.g., shopping-admin). + """ + container = sites._get_container_name(site) + + if not containers._container_running(ctx, container): + logging_utils.print_error(f"Container '{container}' is not running") + raise SystemExit(1) + + logging_utils.print_info(f"Container '{container}' is running") + + client = EnvCtrlDockerClient.create(container) + result = client.status() + + if not result.success: + logging_utils.print_error(f"Failed to get health status: {result.message}") + raise SystemExit(1) + + details = result.details + services = details.get("services", {}) + + if not services: + logging_utils.print_warning("No service status available") + return + + health_data = {} + all_healthy = True + for svc_name, state in services.items(): + is_healthy = str(state).upper() in ("RUNNING", "HEALTHY") + health_data[svc_name] = f"{'โœ“' if is_healthy else 'โœ—'} {state}" + if not is_healthy: + all_healthy = False + + logging_utils.print_table(health_data) + + if not all_healthy: + logging_utils.print_failure("Some services are not healthy") + raise SystemExit(1) + + logging_utils.print_success("All checks passed!") + + +@task +@logging_utils.with_banner() +def pull( + ctx: Context, + site: str, + tag: str = "latest", + force: bool = False, + original: bool = False, + output_dir: str | None = None, + dry_run: bool = False, +) -> None: + """Pull image from Docker Hub or download original from URL. + + Use --site=all to download all original images without loading them. + + Args: + site: Site name (e.g., shopping-admin), or 'all' to pull all original images. + tag: Image tag to pull (default: latest). + force: Always pull/download even if exists locally. + original: Download original image from URL. + output_dir: Directory to save downloaded tar file. + dry_run: Show what would be downloaded without downloading (only with --site=all). + """ + # Handle --site=all + if site.lower() == "all": + if dry_run: + logging_utils.print_info("Dry run - showing what would be downloaded:\n") + + for site_name in sites._list_sites(): + settings = sites._get_site_settings(site_name) + + if dry_run: + info = downloads._get_download_info(settings, output_dir) + if not info["url"]: + logging_utils.print_info(f" [SKIP - no URL] {site_name}") + continue + status = "[SKIP - exists]" if info["exists"] and not force else "[DOWNLOAD]" + if force and info["exists"]: + status = "[DOWNLOAD - force]" + logging_utils.print_info(f" {status} {site_name}") + logging_utils.print_info(f" URL: {info['url']}") + logging_utils.print_info(f" To: {info['output_path']}\n") + else: + logging_utils.print_info(f"Downloading {site_name}...") + downloads._pull_original(ctx, settings, force=force, output_dir=output_dir, load=False) + return + + # Single site + settings = sites._get_site_settings(site) + + if original: + downloads._pull_original(ctx, settings, force=force, output_dir=output_dir, load=True) + else: + downloads._pull_slim(ctx, settings, force=force) + + +@task(name="data-download") +@logging_utils.with_banner() +def data_download( + ctx: Context, + site: str | None = None, + force: bool = False, + output_dir: str | None = None, + dry_run: bool = False, +) -> None: + """Download data files (non-Docker artifacts) for sites. + + Downloads files specified in data_urls settings (e.g., Wikipedia ZIM files). + + Args: + site: Site name (e.g., wikipedia), or omit to download all. + force: Re-download even if file exists locally. + output_dir: Directory to save files (default: from settings or downloads/). + dry_run: Show what would be downloaded without downloading. + """ + # Determine which sites to process + if site: + site_list = [site] + else: + site_list = sites._list_sites() + + if dry_run: + logging_utils.print_info("Dry run - showing what would be downloaded:\n") + + any_downloads = False + for site_name in site_list: + settings = sites._get_site_settings(site_name) + infos = downloads._get_data_download_info(settings, output_dir) + + if not infos: + continue + + any_downloads = True + for info in infos: + if dry_run: + status = "[SKIP - exists]" if info["exists"] and not force else "[DOWNLOAD]" + if force and info["exists"]: + status = "[DOWNLOAD - force]" + logging_utils.print_info(f" {status} {site_name}") + logging_utils.print_info(f" URL: {info['url']}") + logging_utils.print_info(f" To: {info['output_path']}\n") + else: + logging_utils.print_info(f"Downloading data for {site_name}...") + downloads._download_data(ctx, settings, force=force, output_dir=output_dir) + break # _download_data handles all URLs for the site + + if not any_downloads: + logging_utils.print_info("No sites have data_urls configured.") + + +@task +@logging_utils.with_banner() +def build(ctx: Context, site: str, tag: str | None = None, no_cache: bool = False) -> None: + """Build Docker image for a site from its Dockerfile. + + Args: + site: Site name (e.g., shopping-admin). + tag: Image tag (default: latest). + no_cache: Build without using cache. + """ + settings = sites._get_site_settings(site) + + if not settings.dockerfile: + logging_utils.print_error(f"No dockerfile configured for site '{site}' in settings") + raise SystemExit(1) + + repo_root = get_repo_root() + dockerfile = repo_root / settings.dockerfile + + if not dockerfile.exists(): + logging_utils.print_error(f"Dockerfile not found: {dockerfile}") + raise SystemExit(1) + + # Build image name + image_name = settings.docker_img + if tag: + image_name = f"{image_name}:{tag}" + + cache_flag = "--no-cache" if no_cache else "" + cmd = f"docker build {cache_flag} -t {image_name} -f {dockerfile} {repo_root}".strip() + + logging_utils.print_info(f"Building {image_name} from {dockerfile}") + ctx.run(cmd) + + +@task +@logging_utils.with_banner() +def publish( + ctx: Context, + site: str, + tag: str, + image: str | None = None, + name: str | None = None, + yes: bool = False, +) -> None: + """Publish optimized image to Docker Hub. + + Tag must be a valid semver (e.g., 1.0.0, 2.1.3). + Pushes both the semver tag and latest. + + Args: + site: Site name (e.g., shopping-admin). + tag: Semver tag (required, e.g., 1.0.0). + image: Local image name to publish (default: {docker_img}:dev-latest). + name: Image name in repository (default: from settings). + yes: Skip confirmation prompt. + """ + if not SEMVER_PATTERN.match(tag): + raise SystemExit(f"Invalid tag '{tag}'. Must be semver format (e.g., 1.0.0)") + + settings = sites._get_site_settings(site) + + # Local image defaults to the provided tag + local_image = image or f"{settings.docker_img}:{tag}" + + # Remote name defaults to docker_img from settings + remote_name = name or settings.docker_img + + # Push both semver tag and latest + tags_to_push = [tag, "latest"] + remote_images = [f"{remote_name}:{t}" for t in tags_to_push] + + logging_utils.print_info(f"Local image: {local_image}") + for ri in remote_images: + logging_utils.print_info(f"Remote image: {ri}") + + # Tag images for remote + for ri in remote_images: + ctx.run(f"docker tag {local_image} {ri}", hide=True) + + # Use shared publish utility + if containers.publish_images(ctx, remote_images, yes=yes): + logging_utils.print_success("PUBLISH COMPLETE") + + +@task(name="test") +@logging_utils.with_banner() +def docker_test( + ctx: Context, + site: str, + tag: str = "latest", + headed: bool = False, + remote_img: bool = False, + data_dir: str | None = None, +) -> None: + """Run integration tests against a Docker image. + + Stops any existing container, starts from the specified image, and runs + pytest with the site's marker (e.g., -m integration_docker_shopping_admin). + + Args: + site: Site name (e.g., shopping-admin). + tag: Image tag (default: latest). + headed: Run browser tests in headed mode. + remote_img: Delete local image first to force pull from registry. + data_dir: Directory containing data to mount (for map CI tests). + """ + settings = sites._get_site_settings(site) + + # Build image name with tag + image_name = f"{settings.docker_img}:{tag}" + container_name = sites._get_container_name(site) + + # Marker uses underscores (same as site) + marker = f"integration_docker_{site}" + + # Stop and remove any existing container + containers._container_remove(ctx, container_name, quiet=True) + + # Delete local image and pull from registry + if remote_img: + logging_utils.print_info(f"Removing local image {image_name}...") + ctx.run(f"docker rmi {image_name}", warn=True) + logging_utils.print_info(f"Pulling {image_name} from registry...") + ctx.run(f"docker pull {image_name}") + + port = find_free_port() + try: + # Start container + logging_utils.print_info(f"Starting container {container_name} from {image_name}") + containers._container_remove(ctx, container_name, quiet=True) + + # GitLab: skip reconfigure during init to avoid 502 errors after startup + # Also mount local env-ctrl code to test latest changes before image rebuild + extra_args = "" + if site == "gitlab": + env_ctrl_path = get_repo_root() / "packages" / "environment_control" / "environment_control" + extra_args = f"-e WA_ENV_CTRL_SKIP_RECONFIGURE=true -v {env_ctrl_path}:/usr/local/environment_control" + + # Mount data directory if provided (for CI tests with pre-generated data) + if data_dir: + data_path = Path(data_dir).resolve() + logging_utils.print_info(f"Mounting data from: {data_path}") + + if site == "map": + extra_args += f" -v {data_path}/database:/data/database" + extra_args += f" -v {data_path}/routing/car:/data/routing/car" + extra_args += f" -v {data_path}/routing/bike:/data/routing/bike" + extra_args += f" -v {data_path}/routing/foot:/data/routing/foot" + extra_args += f" -v {data_path}/nominatim/postgres:/data/nominatim/postgres" + extra_args += f" -v {data_path}/website/postgres:/var/lib/postgresql/14/main" + + elif site == "wikipedia": + # Find ZIM file in data directory + zim_files = list(data_path.glob("*.zim")) + if not zim_files: + raise ValueError(f"No .zim file found in {data_path}") + zim_file = zim_files[0] + zim_name = zim_file.name + extra_args += f" -v {zim_file}:/data/{zim_name}" + extra_args += f" -e WA_ENV_CTRL_ZIM_FILE=/data/{zim_name}" + + # Skip named volumes if we're mounting data directly to avoid duplicate mount points + skip_volumes = bool(data_dir) + env_ctrl_port = containers._docker_run( + ctx, settings, image_name, container_name, port, extra_args=extra_args, skip_volumes=skip_volumes + ) + containers._wait_and_configure(container_name, port) + + # Run tests + logging_utils.print_info(f"Running tests with marker: {marker}") + headed_flag = " --headed" if headed else "" + site_url = f"http://localhost:{port}" + env_ctrl_url = f"http://localhost:{env_ctrl_port}" + pytest_args = f"--{site}_url={site_url} --{site}_env_ctrl_url={env_ctrl_url}" + if site == "map": + pytest_args += f" --map_tile_url={site_url}" + ctx.run(f"uv run pytest tests/integration/environments/ -m {marker} -v {pytest_args}{headed_flag}") + finally: + # Cleanup: stop and remove container + logging_utils.print_info(f"Cleaning up container {container_name}") + containers._container_remove(ctx, container_name, quiet=True) + + +@task(name="create-base-img") +@logging_utils.with_banner() +def create_base_img_task( + ctx: Context, + site: str, + timeout: int = 300, + dry_run: bool = False, + no_squash: bool = False, +) -> None: + """Create a base image from the original Docker image. + + Pipeline: + 1. Start container with mounts and env vars + 2. Run setup scripts (bootstrap, services, patches, cleanup) + 3. Squash to output image (unless --no-squash) + + Args: + site: Site name (e.g., shopping-admin). + timeout: Timeout for service startup in seconds (default: 300). + dry_run: Run scripts with BUILD_DRY_RUN=true but skip squash. + no_squash: Skip squashing - just commit the image directly. + """ + create_base_img(ctx, site, timeout=timeout, dry_run=dry_run, no_squash=no_squash) + + +@task(name="setup") +@logging_utils.with_banner() +def setup_task( + ctx: Context, + site: str, + data_dir: str, + dry_run: bool = False, + keep_downloads: bool = False, +) -> None: + """Set up Docker volumes and data for a site. + + Downloads required data files (if not present in data-dir) and loads them + into Docker volumes. Site-specific setup logic is in sites//scripts/docker_setup.py. + + Args: + site: Site name (e.g., wikipedia, map). + data_dir: Directory to store/find data files. + dry_run: Show what would be done without doing it. + keep_downloads: Don't delete downloaded files after loading. + """ + # Validate site exists + sites._get_site_settings(site) + + # Try to import site-specific setup module + try: + module = importlib.import_module(f"dev.environments.docker.sites.{site}.scripts.docker_setup") + except ModuleNotFoundError: + logging_utils.print_warning(f"Site '{site}' has no setup configured") + return # exit 0, not error + + module.setup(ctx, Path(data_dir), dry_run=dry_run, keep_downloads=keep_downloads) + + +# ============================================================================= +# Collection +# ============================================================================= + +ns = Collection() +ns.add_task(start) # ty: ignore[invalid-argument-type] +ns.add_task(stop) # ty: ignore[invalid-argument-type] +ns.add_task(check) # ty: ignore[invalid-argument-type] +ns.add_task(pull) # ty: ignore[invalid-argument-type] +ns.add_task(data_download) +ns.add_task(build) # ty: ignore[invalid-argument-type] +ns.add_task(publish) # ty: ignore[invalid-argument-type] +ns.add_task(docker_test) +ns.add_task(create_base_img_task) +ns.add_task(setup_task) diff --git a/dev/environments/docker/utils/__init__.py b/dev/environments/docker/utils/__init__.py new file mode 100644 index 0000000..46d9434 --- /dev/null +++ b/dev/environments/docker/utils/__init__.py @@ -0,0 +1,10 @@ +"""Docker utilities.""" + +import re + +from dev.environments.docker.utils import docker_setup_helpers + +# Semver pattern: MAJOR.MINOR.PATCH (e.g., 1.0.0, 2.1.3) +SEMVER_PATTERN = re.compile(r"^\d+\.\d+\.\d+$") + +__all__ = ["SEMVER_PATTERN", "docker_setup_helpers"] diff --git a/dev/environments/docker/utils/containers.py b/dev/environments/docker/utils/containers.py new file mode 100644 index 0000000..213854f --- /dev/null +++ b/dev/environments/docker/utils/containers.py @@ -0,0 +1,174 @@ +"""Container operation helpers for Docker tasks.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from dev.environments.docker.utils.dockerfile import get_container_port +from dev.environments.settings import get_settings +from dev.utils import logging_utils +from dev.utils.path_utils import get_repo_root +from webarena_verified.environments.env_ctrl_client import EnvCtrlDockerClient + +if TYPE_CHECKING: + from invoke.context import Context + + from dev.environments.settings import BaseSiteSettings + + +def _container_exists(ctx: Context, name: str) -> bool: + """Check if container exists.""" + result = ctx.run(f"docker ps -a --filter name=^{name}$ --format '{{{{.Names}}}}'", hide=True, warn=True) + return bool(result and result.stdout.strip() == name) + + +def _container_running(ctx: Context, name: str) -> bool: + """Check if container is running.""" + result = ctx.run(f"docker ps --filter name=^{name}$ --format '{{{{.Names}}}}'", hide=True, warn=True) + return bool(result and result.stdout.strip() == name) + + +def _container_remove(ctx: Context, name: str, quiet: bool = False) -> None: + """Remove container if it exists.""" + if _container_exists(ctx, name): + if not quiet: + logging_utils.print_info(f"Removing existing container {name}...") + ctx.run(f"docker rm -f {name}", hide=True) + + +def _docker_run( + ctx: Context, + settings: BaseSiteSettings, + image: str, + name: str, + port: int, + idle: bool = False, + extra_args: str = "", + env_ctrl_port: int | None = None, + skip_volumes: bool = False, +) -> int: + """Run a docker container with standard options. + + Args: + ctx: Invoke context. + settings: Site settings. + image: Docker image to run. + name: Container name. + port: Host port for the web service. + idle: If True, run container in idle mode (tail -f /dev/null). + extra_args: Additional docker run arguments. + env_ctrl_port: Host port for env-ctrl. If None, lets Docker assign one. + skip_volumes: If True, skip mounting named volumes from settings (use when + mounting data directly via extra_args to avoid duplicate mount points). + + Returns: + The host port assigned for env-ctrl. + """ + container_port = get_container_port(settings.dockerfile) if settings.dockerfile else 80 + env_ctrl_container_port = get_settings().env_ctrl_container_port + cmd = "docker run -d" + + cmd += f" --name {name} -p {port}:{container_port}" + + # Map env-ctrl port (let Docker assign if not specified to avoid race conditions) + if env_ctrl_port is not None: + cmd += f" -p {env_ctrl_port}:{env_ctrl_container_port}" + else: + cmd += f" -p 0:{env_ctrl_container_port}" + + # Data directory mounting + if settings.data_dir: + data_path = Path(settings.data_dir) + if not data_path.is_absolute(): + data_path = get_repo_root() / data_path + cmd += f" --volume={data_path}:/data" + + # Named Docker volumes (skip if mounting data directly) + if not skip_volumes: + for vol_name, container_path in settings.volumes.items(): + cmd += f" -v {vol_name}:{container_path}" + + if extra_args: + cmd += f" {extra_args}" + cmd += f" {image}" + if idle: + cmd += " tail -f /dev/null" + + desc = "Starting container (idle mode)" if idle else "Starting container" + with logging_utils.StepContext.create(cmd, desc=desc): + ctx.run(cmd, hide=True) + + # Get the assigned env-ctrl port (query Docker for dynamically assigned ports) + if env_ctrl_port is not None: + host_env_ctrl_port = env_ctrl_port + else: + result = ctx.run(f"docker port {name} {env_ctrl_container_port}", hide=True) + # Output format: "0.0.0.0:12345" or ":::12345" + port_mapping = result.stdout.strip().split(":")[-1] + host_env_ctrl_port = int(port_mapping) + + logging_utils.print_info(f"env-ctrl port: {host_env_ctrl_port}") + return host_env_ctrl_port + + +def publish_images(ctx: Context, images: list[str], yes: bool = False) -> bool: + """Push images to Docker registry with confirmation prompt. + + Args: + ctx: Invoke context. + images: List of image names with tags to push. + yes: Skip confirmation prompt if True. + + Returns: + True if images were published, False if skipped/aborted. + """ + if not images: + logging_utils.print_info("No images to publish.") + return False + + # Show what will be pushed + logging_utils.print_info("Images to push:") + for image in images: + logging_utils.print_info(f" {image}") + + # Confirmation prompt + if not yes: + try: + response = input("\nPush to registry? [y/N]: ").strip().lower() + except EOFError: + response = "n" + + if response not in ("y", "yes"): + logging_utils.print_info("Skipping publish.") + return False + + # Push each image + for image in images: + with logging_utils.StepContext.create(f"docker push {image}", desc=f"Pushing {image}"): + ctx.run(f"docker push {image}", pty=True) + + logging_utils.print_success("Published!", Images=", ".join(images)) + return True + + +def _wait_and_configure(container_name: str, port: int, timeout: int = 120, hostname: str = "localhost") -> None: + """Wait for services and configure the container.""" + # Client timeout should be longer than wait timeout to allow for the full wait period + client = EnvCtrlDockerClient.create(container_name, timeout=timeout + 60) + + with logging_utils.StepContext.create( + f"Waiting for services (timeout: {timeout}s)...", desc="Health check" + ) as step: + result = client.start(wait=True, timeout=timeout) + if not result.success: + step.mark_failed("Services did not become ready") + raise RuntimeError(f"Services failed: {result.message}") + + # Configure base URL + base_url = f"http://{hostname}:{port}/" + init_result = client.init(base_url=base_url) + if init_result.success: + logging_utils.print_info("Configuration complete") + else: + logging_utils.print_warning(f"Configuration had issues: {init_result.message}") diff --git a/dev/environments/docker/utils/create_base_img.py b/dev/environments/docker/utils/create_base_img.py new file mode 100644 index 0000000..ca76201 --- /dev/null +++ b/dev/environments/docker/utils/create_base_img.py @@ -0,0 +1,198 @@ +"""Create base Docker image from original image.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from dev.environments.docker.utils.containers import publish_images +from dev.environments.docker.utils.dockerfile import get_container_port +from dev.environments.docker.utils.sites import _get_container_name, _get_site_settings +from dev.environments.settings import get_settings +from dev.utils.logging_utils import StepContext, print_info, print_success, print_warning +from dev.utils.network_utils import find_free_port +from dev.utils.path_utils import get_repo_root + +if TYPE_CHECKING: + from invoke.context import Context + + from dev.environments.settings import BaseSiteSettings + + +def create_base_img( + ctx: Context, + site: str, + timeout: int = 300, + dry_run: bool = False, + no_squash: bool = False, +) -> None: + """Create a base image from the original Docker image. + + Pipeline: + 1. Start container with mounts and env vars + 2. Run setup scripts (bootstrap, services, patches, cleanup) + 3. Squash to output image (unless no_squash) + + Args: + ctx: Invoke context. + site: Site name (e.g., 'shopping_admin'). + timeout: Timeout for service startup in seconds. + dry_run: If True, run scripts with BUILD_DRY_RUN=true but skip squash. + no_squash: If True, commit image directly without squashing. + """ + settings = _get_site_settings(site) + + if not settings.base_docker_img: + raise ValueError(f"Site '{site}' does not have base_docker_img configured") + + port = find_free_port() + env_ctrl_port = find_free_port() + print_info(f"Using ports: web={port}, env-ctrl={env_ctrl_port}") + + container = f"{_get_container_name(site)}_build" + output_image = f"{settings.base_docker_img}:latest" + site_dir = (Path(__file__).parent.parent / "sites" / site).resolve() + env_ctrl_pkg = get_repo_root() / get_settings().env_ctrl_package_path + + try: + _start_container(ctx, container, settings, site_dir, env_ctrl_pkg, port, env_ctrl_port, timeout, dry_run) + + # Always run scripts - they handle dry_run internally via BUILD_DRY_RUN env var + _run_setup_scripts(ctx, container, site_dir / "scripts") + + if dry_run: + print_warning("DRY RUN - squash skipped.") + elif no_squash: + _commit(ctx, container, output_image) + print_success("Image created (not squashed)!", Image=output_image) + ctx.run(f"docker images | grep -E '({output_image.split(':')[0]}|REPOSITORY)'") + else: + _squash(ctx, container, output_image) + print_success("Image created!", Image=output_image) + ctx.run(f"docker images | grep -E '({output_image.split(':')[0]}|REPOSITORY)'") + + # Prompt to publish + publish_images(ctx, [output_image]) + + except KeyboardInterrupt: + print_warning("Interrupted by user (Ctrl-C)") + raise SystemExit(130) from None + finally: + _cleanup_or_commit_wip(ctx, container, settings) + + +def _start_container( + ctx: Context, + container: str, + settings: BaseSiteSettings, + site_dir: Path, + env_ctrl_pkg: Path, + port: int, + env_ctrl_port: int, + timeout: int, + dry_run: bool = False, +) -> None: + """Start build container with mounts and env vars.""" + print_info(f"Starting container from {settings.original_docker_img}...") + ctx.run(f"docker rm -f {container}", warn=True, hide=True) + + base_url = f"http://localhost:{port}/" + mounts = [ + f"-v {site_dir}:/build-site:ro", + f"-v {env_ctrl_pkg}:/build-env-ctrl:ro", + ] + env_vars = [ + f"-e BUILD_BASE_URL={base_url}", + f"-e BUILD_TIMEOUT={timeout}", + f"-e BUILD_DRY_RUN={str(dry_run).lower()}", + "-e BUILD_ENV_CTRL_SRC=/build-env-ctrl", + "-e WA_ENV_CTRL_ROOT=/opt", + f"-e WA_ENV_CTRL_TYPE={settings.site.value}", + ] + + container_port = get_container_port(settings.dockerfile) if settings.dockerfile else 80 + cmd = ( + f"docker run -d --name {container} " + f"-p {port}:{container_port} " + f"-p {env_ctrl_port}:{get_settings().env_ctrl_container_port} " + f"{' '.join(env_vars)} " + f"{' '.join(mounts)} " + f"{settings.original_docker_img} tail -f /dev/null" + ) + with StepContext.create(cmd, desc="Starting container"): + ctx.run(cmd, hide=True) + + +def _run_setup_scripts(ctx: Context, container: str, scripts_dir: Path) -> None: + """Execute build scripts in order.""" + if not scripts_dir.exists(): + print_info("No build scripts found") + return + + for script in sorted(scripts_dir.glob("*.sh")): + print_info(f"Running {script.name}...") + # Use plain=True to avoid Rich markup parsing (output may contain paths like [/etc/gitlab]) + with StepContext.create(f"docker exec {container} bash /build-site/scripts/{script.name}", plain=True): + ctx.run(f"docker exec {container} bash /build-site/scripts/{script.name}") + + +def _commit(ctx: Context, container: str, output: str) -> None: + """Commit container into image without squashing. + + Args: + ctx: Invoke context. + container: Container name to commit. + output: Output image name with tag. + """ + print_info("Committing image (no squash)...") + ctx.run(f"docker stop {container}", hide=True) + + with StepContext.create(f"docker commit {container} {output}", desc="Committing"): + ctx.run(f"docker commit {container} {output}", hide=True) + + +def _squash(ctx: Context, container: str, output: str, squash_timeout: int = 3600) -> None: + """Commit and squash container into slim image. + + Args: + ctx: Invoke context. + container: Container name to commit. + output: Output image name with tag. + squash_timeout: Timeout for docker-squash operations in seconds (default: 3600). + """ + print_info("Squashing image...") + ctx.run(f"docker stop {container}", hide=True) + + temp = f"{output.split(':')[0]}_temp:build" + with StepContext.create(f"docker commit {container} {temp}", desc="Committing"): + ctx.run(f"docker commit {container} {temp}", hide=True) + + with StepContext.create(f"docker-squash -t {output}", desc="Squashing"): + ctx.run(f"DOCKER_TIMEOUT={squash_timeout} uv run docker-squash -v -t {output} {temp}", hide=False) + + ctx.run(f"docker rmi {temp}", warn=True, hide=True) + + +def _cleanup_or_commit_wip(ctx: Context, container: str, settings: BaseSiteSettings) -> None: + """Ask user whether to clean up container or commit as WIP image. + + Args: + ctx: Invoke context. + container: Container name to clean up or commit. + settings: Site settings for WIP image naming. + """ + try: + response = input("\nClean up build container? [Y/n]: ").strip().lower() + except EOFError: + response = "y" + + if response in ("", "y", "yes"): + print_info("Cleaning up build container...") + ctx.run(f"docker rm -f {container}", warn=True, hide=True) + else: + wip_image = f"{settings.base_docker_img}:wip" + print_info(f"Committing container as WIP image: {wip_image}") + ctx.run(f"docker stop {container}", warn=True, hide=True) + ctx.run(f"docker commit {container} {wip_image}", hide=True) + print_success("WIP image created!", Image=wip_image, Container=container) + print_info(f"Container '{container}' kept for debugging. Remove with: docker rm -f {container}") diff --git a/dev/environments/docker/utils/docker_setup_helpers.py b/dev/environments/docker/utils/docker_setup_helpers.py new file mode 100644 index 0000000..c967b38 --- /dev/null +++ b/dev/environments/docker/utils/docker_setup_helpers.py @@ -0,0 +1,45 @@ +"""Shared utilities for Docker volume setup. + +This module provides common functions used by site-specific setup modules +for Docker volume operations. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from dev.utils import logging_utils + +if TYPE_CHECKING: + from invoke.context import Context + + +def volume_exists(ctx: Context, name: str) -> bool: + """Check if Docker volume exists.""" + result = ctx.run(f"docker volume ls -q -f name=^{name}$", hide=True, warn=True) + return bool(result and result.stdout.strip()) + + +def volume_is_empty(ctx: Context, name: str) -> bool: + """Check if Docker volume is empty (has no files).""" + result = ctx.run( + f"docker run --rm -v {name}:/vol:ro alpine sh -c 'ls -A /vol | head -1'", + hide=True, + warn=True, + ) + return not (result and result.stdout.strip()) + + +def create_volume(ctx: Context, name: str) -> None: + """Create Docker volume if not exists.""" + if volume_exists(ctx, name): + logging_utils.print_info(f"Volume {name} already exists") + return + + with logging_utils.StepContext.create(f"docker volume create {name}", desc="Creating volume"): + ctx.run(f"docker volume create {name}", hide=True) + + +def get_volume_name(prefix: str, site: str, suffix: str) -> str: + """Get full volume name: {prefix}-{site}-{suffix}.""" + return f"{prefix}-{site}-{suffix}" diff --git a/dev/environments/docker/utils/dockerfile.py b/dev/environments/docker/utils/dockerfile.py new file mode 100644 index 0000000..508abfc --- /dev/null +++ b/dev/environments/docker/utils/dockerfile.py @@ -0,0 +1,25 @@ +"""Dockerfile parsing utilities.""" + +import re +from pathlib import Path + + +def get_container_port(dockerfile_path: str | Path) -> int: + """Read WA_CONTAINER_PORT from a Dockerfile. + + Parses the Dockerfile for an ENV instruction setting WA_CONTAINER_PORT. + + Args: + dockerfile_path: Path to the Dockerfile. + + Returns: + The container port value, or 80 if not defined. + """ + path = Path(dockerfile_path) + if not path.exists(): + return 80 + + content = path.read_text() + # Match: ENV WA_CONTAINER_PORT= or ENV WA_CONTAINER_PORT + match = re.search(r"ENV\s+WA_CONTAINER_PORT[=\s]+(\d+)", content) + return int(match.group(1)) if match else 80 diff --git a/dev/environments/docker/utils/downloads.py b/dev/environments/docker/utils/downloads.py new file mode 100644 index 0000000..71f1388 --- /dev/null +++ b/dev/environments/docker/utils/downloads.py @@ -0,0 +1,186 @@ +"""Download helpers for Docker tasks.""" + +from __future__ import annotations + +import shutil +from pathlib import Path +from typing import TYPE_CHECKING + +from dev.utils import logging_utils +from dev.utils.path_utils import get_repo_root + +if TYPE_CHECKING: + from invoke.context import Context + + from dev.environments.settings import BaseSiteSettings + + +def download_file_in_container(ctx: Context, url: str, output_dir: Path) -> Path: + """Download file using aria2c in alpine container. + + Returns path to downloaded file. + """ + filename = url.rsplit("/", 1)[-1] + output_path = output_dir / filename + + # Use alpine container with aria2c for consistent, fast downloads + cmd = ( + f"docker run --rm -v {output_dir}:/data alpine sh -c " + f"'apk add --no-cache aria2 && aria2c -x 16 -s 16 --file-allocation=none -d /data -o {filename} \"{url}\"'" + ) + + with logging_utils.StepContext.create(f"Downloading {filename}", desc="Downloading"): + ctx.run(cmd, pty=True) + + return output_path + + +def _get_download_info(settings: BaseSiteSettings, output_dir: str | None = None) -> dict: + """Get download information without downloading.""" + url = settings.original_docker_url + if not url: + return {"url": None, "filename": None, "output_path": None, "exists": False} + filename = url.rsplit("/", 1)[-1] + + if output_dir: + effective_dir = Path(output_dir) + else: + effective_dir = get_repo_root() / "downloads" + + output_path = effective_dir / filename + return {"url": url, "filename": filename, "output_path": output_path, "exists": output_path.exists()} + + +def _get_data_download_info(settings: BaseSiteSettings, output_dir: str | None = None) -> list[dict]: + """Get download information for data URLs without downloading. + + Returns a list of dicts, one per data URL. + """ + if not settings.data_urls: + return [] + + if output_dir: + effective_dir = Path(output_dir) + elif settings.data_dir: + effective_dir = get_repo_root() / settings.data_dir + else: + effective_dir = get_repo_root() / "downloads" + + results = [] + for url in settings.data_urls: + filename = url.rsplit("/", 1)[-1] + output_path = effective_dir / filename + results.append({"url": url, "filename": filename, "output_path": output_path, "exists": output_path.exists()}) + return results + + +def _download_data( + ctx: Context, + settings: BaseSiteSettings, + force: bool = False, + output_dir: str | None = None, +) -> None: + """Download data files for a site.""" + infos = _get_data_download_info(settings, output_dir) + if not infos: + logging_utils.print_info(f"No data_urls configured for {settings.site.value}") + return + + for info in infos: + url, filename, output_path = info["url"], info["filename"], Path(info["output_path"]) + + if not force and output_path.exists(): + logging_utils.print_info(f"File {output_path} already exists. Use --force to re-download.") + continue + + output_path.parent.mkdir(parents=True, exist_ok=True) + _download_file(ctx, url, output_path, filename) + logging_utils.print_success("Download complete!", Path=str(output_path)) + + +def _download_file(ctx: Context, url: str, output_path: Path, filename: str) -> None: + """Download file using best available tool.""" + if shutil.which("aria2c"): + cmd = f"aria2c -x 16 -s 16 --file-allocation=none -d {output_path.parent} -o {filename} {url}" + tool = "aria2c" + elif shutil.which("wget"): + cmd = f"wget -O {output_path} {url}" + tool = "wget" + elif shutil.which("curl"): + cmd = f"curl -L -o {output_path} {url}" + tool = "curl" + else: + logging_utils.print_error("No download tool available. Install aria2c, wget, or curl.") + raise SystemExit(1) + + with logging_utils.StepContext.create(cmd, desc=f"Downloading with {tool}"): + ctx.run(cmd, pty=True) + + +def _load_docker_image(ctx: Context, tar_path: Path, image_name: str) -> None: + """Load a tar file into Docker.""" + if shutil.which("pv"): + cmd = f"pv {tar_path} | docker load" + with logging_utils.StepContext.create(cmd, desc="Loading Docker image"): + ctx.run(cmd, pty=True) + else: + cmd = f"docker load -i {tar_path}" + with logging_utils.StepContext.create(cmd, desc="Loading Docker image"): + ctx.run(cmd, hide=False) + + logging_utils.print_success("Image loaded!", Image=image_name) + + +def _pull_slim(ctx: Context, settings: BaseSiteSettings, force: bool = False) -> None: + """Pull slim image from Docker Hub.""" + full_image = settings.docker_img + + if not force: + result = ctx.run(f"docker images -q {full_image}", hide=True, warn=True) + if result and result.stdout.strip(): + logging_utils.print_info(f"Image {full_image} already exists. Use --force to pull anyway.") + return + + with logging_utils.StepContext.create(f"docker pull {full_image}", desc="Pulling from Docker Hub"): + ctx.run(f"docker pull {full_image}", pty=True) + + logging_utils.print_success("Pull complete!", Image=full_image) + + +def _pull_original( + ctx: Context, + settings: BaseSiteSettings, + force: bool = False, + output_dir: str | None = None, + load: bool = True, +) -> None: + """Download original image from URL.""" + info = _get_download_info(settings, output_dir) + if not info["url"]: + logging_utils.print_warning(f"No original_docker_url configured for {settings.site.value}") + return + + url, filename, output_path = info["url"], info["filename"], Path(info["output_path"]) + is_docker_image = filename.endswith(".tar") + + if not force: + if load and is_docker_image: + result = ctx.run(f"docker images -q {settings.original_docker_img}", hide=True, warn=True) + if result and result.stdout.strip(): + logging_utils.print_info("Image already exists. Use --force to download.") + return + elif output_path.exists(): + logging_utils.print_info(f"File {output_path} already exists. Use --force to download.") + return + + if force or not output_path.exists(): + output_path.parent.mkdir(parents=True, exist_ok=True) + _download_file(ctx, url, output_path, filename) + logging_utils.print_success("Download complete!", Path=str(output_path)) + else: + logging_utils.print_info(f"Using existing file: {output_path}") + + if load and is_docker_image: + _load_docker_image(ctx, output_path, settings.original_docker_img) + elif not is_docker_image: + logging_utils.print_info(f"Data file downloaded: {output_path}") diff --git a/dev/environments/docker/utils/sites.py b/dev/environments/docker/utils/sites.py new file mode 100644 index 0000000..dd664dc --- /dev/null +++ b/dev/environments/docker/utils/sites.py @@ -0,0 +1,43 @@ +"""Site-related helpers for Docker tasks.""" + +from __future__ import annotations + +from dev.environments.settings import BaseSiteSettings, get_settings +from webarena_verified.types.task import WebArenaSite + +# Sites that support Docker operations +_DOCKER_SITES: frozenset[WebArenaSite] = frozenset( + { + WebArenaSite.SHOPPING, + WebArenaSite.SHOPPING_ADMIN, + WebArenaSite.REDDIT, + WebArenaSite.GITLAB, + WebArenaSite.WIKIPEDIA, + WebArenaSite.MAP, + } +) + + +def _list_sites() -> list[str]: + """List all available site names.""" + return sorted(site.value for site in _DOCKER_SITES) + + +def _get_site_settings(site: str) -> BaseSiteSettings: + """Get settings for a site, validating it exists.""" + available = _list_sites() + if site not in available: + raise ValueError(f"Unknown site '{site}'. Available: {', '.join(available)}") + return getattr(get_settings(), site) + + +def _get_container_name(site: str) -> str: + """Get container name for a site. + + Args: + site: Site name (e.g., 'shopping_admin'). + + Returns: + Container name in format 'webarena-verified-'. + """ + return f"webarena-verified-{site}" diff --git a/dev/environments/settings.py b/dev/environments/settings.py new file mode 100644 index 0000000..c4c90c2 --- /dev/null +++ b/dev/environments/settings.py @@ -0,0 +1,189 @@ +"""Settings for Docker image management. + +All site-specific settings are defined here and accessed via the site registry. +Environment variables can override defaults using the WA_DEV_ prefix with nested delimiter. + +Example environment variables: + WA_DEV__SHOPPING_ADMIN__HOSTNAME=myhost.local +""" + +from functools import lru_cache + +from pydantic import BaseModel +from pydantic_settings import BaseSettings, SettingsConfigDict + +from webarena_verified.types.task import WebArenaSite + + +class BaseSiteSettings(BaseModel): + """Base class for site settings with common fields.""" + + site: WebArenaSite + """The WebArena site this settings belongs to.""" + + # Docker image settings + original_docker_img: str + """Original unoptimized Docker image name.""" + + original_docker_url: str | None = None + """URL to download the original Docker image (tar file).""" + + docker_img: str + """Optimized Docker image name (from Docker Hub).""" + + base_docker_img: str = "" + """Base Docker image name for create-base-img output. If empty, not supported.""" + + data_dir: str | None = None + """Optional data directory path for external data files. If None, uses downloads/ at repo root.""" + + data_urls: tuple[str, ...] = () + """URLs for data files that need to be downloaded (non-Docker artifacts like ZIM files).""" + + dockerfile: str | None = None + """Path to Dockerfile relative to repo root. Used by the build task.""" + + volumes: dict[str, str] = {} + """Docker volume mounts. Maps volume name to container path.""" + + +class ShoppingSettings(BaseSiteSettings): + """Settings for the Shopping site.""" + + site: WebArenaSite = WebArenaSite.SHOPPING + original_docker_img: str = "shopping_final_0712" + original_docker_url: str = "http://metis.lti.cs.cmu.edu/webarena-images/shopping_final_0712.tar" + docker_img: str = "am1n3e/webarena-verified-shopping" + base_docker_img: str = "am1n3e/webarena-verified-shopping-base" + dockerfile: str = "dev/environments/docker/sites/shopping/Dockerfile" + + +class ShoppingAdminSettings(BaseSiteSettings): + """Settings for the Shopping Admin site.""" + + site: WebArenaSite = WebArenaSite.SHOPPING_ADMIN + original_docker_img: str = "shopping_admin_final_0719" + original_docker_url: str = "http://metis.lti.cs.cmu.edu/webarena-images/shopping_admin_final_0719.tar" + docker_img: str = "am1n3e/webarena-verified-shopping_admin" + base_docker_img: str = "am1n3e/webarena-verified-shopping_admin-base" + dockerfile: str = "dev/environments/docker/sites/shopping_admin/Dockerfile" + + +class RedditSettings(BaseSiteSettings): + """Settings for the Reddit (Postmill) site.""" + + site: WebArenaSite = WebArenaSite.REDDIT + original_docker_img: str = "postmill-populated-exposed-withimg" + original_docker_url: str = "http://metis.lti.cs.cmu.edu/webarena-images/postmill-populated-exposed-withimg.tar" + docker_img: str = "am1n3e/webarena-verified-reddit" + base_docker_img: str = "am1n3e/webarena-verified-reddit-base" + dockerfile: str = "dev/environments/docker/sites/reddit/Dockerfile" + + +class GitlabSettings(BaseSiteSettings): + """Settings for the GitLab site.""" + + site: WebArenaSite = WebArenaSite.GITLAB + original_docker_img: str = "gitlab-populated-final-port8023" + original_docker_url: str = "http://metis.lti.cs.cmu.edu/webarena-images/gitlab-populated-final-port8023.tar" + docker_img: str = "am1n3e/webarena-verified-gitlab" + base_docker_img: str = "am1n3e/webarena-verified-gitlab-base" + dockerfile: str = "dev/environments/docker/sites/gitlab/Dockerfile" + + +class WikipediaSettings(BaseSiteSettings): + """Settings for the Wikipedia site.""" + + site: WebArenaSite = WebArenaSite.WIKIPEDIA + original_docker_img: str = "ghcr.io/kiwix/kiwix-serve:3.3.0" + original_docker_url: str | None = None + docker_img: str = "am1n3e/webarena-verified-wikipedia" + data_dir: str = "downloads" + data_urls: tuple[str, ...] = ("http://metis.lti.cs.cmu.edu/webarena-images/wikipedia_en_all_maxi_2022-05.zim",) + dockerfile: str = "dev/environments/docker/sites/wikipedia/Dockerfile" + + +class MapSettings(BaseSiteSettings): + """Settings for the OpenStreetMap site.""" + + site: WebArenaSite = WebArenaSite.MAP + original_docker_img: str = "am1n3e/openstreetmap-website-web:base" + original_docker_url: str | None = None + docker_img: str = "am1n3e/webarena-verified-map" + dockerfile: str = "dev/environments/docker/sites/map/Dockerfile" + data_urls: tuple[str, ...] = ( + "https://webarena-map-server-data.s3.amazonaws.com/osm_tile_server.tar", + "https://webarena-map-server-data.s3.amazonaws.com/nominatim_volumes.tar", + "https://webarena-map-server-data.s3.amazonaws.com/osrm_routing.tar", + ) + volumes: dict[str, str] = { + "webarena-verified-map-tile-db": "/data/database", + "webarena-verified-map-routing-car": "/data/routing/car", + "webarena-verified-map-routing-bike": "/data/routing/bike", + "webarena-verified-map-routing-foot": "/data/routing/foot", + "webarena-verified-map-nominatim-db": "/data/nominatim/postgres", + "webarena-verified-map-nominatim-flatnode": "/data/nominatim/flatnode", + "webarena-verified-map-website-db": "/var/lib/postgresql/14/main", + "webarena-verified-map-tiles": "/data/tiles", + "webarena-verified-map-style": "/data/style", + } + + +class DevSettings(BaseSettings): + """Development settings for Docker image management.""" + + model_config = SettingsConfigDict(env_prefix="WA_DEV__", env_nested_delimiter="__") + + env_ctrl_container_port: int = 8877 + """Standard env-ctrl container port used across all images.""" + + env_ctrl_package_path: str = "packages/environment_control/environment_control" + """Path to environment_control package relative to repo root. Staged to containers during optimization.""" + + volume_prefix: str = "webarena-verified" + """Prefix for Docker volume names (e.g., webarena-verified-map-tile-db).""" + + shopping: ShoppingSettings = ShoppingSettings() + """Shopping site settings.""" + + shopping_admin: ShoppingAdminSettings = ShoppingAdminSettings() + """Shopping admin site settings.""" + + reddit: RedditSettings = RedditSettings() + """Reddit (forum) site settings.""" + + gitlab: GitlabSettings = GitlabSettings() + """GitLab site settings.""" + + wikipedia: WikipediaSettings = WikipediaSettings() + """Wikipedia site settings.""" + + map: MapSettings = MapSettings() + """OpenStreetMap site settings.""" + + +@lru_cache +def get_settings() -> DevSettings: + """Get development settings (cached). + + Access site settings via attributes: + get_settings().shopping_admin + get_settings().shopping + get_settings().reddit + get_settings().gitlab + get_settings().wikipedia + """ + return DevSettings() + + +__all__ = [ + "BaseSiteSettings", + "DevSettings", + "GitlabSettings", + "MapSettings", + "RedditSettings", + "ShoppingAdminSettings", + "ShoppingSettings", + "WikipediaSettings", + "get_settings", +] diff --git a/dev/environments/tasks.py b/dev/environments/tasks.py new file mode 100644 index 0000000..b27018b --- /dev/null +++ b/dev/environments/tasks.py @@ -0,0 +1,42 @@ +"""Environment management tasks. + +Provides: +- envs.sites - List available sites +- envs.docker.* - Docker container and image operations +""" + +from __future__ import annotations + +import logging +from typing import TYPE_CHECKING + +from invoke import Collection, task + +if TYPE_CHECKING: + from invoke.context import Context + +from dev.environments.docker import ns as docker_ns +from dev.environments.docker.utils.sites import _DOCKER_SITES, _get_container_name + +# Configure logging +logging.basicConfig(level=logging.INFO, format="%(message)s") +logger = logging.getLogger(__name__) + + +@task +def sites(ctx: Context) -> None: + """List all available sites.""" + logger.info("Available sites:") + for site_enum in sorted(_DOCKER_SITES, key=lambda s: s.value): + site_name = site_enum.value + container_name = _get_container_name(site_name) + logger.info(" %s (container: %s)", site_name, container_name) + + +# --- Collection --- + +ns = Collection() +ns.add_task(sites) # ty: ignore[invalid-argument-type] + +# Add docker sub-namespace +ns.add_collection(docker_ns, name="docker") diff --git a/dev/tasks.py b/dev/tasks.py deleted file mode 100644 index 049d767..0000000 --- a/dev/tasks.py +++ /dev/null @@ -1,285 +0,0 @@ -"""Invoke tasks for WebArena-Verified development.""" - -import json -import shutil -import subprocess -import tempfile -from pathlib import Path - -from compact_json import EolStyle, Formatter -from invoke.tasks import task - -from .utils.docker_utils import ( - DOCKER_IMAGE, - GIT_REPO_URL, - check_repo_clean_at_tag, - confirm, - get_version, - validate_semver, -) - -# Dataset path -DATASET_FILE = Path("assets/dataset/webarena-verified.json") - - -@task -def env_init(c): - """Initialize development environment: sync all dependencies and install pre-commit hooks.""" - c.run("uv sync --all-groups --all-extras") - c.run("uv run pre-commit install") - - -@task -def docs_serve(c): - """Serve the documentation locally with live reload.""" - c.run("uv run mkdocs serve") - - -@task -def docs_build(c): - """Build the documentation site.""" - c.run("uv run mkdocs build") - - -@task -def docker_build(c, push=False, dry_run=False, version=None): - """Build Docker image with current version tag. - - Args: - push: If True, push the image after building (with confirmation prompts) - dry_run: If True, print commands without executing them - version: Optional semver version to build (clones repo at tag in temp dir) - """ - if version: - _docker_build_from_tag(c, version, push, dry_run) - else: - _docker_build_local(c, push, dry_run) - - -def _docker_build_local(c, push: bool, dry_run: bool) -> None: - """Build Docker image from local repo (must be clean and at tag).""" - version = get_version() - check_repo_clean_at_tag(version) - _build_and_push(c, version, push, dry_run, build_dir=".") - - -def _docker_build_from_tag(c, version: str, push: bool, dry_run: bool) -> None: - """Build Docker image by cloning repo at specific tag.""" - validate_semver(version) - tag = f"v{version}" - - with tempfile.TemporaryDirectory() as tmpdir: - tmppath = Path(tmpdir) - print(f"Using temporary directory: {tmpdir}") - - # Clone repo at the exact tag - print(f"Cloning {GIT_REPO_URL} at tag {tag}...") - c.run(f"git clone --depth 1 --branch {tag} {GIT_REPO_URL} {tmpdir}") - - # TODO: Remove once all releases have Dockerfile - _copy_missing_files(tmppath) - - # Sync dependencies (no dev, no extras) - print("Syncing dependencies...") - with c.cd(tmpdir): - c.run("uv sync --no-dev --no-group dev") - c.run("uv lock") - - # TODO: Remove this check once all releases have tests - tests_dir = tmppath / "tests" - if tests_dir.exists() and any(tests_dir.glob("test_*.py")): - print("Running tests...") - c.run("uv run pytest") - else: - print("Skipping tests (no tests directory found)") - - _build_and_push(c, version, push, dry_run, build_dir=tmpdir) - - -def _copy_missing_files(target_dir: Path) -> None: - """Copy Dockerfile from current dir if missing in target. - - TODO: Remove this function once all releases have Dockerfile. - """ - target_dockerfile = target_dir / "Dockerfile" - if not target_dockerfile.exists(): - source_dockerfile = Path("Dockerfile") - if source_dockerfile.exists(): - print(f"Copying Dockerfile to {target_dockerfile}...") - shutil.copy(source_dockerfile, target_dockerfile) - - -def _build_and_push(c, version: str, push: bool, dry_run: bool, build_dir: str) -> None: - """Build and optionally push Docker image.""" - version_tag = f"{DOCKER_IMAGE}:{version}" - latest_tag = f"{DOCKER_IMAGE}:latest" - - def run(cmd: str) -> None: - if dry_run: - print(f"[dry-run] {cmd}") - else: - c.run(cmd) - - # Build the image - print(f"Building {version_tag}...") - run(f"docker build -t {version_tag} {build_dir}") - - if not push: - print(f"Image built: {version_tag}") - print("Run with --push to push the image to the registry.") - return - - # Ask before pushing version tag - if not dry_run and not confirm(f"Push {version_tag}?"): - print("Push cancelled.") - return - - print(f"Pushing {version_tag}...") - run(f"docker push {version_tag}") - - # Ask before tagging and pushing latest - if dry_run or confirm(f"Also tag and push as {latest_tag}?"): - run(f"docker tag {version_tag} {latest_tag}") - print(f"Pushing {latest_tag}...") - run(f"docker push {latest_tag}") - - -@task -def data_format(c): - """Format dataset JSON file.""" - data = load_json(DATASET_FILE) - save_json(DATASET_FILE, data) - - -@task -def code_format_and_check(c): - """Format code using ruff and run type checking.""" - c.run("uv run ruff check src dev --fix --unsafe-fixes") - c.run("uv run ruff format src dev") - c.run("uv run ty check src dev") - - -def load_json(file_path: Path) -> list[dict]: - """Load JSON file. - - Args: - file_path: Path to JSON file - - Returns: - Parsed JSON data - - Raises: - FileNotFoundError: If file doesn't exist - ValueError: If file is invalid JSON - """ - if not file_path.exists(): - raise FileNotFoundError(f"File not found: {file_path}") - - try: - with open(file_path) as f: - data = json.load(f) - except json.JSONDecodeError as e: - raise ValueError(f"Invalid JSON in {file_path}: {e}") from e - - if not isinstance(data, list): - raise ValueError(f"Expected JSON array in {file_path}, got {type(data).__name__}") - - return data - - -def check_git_status(file_path: Path) -> bool: - """Check if the file has uncommitted changes in git. - - Args: - file_path: Path to the file to check - - Returns: - True if file is safe to modify (not tracked or no uncommitted changes) - False if file has uncommitted changes - """ - try: - # Check if file is tracked by git - result = subprocess.run( - ["git", "ls-files", "--error-unmatch", str(file_path)], - capture_output=True, - text=True, - check=False, - ) - - # If file is not tracked, it's safe to modify - if result.returncode != 0: - return True - - # File is tracked, check for uncommitted changes - result = subprocess.run( - ["git", "status", "--porcelain", str(file_path)], - capture_output=True, - text=True, - check=True, - ) - - # If output is empty, no uncommitted changes - return len(result.stdout.strip()) == 0 - - except subprocess.CalledProcessError: - # If git command fails, assume it's safe (not a git repo) - return True - - -def save_json(file_path: Path, data: list[dict], skip_git_check: bool = False) -> None: - """Save JSON data to file with consistent formatting and key ordering. - - Args: - file_path: Path to output file - data: List of task objects - skip_git_check: Skip git status check - - Raises: - ValueError: If file has uncommitted changes in git - """ - # Check git status before modifying file - if not skip_git_check and not check_git_status(file_path): - raise ValueError( - f"{file_path} has uncommitted changes. Please commit or stash them first before running transforms." - ) - - # Define the desired key order - key_order = [ - "sites", - "task_id", - "intent_template_id", - "start_url", - "start_urls", - "intent", - "intent_template", - "instantiation_dict", - "retrieved_data_format_spec", - "start_url_context", - "eval", - "revision", - ] - - # Reorder keys in each task - ordered_data = [] - for task_data in data: - ordered_task = {} - # Add keys in the specified order if they exist - for key in key_order: - if key in task_data: - ordered_task[key] = task_data[key] - # Add any remaining keys in alphabetical order - remaining_keys = sorted(set(task_data.keys()) - set(key_order)) - for key in remaining_keys: - ordered_task[key] = task_data[key] - ordered_data.append(ordered_task) - - # Configure compact_json formatter - formatter = Formatter() - formatter.indent_spaces = 2 - formatter.max_inline_complexity = 10 - formatter.json_eol_style = EolStyle.LF - formatter.omit_trailing_whitespace = True - - # Ensure directory exists and write formatted output - file_path.parent.mkdir(parents=True, exist_ok=True) - formatter.dump(ordered_data, output_file=str(file_path)) diff --git a/dev/utils/git_utils.py b/dev/utils/git_utils.py new file mode 100644 index 0000000..639df60 --- /dev/null +++ b/dev/utils/git_utils.py @@ -0,0 +1,21 @@ +"""Git utilities for dev scripts.""" + +import subprocess + + +def get_short_sha() -> str: + """Get the short git SHA of the current HEAD. + + Returns: + Short git SHA (7 characters) or 'unknown' if not in a git repo. + """ + try: + result = subprocess.run( + ["git", "rev-parse", "--short", "HEAD"], + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + except (subprocess.CalledProcessError, FileNotFoundError): + return "unknown" diff --git a/dev/utils/logging_utils/__init__.py b/dev/utils/logging_utils/__init__.py new file mode 100644 index 0000000..a36b598 --- /dev/null +++ b/dev/utils/logging_utils/__init__.py @@ -0,0 +1,54 @@ +"""CLI output formatting using Rich. + +This module provides consistent, beautiful terminal output for Docker CLI commands. +It replaces plain logger.info() calls with Rich components for better UX. + +Features: + - Banners: Command headers with site context + - Config tables: Display configuration in aligned tables + - Step context: Command execution with spinner and status + - Success/failure: Colored status messages + - List printing: Indented list output + +Requirements: + Rich library (dev dependency): uv sync --group dev + +Usage: + from dev.utils.logging_utils import ( + console, print_banner, StepContext, print_success + ) + + print_banner("START CONTAINER", data={"Site": "shopping-admin"}) + print_table({"Image": "am1n3e/webarena:shopping_admin", "Port": 6680}) + with StepContext.create("docker rm -f shopping_admin", desc="Removing container"): + ctx.run("docker rm -f shopping_admin", hide=True) + print_success("Container started!", URL="http://localhost:6680") +""" + +from .console import console +from .printers import ( + print_banner, + print_error, + print_failure, + print_info, + print_list, + print_success, + print_table, + print_warning, + with_banner, +) +from .step_context import StepContext + +__all__ = [ + "StepContext", + "console", + "print_banner", + "print_error", + "print_failure", + "print_info", + "print_list", + "print_success", + "print_table", + "print_warning", + "with_banner", +] diff --git a/dev/utils/logging_utils/console.py b/dev/utils/logging_utils/console.py new file mode 100644 index 0000000..d749525 --- /dev/null +++ b/dev/utils/logging_utils/console.py @@ -0,0 +1,26 @@ +"""Rich console setup and availability check.""" + +from __future__ import annotations + +import sys + +# Check for Rich library availability +try: + from rich.console import Console + from rich.panel import Panel + from rich.table import Table + from rich.text import Text +except ImportError: + print( + "ERROR: Rich library is required for CLI output formatting.\n" + "Rich is a dev dependency. Install it with:\n\n" + " uv sync --group dev\n\n" + "Or install rich directly:\n\n" + " uv pip install rich\n", + file=sys.stderr, + ) + raise + +console = Console() + +__all__ = ["Console", "Panel", "Table", "Text", "console"] diff --git a/dev/utils/logging_utils/printers.py b/dev/utils/logging_utils/printers.py new file mode 100644 index 0000000..bc95a7f --- /dev/null +++ b/dev/utils/logging_utils/printers.py @@ -0,0 +1,238 @@ +"""Basic print utilities using Rich.""" + +from __future__ import annotations + +import functools +import inspect +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar + +from .console import Panel, Table, Text, console + +if TYPE_CHECKING: + from collections.abc import Callable + +P = ParamSpec("P") +R = TypeVar("R") + + +def print_banner(title: str, data: dict[str, Any] | None = None) -> None: + """Print command banner with title, separator, and key-value pairs. + + Args: + title: Main title text (e.g., "START CONTAINER"). + data: Optional dict of key-value pairs to display below the title. + + Examples: + >>> print_banner("CREATE SLIM IMAGE") + โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ + โ”‚ CREATE SLIM IMAGE โ”‚ + โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ + + >>> print_banner("START CONTAINER", {"Site": "gitlab", "Mode": "slim"}) + โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ + โ”‚ START CONTAINER โ”‚ + โ”‚ Site: gitlab โ”‚ + โ”‚ Mode: slim โ”‚ + โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ + """ + content = Text() + if data: + for i, (key, value) in enumerate(data.items()): + if i > 0: + content.append("\n") + content.append(f"{key}: ", style="dim") + content.append(str(value), style="cyan") + console.print() + console.print(Panel(content, style="blue", title=f"[bold]{title}[/]", title_align="center")) + console.print() + + +def print_table(data: dict[str, Any]) -> None: + """Print key-value data as an aligned table. + + Args: + data: Dictionary of key-value pairs to display. + + Example: + >>> print_table({"Image": "am1n3e/webarena:shopping", "Port": 6680}) + Image am1n3e/webarena:shopping + Port 6680 + """ + table = Table(show_header=False, box=None, padding=(0, 2)) + table.add_column("Key", style="dim") + table.add_column("Value", style="cyan") + for key, value in data.items(): + table.add_row(key, str(value)) + console.print(table) + console.print() + + +def print_success(message: str = "SUCCESS", **details: Any) -> None: + """Print success message with optional details. + + Args: + message: Success message to display (default: "SUCCESS"). + **details: Key-value pairs to display below the message. + + Example: + >>> print_success("Container started!", Container="shopping_admin", URL="http://localhost:6680") + + โœ“ Container started! + Container: shopping_admin + URL: http://localhost:6680 + """ + console.print() + console.print(f"[bold green]โœ“ {message}[/]") + if details: + for key, value in details.items(): + console.print(f" [dim]{key}:[/] [cyan]{value}[/]") + + +def print_failure(message: str = "FAILED", error: str | None = None) -> None: + """Print failure message with optional error details. + + Args: + message: Failure message to display (default: "FAILED"). + error: Optional error details to show below the message. + + Example: + >>> print_failure("Container failed to start", error="Port 6680 already in use") + + โœ— Container failed to start + Port 6680 already in use + """ + console.print() + console.print(f"[bold red]โœ— {message}[/]") + if error: + console.print(f" [dim]{error}[/]") + + +def print_list(lines: list[str], indent_rest: int = 2) -> None: + """Print a list with first line unindented and rest indented. + + Args: + lines: List of strings to print. + indent_rest: Number of spaces to indent lines after the first (default: 2). + + Example: + >>> print_list(["Next steps:", "inv docker.shell", "inv docker.stop"]) + Next steps: + inv docker.shell + inv docker.stop + """ + if not lines: + return + console.print(lines[0]) + prefix = " " * indent_rest + for line in lines[1:]: + console.print(f"{prefix}{line}") + + +def print_info(message: str) -> None: + """Print an info message. + + Args: + message: Message to display. + + Example: + >>> print_info("Container shopping_admin removed.") + Container shopping_admin removed. + """ + console.print(f" {message}") + + +def print_warning(message: str) -> None: + """Print a warning message. + + Args: + message: Warning message to display. + + Example: + >>> print_warning("Port 6680 is already in use, using 6681 instead.") + โš  Port 6680 is already in use, using 6681 instead. + """ + console.print(f" [yellow]โš  {message}[/]") + + +def print_error(message: str) -> None: + """Print an error message (without full failure banner). + + Args: + message: Error message to display. + + Example: + >>> print_error("Failed to connect to Docker daemon.") + โœ— Failed to connect to Docker daemon. + """ + console.print(f" [red]โœ— {message}[/]") + + +def with_banner( + exclude: set[str] | None = None, + include_false: bool = False, +) -> Callable[[Callable[P, R]], Callable[P, R]]: + """Decorator that prints a banner with function name and args before execution. + + Args: + exclude: Additional parameter names to exclude from banner. "self" and "ctx" + are always excluded automatically. + include_false: If True, include parameters with False/None values (default: False). + + Example: + @with_banner() + def create_optimized_img(ctx, source=None, output=None, port=6680, skip_tests=False): + ... + + # When called as: create_optimized_img(ctx, source="img:v1", port=8080) + # Prints banner: + # โ•ญโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฎ + # โ”‚ CREATE OPTIMIZED IMG โ”‚ + # โ”‚ Source: img:v1 โ”‚ + # โ”‚ Port: 8080 โ”‚ + # โ•ฐโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ•ฏ + """ + # Always exclude self and ctx, plus any user-provided excludes + base_exclude = {"self", "ctx"} + effective_exclude = base_exclude | (exclude or set()) + + def decorator(func: Callable[P, R]) -> Callable[P, R]: + @functools.wraps(func) + def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: + # Get function signature + sig = inspect.signature(func) + bound = sig.bind(*args, **kwargs) + bound.apply_defaults() + + # Build banner title from function name + title = func.__name__.replace("_", " ").upper() # type: ignore[attr-defined] + + # Build data dict from arguments + data = {} + for name, value in bound.arguments.items(): + if name in effective_exclude: + continue + if not include_false and (value is None or value is False): + continue + # Format the key nicely + key = name.replace("_", " ").title() + data[key] = value + + print_banner(title, data if data else None) + return func(*args, **kwargs) + + return wrapper + + return decorator + + +__all__ = [ + "print_banner", + "print_error", + "print_failure", + "print_info", + "print_list", + "print_success", + "print_table", + "print_warning", + "with_banner", +] diff --git a/dev/utils/logging_utils/step_context.py b/dev/utils/logging_utils/step_context.py new file mode 100644 index 0000000..11efdfa --- /dev/null +++ b/dev/utils/logging_utils/step_context.py @@ -0,0 +1,108 @@ +"""Step context manager for running commands with spinner and status.""" + +from __future__ import annotations + +from typing import Any + +from rich.markup import escape + +from .console import console + + +class StepContext: + """Context manager for running a step with spinner and status. + + Prints command at top, shows spinner with description while running, + then checkmark or X with description on completion. + + Example: + >>> with StepContext.create("docker build -t myimage .", desc="Building image") as step: + ... result = ctx.run("docker build -t myimage .", hide=True) + ... if result.failed: + ... step.mark_failed("Build failed") + $ docker build -t myimage . + + โœ“ Success: Building image + + >>> with StepContext.create("docker push myimage", desc="Pushing") as step: + ... step.mark_failed("Authentication failed") + $ docker push myimage + โœ— Failed: Pushing + Authentication failed + + Use plain=True to avoid Rich markup parsing (useful when command output + contains paths like [/etc/gitlab] that Rich interprets as markup). + """ + + def __init__(self, cmd: str, desc: str | None = None, plain: bool = False) -> None: + self.cmd = cmd + self.desc = desc or cmd + self.plain = plain + self._failed = False + self._error_msg: str | None = None + self._status = None + + @classmethod + def create(cls, cmd: str, desc: str | None = None, plain: bool = False) -> StepContext: + """Create a step context for running commands with spinner and status. + + Args: + cmd: The command being run (displayed to user). + desc: Optional description shown next to spinner. + plain: Use plain print instead of Rich (avoids markup parsing issues). + + Returns: + StepContext that shows spinner while running, then โœ“ or โœ— on exit. + + Example: + >>> with StepContext.create("docker run -d nginx", desc="Starting container") as step: + ... result = ctx.run("docker run -d nginx", hide=True) + ... if not result.ok: + ... step.mark_failed("Container failed to start") + """ + return cls(cmd, desc, plain) + + def mark_failed(self, msg: str | None = None) -> None: + """Mark this step as failed with optional error message.""" + self._failed = True + self._error_msg = msg + + def __enter__(self) -> StepContext: + if self.plain: + print(f"$ {self.cmd}") + print() + else: + console.print(f"[dim]$[/] [green]{self.cmd}[/]") + console.print() + self._status = console.status(f"[dim]Running:[/] {self.desc}...", spinner="dots") + self._status.start() + return self + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + if self._status: + self._status.stop() + + if exc_type is not None: + self._failed = True + if self._error_msg is None: + self._error_msg = str(exc_val) + + if self.plain: + if self._failed: + print(f"โœ— Failed: {self.desc}") + if self._error_msg: + print(f" {self._error_msg}") + else: + print(f"โœ“ Success: {self.desc}") + print("-" * 40) + else: + if self._failed: + console.print(f"[red]โœ— Failed:[/] {self.desc}") + if self._error_msg: + console.print(f" [dim]{escape(self._error_msg)}[/]") + else: + console.print(f"[green]โœ“ Success:[/] {self.desc}") + console.rule(style="dim") + + +__all__ = ["StepContext"] diff --git a/dev/utils/network_utils.py b/dev/utils/network_utils.py new file mode 100644 index 0000000..0f45088 --- /dev/null +++ b/dev/utils/network_utils.py @@ -0,0 +1,15 @@ +"""Network utilities.""" + +import socket + + +def find_free_port() -> int: + """Find and return an available TCP port. + + Uses the OS to allocate a free port by binding to port 0, + which lets the OS choose an available port. + """ + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("", 0)) + return s.getsockname()[1] diff --git a/dev/utils/path_utils.py b/dev/utils/path_utils.py new file mode 100644 index 0000000..e01182f --- /dev/null +++ b/dev/utils/path_utils.py @@ -0,0 +1,23 @@ +"""Path utilities for dev scripts.""" + +from functools import lru_cache +from pathlib import Path + + +@lru_cache(maxsize=1) +def get_repo_root() -> Path: + """Get the repository root directory. + + Traverses up from this file to find the .git directory. + + Returns: + Path to the repository root. + + Raises: + FileNotFoundError: If the repository root cannot be found. + """ + current = Path(__file__).resolve() + for parent in current.parents: + if (parent / ".git").exists(): + return parent + raise FileNotFoundError("Could not find repository root (no .git directory found)") diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..39d2246 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,76 @@ +name: webarena-verified-envs + +services: + shopping_admin: + image: am1n3e/webarena-verified-shopping_admin:latest + container_name: webarena-verified-shopping_admin + ports: + - "${WA_SHOPPING_ADMIN_PORT:-7780}:80" + - "${WA_SHOPPING_ADMIN_ENV_CTRL_PORT:-7781}:8877" + restart: unless-stopped + + shopping: + image: am1n3e/webarena-verified-shopping:latest + container_name: webarena-verified-shopping + ports: + - "${WA_SHOPPING_PORT:-7770}:80" + - "${WA_SHOPPING_ENV_CTRL_PORT:-7771}:8877" + restart: unless-stopped + + gitlab: + image: am1n3e/webarena-verified-gitlab:latest + container_name: webarena-verified-gitlab + ports: + - "${WA_GITLAB_PORT:-8023}:8023" + - "${WA_GITLAB_ENV_CTRL_PORT:-8024}:8877" + restart: unless-stopped + shm_size: '2gb' + + reddit: + image: am1n3e/webarena-verified-reddit:latest + container_name: webarena-verified-reddit + ports: + - "${WA_REDDIT_PORT:-9999}:80" + - "${WA_REDDIT_ENV_CTRL_PORT:-9998}:8877" + restart: unless-stopped + + wikipedia: + image: am1n3e/webarena-verified-wikipedia:latest + container_name: webarena-verified-wikipedia + ports: + - "${WA_WIKIPEDIA_PORT:-8888}:8080" + - "${WA_WIKIPEDIA_ENV_CTRL_PORT:-8889}:8877" + volumes: + - webarena-verified-wikipedia-data:/data:ro + restart: unless-stopped + + map: + image: am1n3e/webarena-verified-map:latest + container_name: webarena-verified-map + platform: linux/amd64 + ports: + - "${WA_MAP_PORT:-3030}:8080" # Apache (Rails + tiles + OSRM/Nominatim) + - "${WA_MAP_ENV_CTRL_PORT:-3031}:8877" + volumes: + - webarena-verified-map-tile-db:/data/database + - webarena-verified-map-routing-car:/data/routing/car + - webarena-verified-map-routing-bike:/data/routing/bike + - webarena-verified-map-routing-foot:/data/routing/foot + - webarena-verified-map-nominatim-db:/data/nominatim/postgres + - webarena-verified-map-nominatim-flatnode:/data/nominatim/flatnode + - webarena-verified-map-website-db:/var/lib/postgresql/14/main + - webarena-verified-map-tiles:/data/tiles + - webarena-verified-map-style:/data/style + restart: unless-stopped + +volumes: + webarena-verified-wikipedia-data: + webarena-verified-map-tile-db: + webarena-verified-map-routing-car: + webarena-verified-map-routing-bike: + webarena-verified-map-routing-foot: + webarena-verified-map-nominatim-db: + webarena-verified-map-nominatim-flatnode: + webarena-verified-map-website-db: + webarena-verified-map-tiles: + webarena-verified-map-style: diff --git a/docs/environments/docker_images.md b/docs/environments/docker_images.md deleted file mode 100644 index 053b4ff..0000000 --- a/docs/environments/docker_images.md +++ /dev/null @@ -1,184 +0,0 @@ -# Docker Images - -To improve reset time and reduce storage requirements, we provide recipes to create **slim images** - optimized versions of the original environments while keeping the exact content and functionality. - -## Size Improvements - -Slim images are significantly smaller than their original counterparts: - -| Environment | Original Size | Slim Size | Reduction | -|------------|---------------|-----------|-----------| -| Shopping Admin | 19.9 GB | 4.98 GB | **~70% smaller** | -| Shopping | 117 GB | 17.8 GB | **~85% smaller** | -| Reddit | 107 GB | 19 GB | **~82% smaller** | -| GitLab | 155 GB | 34 GB | **~78% smaller** | - -**Benefits:** - -- Smaller storage and memory footprint -- HTTP header-based authentication bypassing UI login -- All functionality preserved - -## Shopping Admin - -### 1. Create the slim image - -```bash -cd scripts/environments/shopping_admin -bash create_slim_image.sh -``` - -### 2. Run the container with auto-initialization - -```bash -docker run -d --name admin-slim -p 7780:80 \ - -e MAGENTO_BASE_URL=http://localhost:7780 \ - shopping_admin_final_0719:slim -``` - -### Access the admin (auto-login via header in Playwright) - -```python -from playwright.async_api import async_playwright - -async with async_playwright() as p: - browser = await p.chromium.launch() - context = await browser.new_context() - - # Set auto-login header - await context.set_extra_http_headers({ - "X-M2-Admin-Auto-Login": "admin:admin1234" - }) - - page = await context.new_page() - await page.goto("http://localhost:7780/admin") - # You're now logged in as admin -``` - -### Manual Initialization - -If you want to manually initialize the environment (e.g., after stopping the container), run: - -```bash -# Start container -docker run -d --name admin-slim -p 7780:80 shopping_admin_final_0719:slim - -# Initialize Magento -docker exec admin-slim magento-init http://localhost:7780 -``` - -### Troubleshooting - -**Auto-login not working:** - -1. Test with curl: - ```bash - # Should redirect to dashboard and set cookies - curl -I -H "X-M2-Admin-Auto-Login: admin:admin1234" \ - http://localhost:7780/admin - - # Without header - should redirect to login page - curl -I http://localhost:7780/admin - ``` - -2. Check module is enabled: - ```bash - docker exec admin-slim /var/www/magento2/bin/magento module:status WebArena_AutoLogin - ``` - Should show "Module is enabled" - -3. Check DI is compiled: - ```bash - docker exec admin-slim ls -la /var/www/magento2/generated/code/ - ``` - Should contain compiled classes - -4. Check logs: - ```bash - docker exec admin-slim tail -f /var/www/magento2/var/log/system.log - ``` - Look for "Auto-login successful" or error messages - -5. Recompile if needed: - ```bash - docker exec admin-slim /var/www/magento2/bin/magento setup:di:compile - ``` - -**Container not initializing:** - -- Check container logs: `docker logs admin-slim` -- Verify MAGENTO_BASE_URL is set correctly -- Wait ~30 seconds for initialization to complete - -**Database reset not working:** - -- Ensure archive exists: `docker exec admin-slim ls -lh /var/backups/mysql/data.tar.gz` -- Check services status: `docker exec admin-slim supervisorctl status` - -## Reddit - -### 1. Create the slim image - -```bash -cd scripts/environments/reddit -bash create_slim_image.sh -``` - -**Note:** The script reuses existing data if available: - -- PostgreSQL archive (~1.6GB) -- Optimized submission images (~6GB) -- This saves ~30-60 minutes on subsequent runs - -### 2. Run the container - -```bash -docker run -d --name reddit-slim -p 9999:80 postmill-populated-exposed-withimg:slim -``` - -The container is self-contained (no volume mounts needed) and auto-initializes on first start (~2-3 minutes). - -### Manual Initialization - -Not typically needed (container auto-initializes), but can be run manually if needed: - -```bash -# Start container -docker run -d --name reddit-slim -p 9999:80 postmill-populated-exposed-withimg:slim - -# Check initialization status -docker exec reddit-slim postmill-init -``` - -### Troubleshooting - -**Container not initializing:** - -- Check container logs: `docker logs reddit-slim` -- Wait ~2-3 minutes for initial data extraction -- Check initialization status: `docker exec reddit-slim cat /run/postmill.env` - -**Database reset not working:** - -- Ensure archives exist: - ```bash - docker exec reddit-slim ls -lh /var/backups/pgsql/data.tar.gz - docker exec reddit-slim ls -lh /var/backups/images/submission_images.tar.gz - ``` -- Check services status: `docker exec reddit-slim supervisorctl status` -- Verify database integrity: `docker exec reddit-slim postmill-init` (shows validation output) - -**Patches not applied:** - -1. Check rate limits removed: - ```bash - docker exec reddit-slim grep -c '@RateLimit' /var/www/html/src/DataObject/SubmissionData.php - ``` - Should return `0` - -2. Check HTTP client configured: - ```bash - docker exec reddit-slim grep 'alias: postmill.http_client.default' \ - /var/www/html/config/packages/http_client.yaml - ``` - Should show the configuration line diff --git a/docs/environments/gitlab.md b/docs/environments/gitlab.md new file mode 100644 index 0000000..489eab3 --- /dev/null +++ b/docs/environments/gitlab.md @@ -0,0 +1,38 @@ +# GitLab + +GitLab Community Edition for source code management and CI/CD. + +| Property | Value | +|----------|-------| +| Port | 8023 | +| Env-Ctrl Port | 8024 | +| Image | `am1n3e/webarena-verified-gitlab` | +| Container | `webarena-verified-gitlab` | + +## Quick Start + +```bash +# Using CLI (recommended) +webarena-verified env start --site gitlab + +# Using Docker directly +docker run -d --name webarena-verified-gitlab -p 8023:8023 -p 8024:8877 am1n3e/webarena-verified-gitlab +``` + +Access at: http://localhost:8023 + +## Resource Requirements + +GitLab requires significant resources: + +- **Memory:** Minimum 4GB RAM recommended +- **CPU:** Multiple cores for responsive performance +- **Startup time:** May take several minutes for all services to initialize + +## Optimizations + +The optimized image includes: + +- Reduced image size (~78% smaller than original) +- Environment control (env-ctrl) for runtime management +- Optimized GitLab configuration for test workloads diff --git a/docs/environments/index.md b/docs/environments/index.md index 48976ca..a7f9ebe 100644 --- a/docs/environments/index.md +++ b/docs/environments/index.md @@ -2,15 +2,170 @@ WebArena-Verified uses Docker containers to provide isolated, reproducible test environments for each website in the benchmark. -## Overview +## Available Environments -Each test environment (Shopping, Reddit, GitLab, etc.) runs as a self-contained Docker container with: +| Environment | Description | Port | Env-Ctrl Port | Documentation | +|-------------|-------------|------|---------------|---------------| +| Shopping Admin | Magento admin panel | 7780 | 7781 | [shopping_admin.md](shopping_admin.md) | +| Shopping | Magento storefront | 7770 | 7771 | [shopping.md](shopping.md) | +| Reddit | Postmill forum | 9999 | 9998 | [reddit.md](reddit.md) | +| GitLab | GitLab CE | 8023 | 8024 | [gitlab.md](gitlab.md) | +| Wikipedia | MediaWiki | 8888 | 8889 | [wikipedia.md](wikipedia.md) | +| Map | OpenStreetMap | 3030 | 3031 | [map.md](map.md) | -- The application and its dependencies -- Pre-populated test data -- Environment control utilities for management via CLI or HTTP +## Docker Images -## Sections +All environments are available as optimized Docker images on Docker Hub: -- [Docker Images](docker_images.md) - Instructions for building and running slim Docker images -- [Environment Control](environment_control.md) - The `environment_control` package for managing environments +| Site | Image | +|------|-------| +| Shopping Admin | `am1n3e/webarena-verified-shopping_admin` | +| Shopping | `am1n3e/webarena-verified-shopping` | +| Reddit | `am1n3e/webarena-verified-reddit` | +| GitLab | `am1n3e/webarena-verified-gitlab` | +| Wikipedia | `am1n3e/webarena-verified-wikipedia` | +| Map | `am1n3e/webarena-verified-map` | + +## Size Improvements + +Optimized images are significantly smaller than their original counterparts: + +| Environment | Original Size | Optimized Size | Reduction | +|-------------|---------------|----------------|-----------| +| Shopping Admin | 19.9 GB | 2.9 GB | ~85% smaller | +| Shopping | 117 GB | 13.3 GB | ~89% smaller | +| Reddit | 107 GB | 8.41 GB | ~92% smaller | +| GitLab | 155 GB | 31.6 GB | ~80% smaller | +| Wikipedia | - | 115 MB | - | +| Map | - | 3.28 GB | - | + +**Benefits of optimized images:** + +- Smaller storage and memory footprint +- HTTP header-based authentication (bypasses UI login) +- Environment control (env-ctrl) for management via CLI or HTTP +- All original functionality preserved + +## Environment Variables + +Docker Compose uses environment variables for port configuration: + +| Variable | Default | Description | +|----------|---------|-------------| +| `WA_SHOPPING_ADMIN_PORT` | 7780 | Shopping Admin main port | +| `WA_SHOPPING_PORT` | 7770 | Shopping main port | +| `WA_GITLAB_PORT` | 8023 | GitLab main port | +| `WA_REDDIT_PORT` | 9999 | Reddit main port | +| `WA_WIKIPEDIA_PORT` | 8888 | Wikipedia main port | +| `WA_MAP_PORT` | 3030 | Map main port | + +Each site also has an `_ENV_CTRL_PORT` variable (e.g., `WA_SHOPPING_ADMIN_ENV_CTRL_PORT`). + +## Quick Start + +### Using the CLI (Recommended) + +The easiest way to run environments is using the built-in CLI: + +```bash +# Start a site (waits for services to be ready) +webarena-verified env start --site shopping + +# Check status +webarena-verified env status --site shopping + +# Stop a site +webarena-verified env stop --site shopping + +# Stop all running sites +webarena-verified env stop-all +``` + +Start multiple sites: + +```bash +webarena-verified env start --site shopping +webarena-verified env start --site shopping_admin +webarena-verified env start --site reddit +webarena-verified env start --site gitlab +``` + +### Wikipedia and Map Data + +Wikipedia and Map require external data files to be downloaded before starting: + +```bash +# Wikipedia (~100GB download) +webarena-verified env setup init --site wikipedia --data-dir ./downloads +webarena-verified env start --site wikipedia --data-dir ./downloads + +# Map (~60GB download) +webarena-verified env setup init --site map --data-dir ./downloads +webarena-verified env start --site map +``` + +See the [Wikipedia](wikipedia.md) and [Map](map.md) documentation for details. + +### Using Docker Directly + +You can also run environments directly with Docker: + +```bash +# Shopping (Magento) +docker run -d --name webarena-verified-shopping -p 7770:80 -p 7771:8877 am1n3e/webarena-verified-shopping + +# Shopping Admin +docker run -d --name webarena-verified-shopping_admin -p 7780:80 -p 7781:8877 am1n3e/webarena-verified-shopping_admin + +# Reddit (Postmill) +docker run -d --name webarena-verified-reddit -p 9999:80 -p 9998:8877 am1n3e/webarena-verified-reddit + +# GitLab +docker run -d --name webarena-verified-gitlab -p 8023:8023 -p 8024:8877 am1n3e/webarena-verified-gitlab +``` + +## CLI Command Reference + +### Container Lifecycle + +```bash +webarena-verified env start --site # Start container (waits by default) +webarena-verified env start --site --no-wait # Start without waiting +webarena-verified env start --site --port 8080 # Custom port +webarena-verified env stop --site # Stop and remove container +webarena-verified env stop-all # Stop all containers +webarena-verified env status --site # Check status +``` + +### Data Setup (Wikipedia, Map) + +```bash +webarena-verified env setup init --site --data-dir ./data # Download and setup volumes +webarena-verified env setup clean --site # Remove volumes +``` + +## Troubleshooting + +### Container not starting + +```bash +# Check container logs +docker logs webarena-verified- + +# Check services inside container +docker exec webarena-verified- supervisorctl status +``` + +### Health check failing + +```bash +# Use env-ctrl to check status +inv envs.docker.check --site + +# Or directly via HTTP +curl http://localhost:/health +``` + +## Further Reading + +- [Environment Control](environment_control.md) - The `env-ctrl` package for runtime management diff --git a/docs/environments/map.md b/docs/environments/map.md new file mode 100644 index 0000000..af7b452 --- /dev/null +++ b/docs/environments/map.md @@ -0,0 +1,93 @@ +# Map + +OpenStreetMap-based mapping platform with tiles, geocoding, and routing. + +| Property | Value | +|----------|-------| +| Port | 3030 | +| Env-Ctrl Port | 3031 | +| Image | `am1n3e/webarena-verified-map` | +| Container | `webarena-verified-map` | + +## Optimization + +Unlike other environments where the focus was on reducing image size, the Map environment optimization focuses on **usability**. The [original WebArena setup](https://github.com/web-arena-x/webarena/blob/main/webarena-map-backend-boot-init.yaml) required deploying 5 separate containers: + +- Tile server (`overv/openstreetmap-tile-server`) +- Nominatim geocoding server (`mediagis/nominatim:4.2`) +- 3 OSRM routing servers for car, bike, and foot + +Each container needed its own configuration, data volumes, and orchestration. The optimized image consolidates all services into a **single container** with unified management through env-ctrl, making deployment straightforward. + +## Quick Start + +Map requires external data files (~60GB) to be downloaded before starting. + +```bash +# 1. Download data and set up volumes +webarena-verified env setup init --site map --data-dir ./downloads + +# 2. Start the container +webarena-verified env start --site map +``` + +Access at: http://localhost:3000 + +## Architecture + +The map environment is a combined image with multiple services: + +| Service | Internal Port | Description | +|---------|---------------|-------------| +| Rails | 3000 | OpenStreetMap website | +| PostgreSQL 14 | 5432 | Website database | +| PostgreSQL 15 | 5433 | Tile database (gis) | +| Apache + mod_tile | 8080 | Tile server | +| renderd | - | Tile rendering daemon | +| OSRM (car) | 5000 | Car routing API | +| OSRM (bike) | 5001 | Bike routing API | +| OSRM (foot) | 5002 | Foot routing API | + +## Data Requirements + +Map requires external data files for tiles and routing: + +```bash +# Download and set up volumes +webarena-verified env setup init --site map --data-dir ./downloads +``` + +### Data Sizes + +| Data | Size | Description | +|------|------|-------------| +| Tile database | ~39 GB | PostgreSQL with pre-rendered tiles | +| Car routing | ~5.8 GB | OSRM routing graph | +| Bike routing | ~7.0 GB | OSRM routing graph | +| Foot routing | ~7.4 GB | OSRM routing graph | + +## Service Endpoints + +When running with full data: + +| Endpoint | URL | +|----------|-----| +| Website | http://localhost:3030/ | +| Tiles | http://localhost:8080/tile/{z}/{x}/{y}.png | +| Routing (car) | http://localhost:5000/route/v1/driving/{coords} | +| Routing (bike) | http://localhost:5001/route/v1/driving/{coords} | +| Routing (foot) | http://localhost:5002/route/v1/driving/{coords} | +| Routing (proxy) | http://localhost:8080/osrm/routed-{car,bike,foot}/route/v1/driving/{coords} | + +## Testing Routing + +```bash +# Test routing API (NYC to Brooklyn) +curl "http://localhost:5000/route/v1/driving/-74.006,40.7128;-73.9352,40.7306?overview=full&steps=true" +``` + +## Resource Requirements + +- **Disk:** Minimum 200GB for full tile and routing data +- **Memory:** 4GB+ recommended +- **Swap:** 4GB recommended diff --git a/docs/environments/reddit.md b/docs/environments/reddit.md new file mode 100644 index 0000000..6923029 --- /dev/null +++ b/docs/environments/reddit.md @@ -0,0 +1,77 @@ +# Reddit + +Postmill-based forum platform (Reddit clone). + +| Property | Value | +|----------|-------| +| Port | 9999 | +| Env-Ctrl Port | 9998 | +| Image | `am1n3e/webarena-verified-reddit` | +| Container | `webarena-verified-reddit` | + +## Quick Start + +```bash +# Using CLI (recommended) +webarena-verified env start --site reddit + +# Using Docker directly +docker run -d --name webarena-verified-reddit -p 9999:80 -p 9998:8877 am1n3e/webarena-verified-reddit +``` + +Access at: http://localhost:9999 + +## Auto-Login + +The optimized image supports HTTP header-based authentication, bypassing UI login. + +**Header:** `X-Postmill-Auto-Login: username:password` + +```python +from playwright.async_api import async_playwright + +async with async_playwright() as p: + browser = await p.chromium.launch() + context = await browser.new_context( + extra_http_headers={ + "X-Postmill-Auto-Login": "MarvelsGrantMan136:test1234" + } + ) + page = await context.new_page() + await page.goto("http://localhost:9999") + # You're now logged in +``` + +## Optimizations + +### Vote System Fix + +**Problem:** Original code recalculates `netScore` from the votes collection. The imported database has `net_score` values but no vote records, causing first vote to reset score to +/-1. + +**Fix:** Use increment/decrement instead of recalculating. New vote adds +/-1, vote change swings +/-2, retract subtracts. + +**Files:** `Submission.php`, `Comment.php`, `Votable.php`, `VoteManager.php` + +### Header Authentication + +**Problem:** UI login via Playwright is slow and requires maintaining test credentials. + +**Fix:** Custom Symfony authenticator accepts `X-Postmill-Auto-Login: username:password` header. + +**Files:** `HeaderAutologinAuthenticator.php`, `security.yaml` + +### URL Rewriting + +**Problem:** Symfony's `NoPrivateNetworkHttpClient` blocks requests to localhost/private IPs. Container can't fetch URLs referencing its own hostname. + +**Fix:** Custom HTTP client rewrites all external URLs to `http://localhost/` before making requests. + +**Files:** `UrlRewritingHttpClient.php`, `http_client.yaml` + +### Rate Limit Removal + +**Problem:** Postmill rate limits submissions, blocking automated testing. + +**Fix:** Removed `@RateLimit` annotations. + +**Files:** `SubmissionData.php` diff --git a/docs/environments/shopping.md b/docs/environments/shopping.md new file mode 100644 index 0000000..9ed4819 --- /dev/null +++ b/docs/environments/shopping.md @@ -0,0 +1,53 @@ +# Shopping + +Magento 2 storefront for e-commerce browsing and purchasing. + +| Property | Value | +|----------|-------| +| Port | 7770 | +| Env-Ctrl Port | 7771 | +| Image | `am1n3e/webarena-verified-shopping` | +| Container | `webarena-verified-shopping` | + +## Quick Start + +```bash +# Using CLI (recommended) +webarena-verified env start --site shopping + +# Using Docker directly +docker run -d --name webarena-verified-shopping -p 7770:80 -p 7771:8877 am1n3e/webarena-verified-shopping +``` + +Access at: http://localhost:7770 + +## Auto-Login + +The optimized image supports HTTP header-based authentication for customer accounts, bypassing UI login. + +**Header:** `X-M2-Customer-Auto-Login: email:password` + +```python +from playwright.async_api import async_playwright + +async with async_playwright() as p: + browser = await p.chromium.launch() + context = await browser.new_context( + extra_http_headers={ + "X-M2-Customer-Auto-Login": "emma.lopez@gmail.com:Password.123" + } + ) + page = await context.new_page() + await page.goto("http://localhost:7770") + # You're now logged in as a customer +``` + +## Optimizations + +### Header Authentication + +**Problem:** UI login via Playwright is slow and requires maintaining test credentials. + +**Fix:** Magento plugin intercepts frontend requests and authenticates customers via `X-M2-Customer-Auto-Login: email:password` header. + +**Files:** `CustomerAutoLogin/Plugin/CustomerAutoLoginPlugin.php`, `CustomerAutoLogin/etc/di.xml`, `CustomerAutoLogin/etc/module.xml`, `CustomerAutoLogin/registration.php` diff --git a/docs/environments/shopping_admin.md b/docs/environments/shopping_admin.md new file mode 100644 index 0000000..a0837f5 --- /dev/null +++ b/docs/environments/shopping_admin.md @@ -0,0 +1,77 @@ +# Shopping Admin + +Magento 2 admin panel for e-commerce management. + +| Property | Value | +|----------|-------| +| Port | 7780 | +| Env-Ctrl Port | 7781 | +| Image | `am1n3e/webarena-verified-shopping_admin` | +| Container | `webarena-verified-shopping_admin` | + +## Quick Start + +```bash +# Using CLI (recommended) +webarena-verified env start --site shopping_admin + +# Using Docker directly +docker run -d --name webarena-verified-shopping_admin -p 7780:80 -p 7781:8877 am1n3e/webarena-verified-shopping_admin +``` + +Access at: http://localhost:7780/admin + +## Auto-Login + +The optimized image supports HTTP header-based authentication, bypassing UI login. + +**Header:** `X-M2-Admin-Auto-Login: username:password` + +```python +from playwright.async_api import async_playwright + +async with async_playwright() as p: + browser = await p.chromium.launch() + context = await browser.new_context( + extra_http_headers={"X-M2-Admin-Auto-Login": "admin:admin1234"} + ) + page = await context.new_page() + await page.goto("http://localhost:7780/admin") + # You're now logged in as admin +``` + +### Testing Auto-Login + +```bash +# Should redirect to dashboard and set cookies +curl -I -H "X-M2-Admin-Auto-Login: admin:admin1234" \ + http://localhost:7780/admin + +# Without header - should redirect to login page +curl -I http://localhost:7780/admin +``` + +### Verify Module is Enabled + +```bash +docker exec webarena-verified-shopping_admin \ + /var/www/magento2/bin/magento module:status WebArena_AutoLogin +``` + +## Optimizations + +### Header Authentication + +**Problem:** UI login via Playwright is slow and requires maintaining test credentials. + +**Fix:** Magento plugin intercepts admin requests and authenticates via `X-M2-Admin-Auto-Login: username:password` header. + +**Files:** `AutoLoginPlugin.php`, `di.xml`, `module.xml`, `registration.php` + +### Mass Action Protection + +**Problem:** Bulk delete actions in admin panel can accidentally destroy test data. + +**Fix:** Plugins disable mass delete actions for products and reviews. + +**Files:** `DisableProductMassActionsPlugin.php`, `DisableReviewMassActionsPlugin.php` diff --git a/docs/environments/wikipedia.md b/docs/environments/wikipedia.md new file mode 100644 index 0000000..67ce2cf --- /dev/null +++ b/docs/environments/wikipedia.md @@ -0,0 +1,42 @@ +# Wikipedia + +MediaWiki-based encyclopedia with pre-populated content. + +| Property | Value | +|----------|-------| +| Port | 8888 | +| Env-Ctrl Port | 8889 | +| Image | `am1n3e/webarena-verified-wikipedia` | +| Container | `webarena-verified-wikipedia` | + +## Quick Start + +Wikipedia requires external data files (~100GB) to be downloaded before starting. + +```bash +# 1. Download data and set up volumes +webarena-verified env setup init --site wikipedia --data-dir ./downloads + +# 2. Start the container +webarena-verified env start --site wikipedia --data-dir ./downloads +``` + +Or using Docker directly: + +```bash +# Download ZIM file first, then run +docker run -d --name webarena-verified-wikipedia \ + -p 8888:8080 -p 8889:8874 \ + -v /path/to/downloads:/data:ro \ + am1n3e/webarena-verified-wikipedia +``` + +Access at: http://localhost:8888 + +## Optimizations + +The optimized image includes: + +- Environment control (env-ctrl) for runtime management +- Pre-configured MediaWiki settings +- Volume mounts for Wikipedia content database diff --git a/examples/tasks.py b/examples/tasks.py index 6f6f4f6..b043561 100644 --- a/examples/tasks.py +++ b/examples/tasks.py @@ -6,7 +6,7 @@ @task -def demo_gitlab_start(c, password="demopass", port="8012"): +def gitlab_start(c, password="demopass", port="8012"): """Start GitLab container for development and print root password when ready. Args: @@ -97,7 +97,7 @@ def demo_gitlab_start(c, password="demopass", port="8012"): @task -def demo_gitlab_stop(c): +def gitlab_stop(c): """Stop and remove the GitLab container.""" container_name = "wa-demo-gitlab" diff --git a/mkdocs.yml b/mkdocs.yml index a4eeca7..8a640ec 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -104,7 +104,13 @@ nav: - WebArena-Verified Hard: getting_started/hard_subset.md - Environments: - Overview: environments/index.md - - Docker Images: environments/docker_images.md + - Sites: + - Shopping Admin: environments/shopping_admin.md + - Shopping: environments/shopping.md + - Reddit: environments/reddit.md + - GitLab: environments/gitlab.md + - Wikipedia: environments/wikipedia.md + - Map: environments/map.md - Environment Control: environments/environment_control.md - Evaluation: - Quick Start: evaluation/index.md diff --git a/packages/environment_control/environment_control/env-ctrl b/packages/environment_control/environment_control/env-ctrl new file mode 100755 index 0000000..b9e1242 --- /dev/null +++ b/packages/environment_control/environment_control/env-ctrl @@ -0,0 +1,5 @@ +#!/bin/sh +# env-ctrl wrapper (for local development) +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +export PYTHONPATH="${SCRIPT_DIR%/*}" +exec python3 -m environment_control.cli "$@" diff --git a/pyproject.toml b/pyproject.toml index 602b8c6..6c01f1a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ include = ["src/**", "assets/**"] [dependency-groups] dev = [ "click>=8.3.0", + "docker-squash>=1.2.2", "environment-control", "pydantic-settings>=2.11.0", "litellm>=1.80.5", @@ -224,6 +225,12 @@ filterwarnings = [ ] markers = [ "docker: marks tests that require Docker", + "integration_docker_shopping: marks tests for shopping (storefront) container", + "integration_docker_shopping_admin: marks tests for shopping admin container", + "integration_docker_wikipedia: marks tests for wikipedia container", + "integration_docker_reddit: marks tests for reddit container", + "integration_docker_gitlab: marks tests for gitlab container", + "integration_docker_map: marks tests for map (OpenStreetMap) container", ] [tool.coverage.run] diff --git a/src/webarena_verified/__main__.py b/src/webarena_verified/__main__.py index 88ab4d3..27a8481 100644 --- a/src/webarena_verified/__main__.py +++ b/src/webarena_verified/__main__.py @@ -24,7 +24,9 @@ setup_webarena_verified_logging, ) from webarena_verified.core.utils.trim_network_logs import trim_har_file +from webarena_verified.environments.container import ContainerManager, ContainerStatus from webarena_verified.environments.env_ctrl_client import EnvCtrlClient +from webarena_verified.environments.setup import setup_clean, setup_init from webarena_verified.types.agent_response import MainObjectiveType from webarena_verified.types.config import WebArenaVerifiedConfig from webarena_verified.types.eval import EvalStatus, TaskEvalResult, TasksEvalResults, TransformedAgentResponse @@ -38,23 +40,44 @@ def _add_env_subcommand(subparsers: argparse._SubParsersAction) -> None: """Add the env subcommand group for Docker container control.""" + # Valid site choices (exclude homepage) + site_choices = [s.value for s in WebArenaSite if s != WebArenaSite.HOMEPAGE] + # Sites that require setup (have data files) + setup_site_choices = ["wikipedia", "map"] + env_parser = subparsers.add_parser( "env", help="Control Docker container environments", - description="Control and monitor Docker container environments via the environment control REST API", + description="Control and monitor Docker container environments", epilog=textwrap.dedent(""" examples: - # Check environment status - webarena-verified env status --url http://localhost:8877 + # Start a site container (auto-assign ports) + webarena-verified env start --site shopping + + # Start on specific ports + webarena-verified env start --site shopping --port 8080 --env-ctrl-port 9000 + + # Start wikipedia (requires --data-dir with downloaded ZIM file) + webarena-verified env setup init --site wikipedia --data-dir ./data + webarena-verified env start --site wikipedia --data-dir ./data + + # Check container status + webarena-verified env status --site shopping - # Start environment (wait for ready) + # Stop container + webarena-verified env stop --site shopping + + # Communicate with running container via HTTP + webarena-verified env status --url http://localhost:8877 webarena-verified env start --url http://localhost:8877 --wait - # Stop environment - webarena-verified env stop --url http://localhost:8877 + # Set up data (download files for sites that need them) + webarena-verified env setup init --site wikipedia --data-dir ./data + webarena-verified env setup init --site map --data-dir ./data - # Initialize environment - webarena-verified env init --url http://localhost:8877 + # Clean up volumes (for map site) + webarena-verified env setup clean --site map + webarena-verified env setup clean --force """), formatter_class=argparse.RawDescriptionHelpFormatter, ) @@ -62,25 +85,84 @@ def _add_env_subcommand(subparsers: argparse._SubParsersAction) -> None: # env status subcommand env_status_parser = env_subparsers.add_parser("status", help="Get environment status") - env_status_parser.add_argument("--url", type=str, required=True, help="Environment control server URL") + env_status_group = env_status_parser.add_mutually_exclusive_group(required=True) + env_status_group.add_argument("--site", type=str, choices=site_choices, help="Site name (manages Docker container)") + env_status_group.add_argument("--url", type=str, help="Environment control server URL (HTTP client)") env_status_parser.add_argument("--timeout", type=int, default=30, help="Request timeout in seconds") # env start subcommand env_start_parser = env_subparsers.add_parser("start", help="Start the environment") - env_start_parser.add_argument("--url", type=str, required=True, help="Environment control server URL") - env_start_parser.add_argument("--wait", action="store_true", help="Wait until environment is ready") - env_start_parser.add_argument("--timeout", type=int, default=30, help="Request timeout in seconds") + env_start_group = env_start_parser.add_mutually_exclusive_group(required=True) + env_start_group.add_argument("--site", type=str, choices=site_choices, help="Site name (manages Docker container)") + env_start_group.add_argument("--url", type=str, help="Environment control server URL (HTTP client)") + env_start_parser.add_argument("--port", type=int, help="Host port for site (default: container's default port)") + env_start_parser.add_argument( + "--env-ctrl-port", type=int, help="Host port for env-ctrl (default: container's default port)" + ) + env_start_parser.add_argument( + "--no-wait", dest="wait", action="store_false", help="Don't wait for environment to be ready" + ) + env_start_parser.set_defaults(wait=True) + env_start_parser.add_argument("--timeout", type=int, default=120, help="Wait timeout in seconds (default: 120)") + env_start_parser.add_argument("--data-dir", type=str, help="Data directory to bind-mount (required for wikipedia)") # env stop subcommand env_stop_parser = env_subparsers.add_parser("stop", help="Stop the environment") - env_stop_parser.add_argument("--url", type=str, required=True, help="Environment control server URL") + env_stop_group = env_stop_parser.add_mutually_exclusive_group(required=True) + env_stop_group.add_argument("--site", type=str, choices=site_choices, help="Site name (manages Docker container)") + env_stop_group.add_argument("--url", type=str, help="Environment control server URL (HTTP client)") env_stop_parser.add_argument("--timeout", type=int, default=30, help="Request timeout in seconds") - # env init subcommand - env_init_parser = env_subparsers.add_parser("init", help="Initialize the environment") + # env stop-all subcommand + env_subparsers.add_parser("stop-all", help="Stop all running webarena containers") + + # env init subcommand (HTTP client only) + env_init_parser = env_subparsers.add_parser("init", help="Initialize the environment (HTTP client)") env_init_parser.add_argument("--url", type=str, required=True, help="Environment control server URL") env_init_parser.add_argument("--timeout", type=int, default=30, help="Request timeout in seconds") + # env setup subcommand group + env_setup_parser = env_subparsers.add_parser( + "setup", + help="Manage Docker data volumes for sites that require setup", + description="Download data files and set up Docker volumes for sites (wikipedia, map)", + ) + setup_subparsers = env_setup_parser.add_subparsers(dest="setup_command", required=True) + + # env setup init + setup_init_parser = setup_subparsers.add_parser("init", help="Initialize volumes (download data, create volumes)") + setup_init_parser.add_argument( + "--site", + type=str, + choices=setup_site_choices, + help="Site to set up (default: all sites)", + ) + setup_init_parser.add_argument( + "--data-dir", + type=str, + required=True, + help="Directory to store downloaded data files", + ) + setup_init_parser.add_argument( + "--dry-run", + action="store_true", + help="Show what would be done without making changes", + ) + + # env setup clean + setup_clean_parser = setup_subparsers.add_parser("clean", help="Remove volumes") + setup_clean_parser.add_argument( + "--site", + type=str, + choices=setup_site_choices, + help="Site to clean (default: all sites)", + ) + setup_clean_parser.add_argument( + "--force", + action="store_true", + help="Skip confirmation prompt", + ) + def create_parser() -> argparse.ArgumentParser: """Create the argument parser for the CLI.""" @@ -1214,7 +1296,106 @@ def trim_network_logs(args: argparse.Namespace) -> int: def env_command(args: argparse.Namespace) -> int | None: - """Execute env subcommands (status, start, stop, init).""" + """Execute env subcommands (status, start, stop, stop-all, init, setup).""" + # Handle setup subcommand + if args.env_command == "setup": + return _env_setup_command(args) + + # Handle stop-all subcommand + if args.env_command == "stop-all": + return _env_stop_all_command() + + # Handle --site mode (Docker container management) + if hasattr(args, "site") and args.site: + return _env_container_command(args) + + # Handle --url mode (HTTP client) + return _env_http_command(args) + + +def _env_container_command(args: argparse.Namespace) -> int: # noqa: PLR0911 + """Execute env subcommands using Docker container management.""" + site = WebArenaSite(args.site) + manager = ContainerManager(site=site) + + try: + if args.env_command == "status": + result = manager.status() + logger.info(f"Container: {result.container_name}") + logger.info(f"Status: {result.status.value}") + if result.url: + logger.info(f"URL: {result.url}") + if result.env_ctrl_url: + logger.info(f"Env-ctrl: {result.env_ctrl_url}") + if result.env_ctrl_status: + logger.info(f"Services: {json.dumps(result.env_ctrl_status, indent=2)}") + return 0 if result.status == ContainerStatus.RUNNING else 1 + + if args.env_command == "start": + data_dir = Path(args.data_dir) if getattr(args, "data_dir", None) else None + port = getattr(args, "port", None) or manager.config.host_port + env_ctrl_port = getattr(args, "env_ctrl_port", None) or manager.config.host_env_ctrl_port + if not port: + logger.error(f"Site {args.site} requires --port to be specified") + return 1 + if not env_ctrl_port: + logger.error(f"Site {args.site} requires --env-ctrl-port to be specified") + return 1 + result = manager.start( + port=port, + env_ctrl_port=env_ctrl_port, + wait=args.wait, + timeout=args.timeout, + data_dir=data_dir, + ) + logger.info(f"Container started: {result.container_name}") + logger.info(f" URL: {result.url}") + logger.info(f" Env-ctrl: {result.env_ctrl_url}") + return 0 + + if args.env_command == "stop": + manager.stop() + logger.info(f"Container stopped: {manager.container_name}") + return 0 + + logger.error(f"Unknown env command: {args.env_command}") + return 1 + + except ValueError as e: + logger.error(str(e)) + return 1 + except RuntimeError as e: + logger.error(f"Container error: {e}") + return 1 + + +def _env_stop_all_command() -> int: + """Stop all running webarena containers.""" + stopped = [] + not_running = [] + + # Iterate over all sites (except HOMEPAGE) + for site in WebArenaSite: + if site == WebArenaSite.HOMEPAGE: + continue + + manager = ContainerManager(site=site) + if manager.is_running(): + manager.stop() + stopped.append(site.value) + else: + not_running.append(site.value) + + if stopped: + logger.info(f"Stopped containers: {', '.join(stopped)}") + else: + logger.info("No running containers found") + + return 0 + + +def _env_http_command(args: argparse.Namespace) -> int: + """Execute env subcommands using HTTP client.""" client = EnvCtrlClient(base_url=args.url, timeout=args.timeout) try: @@ -1243,6 +1424,34 @@ def env_command(args: argparse.Namespace) -> int | None: return 1 +def _env_setup_command(args: argparse.Namespace) -> int: + """Execute env setup subcommands (init, clean).""" + try: + site = WebArenaSite(args.site) if args.site else None + + if args.setup_command == "init": + data_dir = Path(args.data_dir) + setup_init(site=site, data_dir=data_dir, dry_run=args.dry_run) + return 0 + + if args.setup_command == "clean": + setup_clean(site=site, force=args.force) + return 0 + + logger.error(f"Unknown setup command: {args.setup_command}") + return 1 + + except ValueError as e: + logger.error(f"Setup error: {e}") + return 1 + except RuntimeError as e: + logger.error(f"Setup error: {e}") + return 1 + except SystemExit: + # User aborted + return 0 + + def create_submission_pkg(args: argparse.Namespace) -> int: """Execute create-submission-pkg command""" # Display command info diff --git a/src/webarena_verified/api/internal/__init__.py b/src/webarena_verified/api/internal/__init__.py index afe1a40..0912409 100644 --- a/src/webarena_verified/api/internal/__init__.py +++ b/src/webarena_verified/api/internal/__init__.py @@ -2,11 +2,9 @@ from .data_reader import WebArenaVerifiedDataReader from .evaluator import WebArenaVerifiedEvaluator -from .patch_manager import PatchManager from .subsets_manager import SubsetsManager __all__ = [ - "PatchManager", "SubsetsManager", "WebArenaVerifiedDataReader", "WebArenaVerifiedEvaluator", diff --git a/src/webarena_verified/api/internal/patch_manager.py b/src/webarena_verified/api/internal/patch_manager.py deleted file mode 100644 index 6f0cb77..0000000 --- a/src/webarena_verified/api/internal/patch_manager.py +++ /dev/null @@ -1,154 +0,0 @@ -"""Patch manager for WebArena environment patches. - -This module provides functionality to discover, manage, and apply patches to -WebArena environment containers. Patches are organized by site (reddit, shopping, -shopping_admin, etc.) and applied in order based on filename prefix. -""" - -import importlib.util -import logging -from collections.abc import Callable -from pathlib import Path - -logger = logging.getLogger(__name__) - - -class PatchManager: - """Manager for discovering and applying environment patches. - - Patches are organized in the following structure: - patches/ - reddit/ - p01_remove_rate_limits.py - p02_configure_http_client.py - shopping/ - p01_contact_form.py - shopping_admin/ - p01_autologin_session.py - - Patch files must follow the naming pattern p_*.py (e.g., p01_example.py). - Each patch module must expose an `apply_patch(handler) -> bool` function that receives - a SiteInstanceHandler for executing commands on the target environment. - """ - - def __init__(self, exec_patch: Callable[..., bool], patches_dir: Path | None = None) -> None: - """Initialize the patch manager. - - Args: - exec_patch: Function to execute patches. Should have signature: - exec_patch(patch_fn: Callable, site: str, **kwargs) -> bool - The executor is responsible for Docker operations and calling patch_fn - patches_dir: Path to the patches directory. If None, uses the default - patches directory relative to this module. - """ - if patches_dir is None: - # Default to patches directory in environments - patches_dir = Path(__file__).parent.parent.parent / "environments" / "patches" - - self.exec_patch = exec_patch - self.patches_dir = patches_dir - logger.debug(f"Initialized PatchManager with patches directory: {patches_dir}") - - def discover_patches(self, site: str) -> list[tuple[str, Callable[[str], bool]]]: - """Discover all patches for a given site. - - Args: - site: Site identifier (e.g., "reddit", "shopping") - - Returns: - List of (patch_name, apply_patch_function) tuples, sorted by filename - - Raises: - ValueError: If a patch file matching p_*.py pattern is invalid - (missing apply_patch function or apply_patch is not callable) - """ - site_dir = self.patches_dir / site - if not site_dir.exists() or not site_dir.is_dir(): - logger.warning(f"No patches directory found for site '{site}' at {site_dir}") - return [] - - patches = [] - - # Find all Python files matching p_*.py pattern - for patch_file in sorted(site_dir.glob("p[0-9]*.py")): - if patch_file.name.startswith("__"): - continue - - # Load the module - try: - spec = importlib.util.spec_from_file_location( - f"webarena_verified.environments.patches.{site}.{patch_file.stem}", - patch_file, - ) - if spec is None or spec.loader is None: - logger.warning(f"Could not load spec for patch: {patch_file}") - continue - - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - - # Get the apply_patch function - if not hasattr(module, "apply_patch"): - raise ValueError( - f"Patch {patch_file.name} does not have an 'apply_patch' function. " - f"All patches matching p_*.py must expose an apply_patch(handler) -> bool function." - ) - - apply_patch = module.apply_patch - if not callable(apply_patch): - raise ValueError( - f"Patch {patch_file.name} has 'apply_patch' attribute but it's not callable. " - f"The apply_patch must be a function with signature: apply_patch(handler) -> bool" - ) - - patches.append((patch_file.stem, apply_patch)) - logger.debug(f"Discovered patch: {site}/{patch_file.name}") - - except Exception as e: - logger.error(f"Error loading patch {patch_file.name}: {e}") - continue - - logger.info(f"Discovered {len(patches)} patch(es) for site '{site}'") - return patches - - def apply_patches_for_site(self, site: str) -> bool: - """Apply all patches for a given site. - - Args: - site: Site identifier (e.g., "reddit", "shopping") - - Returns: - True if all patches applied successfully, False otherwise - - Note: - Patches are applied in alphabetical order by filename. Use numeric prefixes - (e.g., 01_, 02_) to control ordering. - - If any patch fails, the process stops immediately and returns False. - - The exec_patch function provided during initialization is responsible for - Docker operations (container discovery, docker exec, etc.) and calling the patch function. - """ - # Discover patches - patches = self.discover_patches(site) - if not patches: - logger.info(f"No patches to apply for site '{site}'") - return True - - logger.info(f"Applying {len(patches)} patch(es) for site '{site}'") - - # Apply patches in order using the injected exec_patch function - for patch_name, apply_patch in patches: - logger.info(f"Applying patch: {site}/{patch_name}") - try: - success = self.exec_patch(apply_patch, site, patch_name=patch_name) - if not success: - logger.error(f"Patch {site}/{patch_name} failed to apply") - return False - logger.info(f"Successfully applied patch: {site}/{patch_name}") - except Exception as e: - logger.error(f"Error applying patch {site}/{patch_name}: {e}") - return False - - logger.info(f"All patches applied successfully for site '{site}'") - return True diff --git a/src/webarena_verified/api/webarena_verified.py b/src/webarena_verified/api/webarena_verified.py index d950339..29abc22 100644 --- a/src/webarena_verified/api/webarena_verified.py +++ b/src/webarena_verified/api/webarena_verified.py @@ -14,7 +14,6 @@ from .internal.data_reader import WebArenaVerifiedDataReader from .internal.evaluator import WebArenaVerifiedEvaluator -from .internal.patch_manager import PatchManager from .internal.submission_handler import SubmissionHandler if TYPE_CHECKING: @@ -240,49 +239,6 @@ def get_custom_auth_header_name(self, site: str | WebArenaSite) -> str | None: return MAGENTO_ADMIN_AUTO_LOGIN_HEADER return None - def apply_patches_for_site(self, site: str, exec_patch: Callable[[Callable, str, Any], bool]) -> bool: - """Apply all patches for a given site using the provided executor. - - This method discovers and applies all patches for the specified site in order. - Patches are organized by site (reddit, shopping, shopping_admin, etc.) and - applied based on filename prefix (p01_, p02_, etc.). - - Args: - site: Site identifier to apply patches for. Valid values: - - "reddit": Reddit environment patches - - "shopping": Shopping/Magento environment patches - - "shopping_admin": Shopping admin environment patches - exec_patch: Function to execute patches with Docker operations. - Should have signature: exec_patch(patch_fn, site, **kwargs) -> bool - The executor is responsible for: - - Resolving the Docker container name for the site - - Calling patch_fn(container_name) - - Handling errors and returning success/failure - - Returns: - True if all patches applied successfully, False if any patch failed - - Example: - ```python - from webarena_verified.api import WebArenaVerified - - def my_exec_patch(patch_fn, site: str, **kwargs) -> bool: - try: - container_name = get_container_name(site) - return patch_fn(container_name) - except Exception as e: - logger.error(f"Failed to execute patch: {e}") - return False - - wa = WebArenaVerified() - success = wa.apply_patches_for_site("reddit", my_exec_patch) - if success: - print("All Reddit patches applied successfully") - ``` - """ - patch_manager = PatchManager(exec_patch) - return patch_manager.apply_patches_for_site(site) - def create_submission( self, output_dirs: list[Path], diff --git a/src/webarena_verified/environments/__init__.py b/src/webarena_verified/environments/__init__.py index ce2a1a1..8f4a142 100644 --- a/src/webarena_verified/environments/__init__.py +++ b/src/webarena_verified/environments/__init__.py @@ -1,7 +1,14 @@ -"""WebArena environment management utilities and patches.""" +"""WebArena environment management utilities.""" from .site_handler import SiteInstanceHandler MAGENTO_ADMIN_AUTO_LOGIN_HEADER = "X-M2-Admin-Auto-Login" -__all__ = ["MAGENTO_ADMIN_AUTO_LOGIN_HEADER", "SiteInstanceHandler"] +__all__ = [ + "MAGENTO_ADMIN_AUTO_LOGIN_HEADER", + "SiteInstanceHandler", + # Re-export subpackages for convenient access + "container", + "env_ctrl_client", + "setup", +] diff --git a/src/webarena_verified/environments/container/__init__.py b/src/webarena_verified/environments/container/__init__.py new file mode 100644 index 0000000..6396b38 --- /dev/null +++ b/src/webarena_verified/environments/container/__init__.py @@ -0,0 +1,24 @@ +"""Container management for WebArena Docker environments. + +This package provides utilities for managing Docker containers for WebArena sites, +including starting, stopping, and checking status of containers. +""" + +from webarena_verified.types.container import ContainerStartResult, ContainerStatus, ContainerStatusResult + +from .backend import ContainerBackend, DockerBackend, get_default_backend +from .config import DEFAULT_CONTAINER_CONFIGS, get_container_config, get_sites_with_setup +from .manager import ContainerManager + +__all__ = [ + "DEFAULT_CONTAINER_CONFIGS", + "ContainerBackend", + "ContainerManager", + "ContainerStartResult", + "ContainerStatus", + "ContainerStatusResult", + "DockerBackend", + "get_container_config", + "get_default_backend", + "get_sites_with_setup", +] diff --git a/src/webarena_verified/environments/container/backend/__init__.py b/src/webarena_verified/environments/container/backend/__init__.py new file mode 100644 index 0000000..fe38747 --- /dev/null +++ b/src/webarena_verified/environments/container/backend/__init__.py @@ -0,0 +1,23 @@ +"""Container backend abstraction for WebArena environments. + +This package provides a Protocol for container backends and a Docker implementation. +This allows for future support of other container runtimes (e.g., Podman). +""" + +from .docker import DockerBackend +from .protocol import ContainerBackend + +# Default backend instance +_default_backend: ContainerBackend = DockerBackend() + + +def get_default_backend() -> ContainerBackend: + """Get the default container backend (Docker).""" + return _default_backend + + +__all__ = [ + "ContainerBackend", + "DockerBackend", + "get_default_backend", +] diff --git a/src/webarena_verified/environments/container/backend/docker.py b/src/webarena_verified/environments/container/backend/docker.py new file mode 100644 index 0000000..1ff44fc --- /dev/null +++ b/src/webarena_verified/environments/container/backend/docker.py @@ -0,0 +1,107 @@ +"""Docker backend implementation. + +Uses the docker CLI to manage containers. +""" + +from __future__ import annotations + +import socket +import subprocess + + +class DockerBackend: + """Docker implementation of ContainerBackend. + + Uses the docker CLI to manage containers. + """ + + def container_exists(self, *, name: str) -> bool: + """Check if a Docker container exists (running or stopped).""" + result = subprocess.run( + ["docker", "ps", "-a", "--filter", f"name=^{name}$", "--format", "{{.Names}}"], + capture_output=True, + text=True, + ) + return result.stdout.strip() == name + + def container_running(self, *, name: str) -> bool: + """Check if a Docker container is currently running.""" + result = subprocess.run( + ["docker", "ps", "--filter", f"name=^{name}$", "--format", "{{.Names}}"], + capture_output=True, + text=True, + ) + return result.stdout.strip() == name + + def container_remove(self, *, name: str) -> None: + """Remove a Docker container if it exists.""" + if self.container_exists(name=name): + subprocess.run( + ["docker", "rm", "-f", name], + capture_output=True, + check=True, + ) + + def container_run( + self, + *, + name: str, + image: str, + port_mappings: dict[int, int], + volume_mappings: dict[str, str], + env_vars: dict[str, str] | None = None, + ) -> None: + """Run a Docker container with the given configuration.""" + cmd = [ + "docker", + "run", + "-d", + "--name", + name, + ] + + # Add port mappings (host_port:container_port) + for host_port, container_port in port_mappings.items(): + cmd.extend(["-p", f"{host_port}:{container_port}"]) + + # Add volume mappings (volume_name:container_path) + for volume_name, container_path in volume_mappings.items(): + cmd.extend(["-v", f"{volume_name}:{container_path}"]) + + # Add environment variables + if env_vars: + for key, value in env_vars.items(): + cmd.extend(["-e", f"{key}={value}"]) + + cmd.append(image) + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"Failed to start container: {result.stderr}") + + def get_container_ports(self, *, name: str) -> dict[int, int]: + """Get the published port mappings for a running Docker container.""" + result = subprocess.run( + ["docker", "port", name], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return {} + + ports: dict[int, int] = {} + for line in result.stdout.strip().split("\n"): + if "->" in line: + # Format: "80/tcp -> 0.0.0.0:8080" + container_part, host_mapping = line.split(" -> ") + container_port = int(container_part.split("/")[0]) + host_port = int(host_mapping.rsplit(":", 1)[1]) + ports[container_port] = host_port + return ports + + def find_free_port(self) -> int: + """Find and return an available TCP port.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind(("", 0)) + return s.getsockname()[1] diff --git a/src/webarena_verified/environments/container/backend/protocol.py b/src/webarena_verified/environments/container/backend/protocol.py new file mode 100644 index 0000000..92ae86f --- /dev/null +++ b/src/webarena_verified/environments/container/backend/protocol.py @@ -0,0 +1,90 @@ +"""Container backend protocol definition. + +Defines the interface for container operations that can be implemented +by different container runtimes (Docker, Podman, etc.). +""" + +from __future__ import annotations + +from typing import Protocol + + +class ContainerBackend(Protocol): + """Protocol for container backend implementations. + + Defines the interface for container operations that can be implemented + by different container runtimes (Docker, Podman, etc.). + """ + + def container_exists(self, *, name: str) -> bool: + """Check if a container exists (running or stopped). + + Args: + name: Container name to check. + + Returns: + True if container exists, False otherwise. + """ + ... + + def container_running(self, *, name: str) -> bool: + """Check if a container is currently running. + + Args: + name: Container name to check. + + Returns: + True if container is running, False otherwise. + """ + ... + + def container_remove(self, *, name: str) -> None: + """Remove a container if it exists. + + Args: + name: Container name to remove. + """ + ... + + def container_run( + self, + *, + name: str, + image: str, + port_mappings: dict[int, int], + volume_mappings: dict[str, str], + env_vars: dict[str, str] | None = None, + ) -> None: + """Run a container with the given configuration. + + Args: + name: Container name. + image: Docker image to run. + port_mappings: Host port -> container port mappings. + volume_mappings: Volume name -> container path mappings. + env_vars: Environment variables to set in the container. + + Raises: + RuntimeError: If container fails to start. + """ + ... + + def get_container_ports(self, *, name: str) -> dict[int, int]: + """Get the published port mappings for a running container. + + Args: + name: Container name. + + Returns: + Dict mapping container port to host port. + Returns empty dict if container not found or not running. + """ + ... + + def find_free_port(self) -> int: + """Find and return an available TCP port. + + Returns: + An available port number. + """ + ... diff --git a/src/webarena_verified/environments/container/config.py b/src/webarena_verified/environments/container/config.py new file mode 100644 index 0000000..bbbee79 --- /dev/null +++ b/src/webarena_verified/environments/container/config.py @@ -0,0 +1,169 @@ +"""Default container configurations for WebArena sites. + +This module provides default Docker container configurations for each WebArena site. +These defaults can be overridden by user configurations in config.json. +""" + +from __future__ import annotations + +from webarena_verified.types.config import ContainerConfig, ContainerSetupConfig, ContainerVolumeSpec +from webarena_verified.types.task import WebArenaSite + +# Default container configurations for all sites +DEFAULT_CONTAINER_CONFIGS: dict[WebArenaSite, ContainerConfig] = { + WebArenaSite.SHOPPING: ContainerConfig( + docker_img="am1n3e/webarena-verified-shopping", + container_port=80, + env_ctrl_port=8877, + host_port=7770, + host_env_ctrl_port=7771, + health_check_path="/customer/account/login", + ), + WebArenaSite.SHOPPING_ADMIN: ContainerConfig( + docker_img="am1n3e/webarena-verified-shopping_admin", + container_port=80, + env_ctrl_port=8877, + host_port=7780, + host_env_ctrl_port=7781, + health_check_path="/", + ), + WebArenaSite.REDDIT: ContainerConfig( + docker_img="am1n3e/webarena-verified-reddit", + container_port=80, + env_ctrl_port=8877, + host_port=9999, + host_env_ctrl_port=9998, + health_check_path="/login", + ), + WebArenaSite.GITLAB: ContainerConfig( + docker_img="am1n3e/webarena-verified-gitlab", + container_port=8023, # GitLab uses non-standard port + env_ctrl_port=8877, + host_port=8023, + host_env_ctrl_port=8024, + health_check_path="/users/sign_in", + ), + WebArenaSite.WIKIPEDIA: ContainerConfig( + docker_img="am1n3e/webarena-verified-wikipedia", + container_port=8080, + env_ctrl_port=8874, + host_port=8888, + host_env_ctrl_port=8889, + health_check_path="/", + data_dir_mount="/data", + setup=ContainerSetupConfig( + data_urls=("http://metis.lti.cs.cmu.edu/webarena-images/wikipedia_en_all_maxi_2022-05.zim",), + volumes=(), + ), + ), + WebArenaSite.MAP: ContainerConfig( + docker_img="am1n3e/webarena-verified-map", + container_port=8080, + env_ctrl_port=8877, + host_port=3000, + host_env_ctrl_port=3001, + health_check_path="/", + setup=ContainerSetupConfig( + data_urls=( + "https://webarena-map-server-data.s3.amazonaws.com/osm_tile_server.tar", + "https://webarena-map-server-data.s3.amazonaws.com/nominatim_volumes.tar", + "https://webarena-map-server-data.s3.amazonaws.com/osrm_routing.tar", + ), + volumes=( + ContainerVolumeSpec( + volume_name="webarena_verified_map_tile_db", + mount_path="/data/database", + source_tar="osm_tile_server.tar", + tar_extract_path="projects/ogma3/docker/volumes/osm-data/_data", + strip_components=6, + ), + ContainerVolumeSpec( + volume_name="webarena_verified_map_routing_car", + mount_path="/data/routing/car", + source_tar="osrm_routing.tar", + tar_extract_path="car", + strip_components=1, + ), + ContainerVolumeSpec( + volume_name="webarena_verified_map_routing_bike", + mount_path="/data/routing/bike", + source_tar="osrm_routing.tar", + tar_extract_path="bike", + strip_components=1, + ), + ContainerVolumeSpec( + volume_name="webarena_verified_map_routing_foot", + mount_path="/data/routing/foot", + source_tar="osrm_routing.tar", + tar_extract_path="foot", + strip_components=1, + ), + ContainerVolumeSpec( + volume_name="webarena_verified_map_nominatim_db", + mount_path="/data/nominatim/postgres", + source_tar="nominatim_volumes.tar", + tar_extract_path="projects/metis2/docker/docker/volumes/nominatim-data/_data", + strip_components=7, + ), + ContainerVolumeSpec( + volume_name="webarena_verified_map_nominatim_flatnode", + mount_path="/data/nominatim/flatnode", + source_tar="nominatim_volumes.tar", + tar_extract_path="projects/metis2/docker/docker/volumes/nominatim-flatnode/_data", + strip_components=7, + ), + ContainerVolumeSpec( + volume_name="webarena_verified_map_website_db", + mount_path="/var/lib/postgresql/14/main", + ), + ContainerVolumeSpec( + volume_name="webarena_verified_map_tiles", + mount_path="/data/tiles", + ), + ContainerVolumeSpec( + volume_name="webarena_verified_map_style", + mount_path="/data/style", + ), + ), + ), + ), +} + + +def get_container_config( + *, + site: WebArenaSite, + user_config: ContainerConfig | None = None, +) -> ContainerConfig: + """Get container config, using user override if provided. + + Args: + site: WebArena site to get config for. + user_config: Optional user-provided container config override. + + Returns: + ContainerConfig for the site. + + Raises: + ValueError: If site is not supported (HOMEPAGE) and no user config provided. + """ + if user_config is not None: + return user_config + + if site not in DEFAULT_CONTAINER_CONFIGS: + raise ValueError(f"No default container config for site {site.value}. Site may not support Docker deployment.") + + return DEFAULT_CONTAINER_CONFIGS[site] + + +def get_sites_with_setup() -> list[WebArenaSite]: + """Get list of sites that require setup (have data files to download). + + Returns: + List of WebArenaSite values that have setup configuration. + """ + return [ + site + for site, config in DEFAULT_CONTAINER_CONFIGS.items() + if config.setup is not None and config.setup.data_urls + ] diff --git a/src/webarena_verified/environments/container/manager.py b/src/webarena_verified/environments/container/manager.py new file mode 100644 index 0000000..4be74a2 --- /dev/null +++ b/src/webarena_verified/environments/container/manager.py @@ -0,0 +1,263 @@ +"""Container manager for WebArena Docker containers. + +This module provides the ContainerManager class for starting, stopping, +and managing Docker containers for WebArena sites. +""" + +from __future__ import annotations + +import time +import urllib.error +import urllib.request +from typing import TYPE_CHECKING + +from webarena_verified.core.utils import logger +from webarena_verified.environments.env_ctrl_client import EnvCtrlDockerClient, HttpClient +from webarena_verified.types.container import ContainerStartResult, ContainerStatus, ContainerStatusResult + +from .backend import ContainerBackend, get_default_backend +from .config import get_container_config + +if TYPE_CHECKING: + from pathlib import Path + + from webarena_verified.types.config import ContainerConfig + from webarena_verified.types.task import WebArenaSite + + +class ContainerManager: + """Manages Docker containers for WebArena sites. + + Provides methods to start, stop, and check status of containers + for individual WebArena sites. + + Args: + site: WebArena site to manage. + config: Optional container configuration override. + If None, uses the default config for the site. + backend: Optional container backend. If None, uses Docker. + hostname: Hostname for constructing URLs (default: "localhost"). + + Example: + >>> manager = ContainerManager(site=WebArenaSite.SHOPPING) + >>> result = manager.start(port=8080) + >>> print(f"Site running at: {result.url}") + >>> manager.stop() + """ + + def __init__( + self, + *, + site: WebArenaSite, + config: ContainerConfig | None = None, + backend: ContainerBackend | None = None, + hostname: str = "localhost", + ) -> None: + self.site = site + self.config = get_container_config(site=site, user_config=config) + self.container_name = f"webarena_verified_{site.value}" + self.backend = backend or get_default_backend() + self.hostname = hostname + + def start( + self, + *, + port: int, + env_ctrl_port: int, + wait: bool = True, + timeout: int = 120, + data_dir: Path | None = None, + ) -> ContainerStartResult: + """Start the container and optionally wait for services. + + If a container with the same name already exists, it will be removed + before starting a new one. + + Args: + port: Host port for the site. + env_ctrl_port: Host port for env-ctrl API. + wait: If True, wait for services to be ready before returning. + timeout: Timeout in seconds for waiting (only used with wait=True). + data_dir: Path to data directory for bind-mount (required for sites with data_dir_mount). + + Returns: + ContainerStartResult with URLs and port information. + + Raises: + RuntimeError: If container fails to start or services don't become ready. + ValueError: If data_dir is required but not provided. + """ + # Validate data_dir requirement + if self.config.data_dir_mount and not data_dir: + raise ValueError( + f"Site {self.site.value} requires --data-dir to be specified. " + f"Data will be bind-mounted to {self.config.data_dir_mount}" + ) + + # Remove existing container if present + logger.info(f"Removing existing container if present: {self.container_name}") + self.backend.container_remove(name=self.container_name) + + host_port = port + host_env_ctrl_port = env_ctrl_port + + logger.info(f"Using ports - site: {host_port}, env-ctrl: {host_env_ctrl_port}") + + # Build port and volume mappings + port_mappings = { + host_port: self.config.container_port, + host_env_ctrl_port: self.config.env_ctrl_port, + } + + # Build volume mappings: use bind-mount if data_dir_mount is set, otherwise use named volumes + if self.config.data_dir_mount and data_dir: + volume_mappings = {str(data_dir.resolve()): self.config.data_dir_mount} + logger.info(f"Bind-mounting data directory: {data_dir} -> {self.config.data_dir_mount}") + else: + volume_mappings = self.config.volumes + + # Set env vars for env-ctrl init (runs automatically in entrypoint) + url = f"http://{self.hostname}:{host_port}" + env_vars = {"WA_ENV_CTRL_EXTERNAL_SITE_URL": url} + + # Run container + logger.info(f"Starting container: {self.container_name} (image: {self.config.docker_img})") + self.backend.container_run( + name=self.container_name, + image=self.config.docker_img, + port_mappings=port_mappings, + volume_mappings=volume_mappings, + env_vars=env_vars, + ) + logger.info("Container started successfully") + + env_ctrl_url = f"http://{self.hostname}:{host_env_ctrl_port}" + + # Wait for services if requested + if wait: + logger.info(f"Waiting for services to be ready (timeout: {timeout}s)") + health_check_url = f"{url}{self.config.health_check_path}" + self._wait_for_ready(timeout=timeout, health_check_url=health_check_url) + + return ContainerStartResult( + container_name=self.container_name, + url=url, + env_ctrl_url=env_ctrl_url, + host_port=host_port, + env_ctrl_host_port=host_env_ctrl_port, + ) + + def stop(self) -> None: + """Stop and remove the container. + + Does nothing if container doesn't exist. + """ + logger.info(f"Stopping container: {self.container_name}") + self.backend.container_remove(name=self.container_name) + logger.info("Container stopped") + + def status(self) -> ContainerStatusResult: + """Get container status and URLs if running. + + First checks Docker state, then queries env-ctrl if running. + + Returns: + ContainerStatusResult with current status and URLs (if running). + """ + logger.info(f"Checking status of container: {self.container_name}") + # Check Docker state first + if not self.backend.container_exists(name=self.container_name): + return ContainerStatusResult( + container_name=self.container_name, + status=ContainerStatus.NOT_FOUND, + ) + + if not self.backend.container_running(name=self.container_name): + return ContainerStatusResult( + container_name=self.container_name, + status=ContainerStatus.STOPPED, + ) + + # Get port mappings + ports = self.backend.get_container_ports(name=self.container_name) + + url = None + env_ctrl_url = None + env_ctrl_status = None + + # Look for the web service port + if self.config.container_port in ports: + host_port = ports[self.config.container_port] + url = f"http://{self.hostname}:{host_port}" + + # Look for env-ctrl port and query status + if self.config.env_ctrl_port in ports: + host_env_ctrl_port = ports[self.config.env_ctrl_port] + env_ctrl_url = f"http://{self.hostname}:{host_env_ctrl_port}" + + # Query env-ctrl for detailed status + try: + client = HttpClient(base_url=env_ctrl_url, timeout=10) + result = client.status() + if result.get("success"): + env_ctrl_status = result + except Exception: + # env-ctrl not responding, but container is running + pass + + return ContainerStatusResult( + container_name=self.container_name, + status=ContainerStatus.RUNNING, + url=url, + env_ctrl_url=env_ctrl_url, + env_ctrl_status=env_ctrl_status, + ) + + def is_running(self) -> bool: + """Check if container is currently running. + + Returns: + True if container is running, False otherwise. + """ + return self.backend.container_running(name=self.container_name) + + def _wait_for_ready(self, *, timeout: int, health_check_url: str) -> None: + """Wait for services to become ready. + + Args: + timeout: Timeout in seconds. + health_check_url: External URL to poll for health check. + + Raises: + RuntimeError: If services don't become ready within timeout. + """ + logger.info("Connecting to env-ctrl service inside container") + client = EnvCtrlDockerClient.create(self.container_name, timeout=timeout + 60) + + logger.info("Waiting for internal services to be ready...") + result = client.wait_until_ready(timeout=timeout) + if not result.success: + raise RuntimeError(f"Services failed to become ready: {result.message}") + logger.info("Internal services are ready") + + # Poll external URL + logger.info(f"Waiting for external URL to be ready: {health_check_url}") + deadline = time.time() + timeout + while time.time() < deadline: + try: + req = urllib.request.Request(health_check_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as resp: + if resp.status < 500: + logger.info("External URL is ready") + return + except urllib.error.HTTPError as e: + # HTTPError is raised for 4xx/5xx but also has a status code + # Accept 4xx (client errors like auth required) as "site is up" + if e.code < 500: + logger.info(f"External URL is ready (status {e.code})") + return + except (urllib.error.URLError, TimeoutError): + pass + time.sleep(2) + + raise RuntimeError(f"External URL failed to become ready: {health_check_url}") diff --git a/src/webarena_verified/environments/env_ctrl_client/docker_client.py b/src/webarena_verified/environments/env_ctrl_client/docker_client.py index 9c6ca8c..01cecd7 100644 --- a/src/webarena_verified/environments/env_ctrl_client/docker_client.py +++ b/src/webarena_verified/environments/env_ctrl_client/docker_client.py @@ -7,7 +7,6 @@ import json import subprocess import time -from pathlib import Path from webarena_verified.types.environment import EnvCtrlResult @@ -250,95 +249,6 @@ def wait_until_ready( }, ) - def patch(self) -> EnvCtrlResult: - """Apply patches from /tmp/patches/. - - Returns: - EnvCtrlResult with success, message, and details. - """ - return self._run_setup_cli("patch") - - def stage_files(self, source_dir: str, target_dir: str = "/tmp/patches") -> EnvCtrlResult: - """Stage files from host directory to container. - - Creates a tarball of source_dir and pipes it to env-ctrl setup stage. - - Args: - source_dir: Local directory containing files to stage. - target_dir: Target directory inside container (default: /tmp/patches). - - Returns: - EnvCtrlResult with success, message, and details. - """ - source = Path(source_dir) - if not source.is_dir(): - return EnvCtrlResult( - success=False, - message=f"Source directory does not exist: {source_dir}", - details={}, - ) - - # Create tar and pipe to env-ctrl setup stage - tar_cmd = ["tar", "-C", str(source), "-cf", "-", "."] - stage_cmd = [ - "docker", - "exec", - "-i", - self._container_name, - "env-ctrl", - "setup", - "stage", - "--target", - target_dir, - ] - - try: - tar_proc = subprocess.Popen(tar_cmd, stdout=subprocess.PIPE) - try: - result = subprocess.run( - stage_cmd, - stdin=tar_proc.stdout, - capture_output=True, - text=True, - timeout=self._timeout, - ) - finally: - # Close the pipe in parent to avoid resource leaks - if tar_proc.stdout: - tar_proc.stdout.close() - tar_proc.wait() - - # Check tar exit code - if tar_proc.returncode != 0: - return EnvCtrlResult( - success=False, - message=f"tar failed with exit code {tar_proc.returncode}", - details={"source": source_dir}, - ) - - if result.returncode == 0: - try: - parsed = json.loads(result.stdout.strip()) - return EnvCtrlResult.model_validate(parsed) - except json.JSONDecodeError: - return EnvCtrlResult( - success=True, - message=f"Staged files to {target_dir}", - details={"source": source_dir, "target": target_dir}, - ) - else: - return EnvCtrlResult( - success=False, - message=result.stderr or f"Stage failed with exit code {result.returncode}", - details={"exit_code": result.returncode}, - ) - except subprocess.TimeoutExpired: - return EnvCtrlResult( - success=False, - message=f"Stage timed out after {self._timeout}s", - details={}, - ) - def analyze_disk(self, top: int = 20) -> EnvCtrlResult: """Analyze disk usage in container. diff --git a/src/webarena_verified/environments/patches/__init__.py b/src/webarena_verified/environments/patches/__init__.py deleted file mode 100644 index 283b1ab..0000000 --- a/src/webarena_verified/environments/patches/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -"""Patch system interfaces and models for WebArena environments.""" - -from webarena_verified.environments import SiteInstanceHandler -from webarena_verified.types.environment import SiteInstanceCommandResult - -__all__ = [ - "SiteInstanceCommandResult", - "SiteInstanceHandler", -] diff --git a/src/webarena_verified/environments/patches/reddit/p01_remove_rate_limits.py b/src/webarena_verified/environments/patches/reddit/p01_remove_rate_limits.py deleted file mode 100644 index 06135ea..0000000 --- a/src/webarena_verified/environments/patches/reddit/p01_remove_rate_limits.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python3 -""" -Remove rate limiting from Reddit (Postmill) submissions. - -This script removes the rate limiting annotations from the SubmissionData class -to allow unlimited post submissions without waiting periods. - -Usage: - As module: apply_patch(handler) - As script: python p01_remove_rate_limits.py -""" - -import logging - -from webarena_verified.environments import SiteInstanceHandler - -logger = logging.getLogger(__name__) - - -def backup_submission_data(handler: SiteInstanceHandler) -> None: - """Backup the SubmissionData.php file.""" - file_path = "/var/www/html/src/DataObject/SubmissionData.php" - backup_path = f"{file_path}.bak" - - handler.exec_cmd( - ["cp", file_path, backup_path], - check=True, - ) - logger.info(f"Backed up {file_path} to {backup_path}") - - -def remove_rate_limits(handler: SiteInstanceHandler) -> None: - """Remove rate limit annotations from SubmissionData.php.""" - file_path = "/var/www/html/src/DataObject/SubmissionData.php" - - # Read the current file - result = handler.exec_cmd( - ["cat", file_path], - capture_output=True, - text=True, - check=True, - ) - - content = result.stdout - lines = content.split("\n") - new_lines = [] - skip_next = False - - for _i, line in enumerate(lines): - # Skip lines with @RateLimit annotation - if "@RateLimit" in line: - # Check if this is a single-line or multi-line annotation - if line.strip().endswith(")"): - # Single line, skip it - continue - # Multi-line annotation, mark to skip until we find the closing paren - skip_next = True - continue - - if skip_next: - # Continue skipping until we find the end of the annotation - if ")" in line: - skip_next = False - continue - - new_lines.append(line) - - # Write the modified content back - new_content = "\n".join(new_lines) - - handler.exec_cmd( - ["tee", file_path], - input_data=new_content, - capture_output=True, - check=True, - ) - logger.info(f"Removed rate limit annotations from {file_path}") - - -def clear_symfony_cache(handler: SiteInstanceHandler) -> None: - """Clear Symfony cache to apply changes.""" - handler.exec_cmd( - ["php", "bin/console", "cache:clear"], - capture_output=True, - check=True, - ) - logger.info("Cleared Symfony cache") - - -def verify_changes(handler: SiteInstanceHandler) -> bool: - """Verify that rate limits were removed.""" - file_path = "/var/www/html/src/DataObject/SubmissionData.php" - - result = handler.exec_cmd( - ["grep", "-c", "@RateLimit", file_path], - capture_output=True, - text=True, - ) - - # grep returns exit code 1 if no matches found (which is what we want) - if result.returncode == 1: - logger.info("Rate limit annotations successfully removed") - return True - count = result.stdout.strip() - logger.warning(f"Still found {count} @RateLimit annotation(s)") - return False - - -def restart_php_fpm(handler: SiteInstanceHandler) -> None: - """Restart PHP-FPM to reload the code.""" - handler.exec_cmd( - ["supervisorctl", "restart", "php-fpm"], - check=True, - ) - logger.info("Restarted PHP-FPM") - - -def apply_patch(handler: SiteInstanceHandler) -> bool: - """Apply the rate limit removal patch. - - Args: - handler: SiteInstanceHandler for executing commands - - Returns: - True if patch was applied successfully, False otherwise - """ - try: - logger.info("Removing Reddit post rate limiting...") - - # Backup original file - backup_submission_data(handler) - - # Remove rate limits - remove_rate_limits(handler) - - # Clear cache - clear_symfony_cache(handler) - - # Verify - if not verify_changes(handler): - logger.error("Patch verification failed") - return False - - # Restart PHP-FPM - restart_php_fpm(handler) - - logger.info("Rate limiting removal patch applied successfully") - return True - - except Exception as e: - logger.error(f"Error applying patch: {e}") - return False diff --git a/src/webarena_verified/environments/patches/reddit/p02_configure_http_client.py b/src/webarena_verified/environments/patches/reddit/p02_configure_http_client.py deleted file mode 100644 index d456882..0000000 --- a/src/webarena_verified/environments/patches/reddit/p02_configure_http_client.py +++ /dev/null @@ -1,235 +0,0 @@ -#!/usr/bin/env python3 -""" -Configure Reddit (Postmill) Symfony HTTP client to allow localhost and host.docker.internal. - -This script modifies the http_client.yaml configuration to whitelist localhost -and host.docker.internal hosts, allowing the Reddit container to fetch metadata -from local GitLab instances. - -It also creates a custom HTTP client decorator that rewrites localhost:9999 URLs -to localhost (no port), allowing the container to handle submission URLs that -reference the external port mapping. - -Usage: - As module: apply_patch(handler) - As script: python p02_configure_http_client.py -""" - -import logging -import textwrap - -from webarena_verified.environments import SiteInstanceHandler - -logger = logging.getLogger(__name__) - - -def backup_config(handler: SiteInstanceHandler) -> None: - """Backup the existing HTTP client configuration.""" - config_path = "/var/www/html/config/packages/http_client.yaml" - backup_path = f"{config_path}.bak" - - handler.exec_cmd( - ["cp", config_path, backup_path], - check=True, - ) - logger.info(f"Backed up config to {backup_path}") - - -def update_http_client_config(handler: SiteInstanceHandler) -> None: - """Update HTTP client configuration to allow localhost and host.docker.internal. - - This removes the NoPrivateNetworkHttpClient wrapper entirely to allow - connections to private IPs like host.docker.internal. - - Also adds a custom HTTP client decorator that rewrites localhost:9999 URLs - to localhost (no port) to handle same-container URL submissions. - """ - - # New configuration without NoPrivateNetworkHttpClient restriction - new_config = textwrap.dedent(""" - parameters: - postmill.http_client.default_user_agent: "Postmill/%env(APP_VERSION)% (https://postmill.xyz)" - postmill.http_client.external_client_options: - headers: - User-Agent: "%postmill.http_client.default_user_agent%" - base_uri: null - - services: - _defaults: - public: false - - Symfony\\Contracts\\HttpClient\\HttpClientInterface $externalClient: "@postmill.http_client.external_client" - - postmill.http_client.default: - class: Symfony\\Contracts\\HttpClient\\HttpClientInterface - factory: ['@Symfony\\Contracts\\HttpClient\\HttpClientInterface', withOptions] - arguments: - - "%postmill.http_client.external_client_options%" - - # Decorator to rewrite localhost:9999 URLs to localhost (same container) - postmill.http_client.url_rewriter: - class: App\\HttpClient\\UrlRewritingHttpClient - arguments: ['@.inner'] - decorates: postmill.http_client.default - decoration_priority: 10 - - # Use the default client directly without NoPrivateNetworkHttpClient wrapper - postmill.http_client.external_client: - alias: postmill.http_client.default - - postmill.http_client.external_client.psr18: - class: Symfony\\Component\\HttpClient\\Psr18Client - arguments: ['@postmill.http_client.external_client'] - """).strip() - - # Write the new configuration - config_path = "/var/www/html/config/packages/http_client.yaml" - - # Use exec_cmd with input_data to write the file - handler.exec_cmd( - ["tee", config_path], - input_data=new_config, - capture_output=True, - check=True, - ) - logger.info(f"Updated {config_path} - removed NoPrivateNetworkHttpClient restriction") - - -def create_url_rewriting_client(handler: SiteInstanceHandler) -> None: - """Create the UrlRewritingHttpClient PHP class.""" - - php_code = textwrap.dedent(""" - client = $client; - } - - public function request(string $method, string $url, array $options = []): ResponseInterface - { - // Rewrite localhost:9999 to localhost (internal container address) - $rewrittenUrl = preg_replace('/^http:\\/\\/localhost:9999\\//', 'http://localhost/', $url); - - return $this->client->request($method, $rewrittenUrl, $options); - } - - public function stream($responses, float $timeout = null): ResponseStreamInterface - { - return $this->client->stream($responses, $timeout); - } - - public function withOptions(array $options): static - { - $clone = clone $this; - $clone->client = $this->client->withOptions($options); - return $clone; - } - } - """).strip() - - # Write the PHP class - class_path = "/var/www/html/src/HttpClient/UrlRewritingHttpClient.php" - class_dir = "/var/www/html/src/HttpClient" - - # Create directory if it doesn't exist - handler.exec_cmd( - ["mkdir", "-p", class_dir], - check=True, - ) - - # Write the file - handler.exec_cmd( - ["tee", class_path], - input_data=php_code, - capture_output=True, - check=True, - ) - logger.info(f"Created {class_path}") - - -def clear_symfony_cache(handler: SiteInstanceHandler) -> None: - """Clear Symfony cache completely and warm up production cache.""" - # Remove all cached files - handler.exec_cmd( - ["sh", "-c", "rm -rf /var/www/html/var/cache/*"], - check=True, - ) - logger.info("Removed all cached files") - - # Warm up production cache - handler.exec_cmd( - ["sh", "-c", "APP_ENV=prod php bin/console cache:warmup"], - capture_output=True, - check=True, - ) - logger.info("Warmed up production cache") - - -def verify_configuration(handler: SiteInstanceHandler) -> bool: - """Verify the configuration was applied correctly.""" - result = handler.exec_cmd( - ["cat", "/var/www/html/config/packages/http_client.yaml"], - capture_output=True, - text=True, - check=True, - ) - - if "alias: postmill.http_client.default" in result.stdout: - logger.info("Configuration verified successfully") - return True - logger.error("Configuration verification failed") - return False - - -def apply_patch(handler: SiteInstanceHandler) -> bool: - """Apply the HTTP client configuration patch. - - Args: - handler: SiteInstanceHandler for executing commands - - Returns: - True if patch was applied successfully, False otherwise - """ - try: - logger.info("Configuring Reddit HTTP client to allow localhost and host.docker.internal...") - - # Backup existing configuration - backup_config(handler) - - # Create URL rewriting HTTP client - create_url_rewriting_client(handler) - - # Update configuration - update_http_client_config(handler) - - # Clear cache - clear_symfony_cache(handler) - - # Verify - if not verify_configuration(handler): - logger.error("Patch verification failed") - return False - - logger.info("HTTP client configuration patch applied successfully") - return True - - except Exception as e: - logger.error(f"Error applying patch: {e}") - return False diff --git a/src/webarena_verified/environments/patches/shopping/p01_contact_form.py b/src/webarena_verified/environments/patches/shopping/p01_contact_form.py deleted file mode 100644 index 4c332db..0000000 --- a/src/webarena_verified/environments/patches/shopping/p01_contact_form.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -""" -Patch Magento contact form to post data in real-time to a dummy endpoint. - -This script modifies the contact form template to send form data as JSON -to localhost/dummy_bin on every keystroke (no debouncing). - -Usage: - As module: apply_patch(handler) - As script: python p01_contact_form.py -""" - -import contextlib -import logging -import textwrap - -from webarena_verified.environments import SiteInstanceHandler - -logger = logging.getLogger(__name__) - - -def backup_form_template(handler: SiteInstanceHandler) -> None: - """Backup the original contact form template.""" - file_path = "/var/www/magento2/vendor/magento/module-contact/view/frontend/templates/form.phtml" - backup_path = f"{file_path}.bak" - - handler.exec_cmd( - ["cp", file_path, backup_path], - check=True, - ) - logger.info(f"Backed up {file_path} to {backup_path}") - - -def restore_from_backup(handler: SiteInstanceHandler) -> None: - """Restore the form template from backup if it exists.""" - file_path = "/var/www/magento2/vendor/magento/module-contact/view/frontend/templates/form.phtml" - backup_path = f"{file_path}.bak" - - # Check if backup exists - result = handler.exec_cmd( - ["test", "-f", backup_path], - capture_output=True, - ) - - if result.returncode == 0: - # Backup exists, restore it - handler.exec_cmd( - ["cp", backup_path, file_path], - check=True, - ) - logger.info(f"Restored original form from {backup_path}") - else: - logger.info("No backup found, will patch current file") - - -def patch_contact_form(handler: SiteInstanceHandler) -> None: - """Add real-time form posting JavaScript to the contact form.""" - file_path = "/var/www/magento2/vendor/magento/module-contact/view/frontend/templates/form.phtml" - - # Read the current file - result = handler.exec_cmd( - ["cat", file_path], - capture_output=True, - text=True, - check=True, - ) - - content = result.stdout - - # JavaScript code to add real-time form posting - realtime_script = textwrap.dedent(""" - - """).strip() - - # Insert the script before the closing tag or at the end - if "" in content: - # Insert before closing form tag - content = content.replace("", realtime_script + "\n") - else: - # Append at the end if no closing form tag found - content += realtime_script - - # Write the modified content back - handler.exec_cmd( - ["tee", file_path], - input_data=content, - capture_output=True, - check=True, - ) - logger.info(f"Patched {file_path} with real-time form posting") - - -def clear_magento_cache(handler: SiteInstanceHandler) -> None: - """Clear Magento cache to apply changes.""" - logger.info("Clearing Magento cache...") - - # Try multiple cache clearing methods - commands = [ - ["php", "/var/www/magento2/bin/magento", "cache:clean"], - ["php", "/var/www/magento2/bin/magento", "cache:flush"], - ["rm", "-rf", "/var/www/magento2/var/cache/*"], - ["rm", "-rf", "/var/www/magento2/var/page_cache/*"], - ["rm", "-rf", "/var/www/magento2/var/view_preprocessed/*"], - ] - - for cmd in commands: - with contextlib.suppress(Exception): - handler.exec_cmd(cmd, capture_output=True, check=False) - - logger.info("Cleared Magento cache") - - -def verify_patch(handler: SiteInstanceHandler) -> bool: - """Verify that the patch was applied correctly.""" - file_path = "/var/www/magento2/vendor/magento/module-contact/view/frontend/templates/form.phtml" - - result = handler.exec_cmd( - ["grep", "-c", "dummy_bin", file_path], - capture_output=True, - text=True, - ) - - if result.returncode == 0 and int(result.stdout.strip()) > 0: - logger.info("Patch applied successfully") - return True - logger.error("Patch verification failed") - return False - - -def apply_patch(handler: SiteInstanceHandler) -> bool: - """Apply the contact form real-time posting patch. - - Args: - handler: SiteInstanceHandler for executing commands - - Returns: - True if patch was applied successfully, False otherwise - """ - try: - logger.info("Patching Magento contact form for real-time posting...") - - # Restore from backup if it exists (to allow re-running the script) - restore_from_backup(handler) - - # Backup original file - backup_form_template(handler) - - # Apply patch - patch_contact_form(handler) - - # Clear cache - clear_magento_cache(handler) - - # Verify - if not verify_patch(handler): - logger.error("Patch verification failed") - return False - - logger.info("Contact form patch applied successfully") - return True - - except Exception as e: - logger.error(f"Error applying patch: {e}") - return False diff --git a/src/webarena_verified/environments/setup/__init__.py b/src/webarena_verified/environments/setup/__init__.py new file mode 100644 index 0000000..857fef6 --- /dev/null +++ b/src/webarena_verified/environments/setup/__init__.py @@ -0,0 +1,262 @@ +"""Setup utilities for WebArena Docker environments. + +This package provides functions for setting up Docker volumes with data +required by WebArena sites (wikipedia, map) and cleaning up those volumes. +""" + +from __future__ import annotations + +from collections.abc import Callable +from typing import TYPE_CHECKING + +from ..container.config import get_container_config, get_sites_with_setup +from .docker_ops import ( + create_volume, + download_file, + extract_tar_to_volume, + remove_volume, + volume_exists, + volume_is_empty, +) + +if TYPE_CHECKING: + from pathlib import Path + + from webarena_verified.types.task import WebArenaSite + +# Type for progress callbacks +ProgressCallback = Callable[[str, str], None] # (phase, message) + + +def _print_default(phase: str, message: str) -> None: + """Default progress callback that prints to stdout.""" + print(f"[{phase}] {message}") + + +def setup_init( # noqa: C901, PLR0912 + *, + site: WebArenaSite | None, + data_dir: Path, + dry_run: bool = False, + progress: ProgressCallback | None = None, +) -> dict[str, list[str]]: + """Initialize Docker volumes for a site (or all sites). + + Downloads required data files and sets up Docker volumes with the data. + Operations are idempotent: existing files won't be re-downloaded and + non-empty volumes won't be overwritten. + + Args: + site: Site to initialize, or None for all sites requiring setup. + data_dir: Directory to store/find downloaded data files. + dry_run: If True, show what would be done without making changes. + progress: Optional callback for progress messages. + + Returns: + Dict with keys: + - "downloads": List of files downloaded + - "volumes_created": List of volumes created + - "volumes_populated": List of volumes that were populated with data + - "skipped": List of operations skipped (file/volume already exists) + + Raises: + ValueError: If site doesn't support setup. + RuntimeError: If setup fails. + """ + if progress is None: + progress = _print_default + + # Determine which sites to set up + if site is not None: + sites = [site] + else: + sites = get_sites_with_setup() + + result: dict[str, list[str]] = { + "downloads": [], + "volumes_created": [], + "volumes_populated": [], + "skipped": [], + } + + for s in sites: + config = get_container_config(site=s) + if config.setup is None or not config.setup.data_urls: + if site is not None: + raise ValueError(f"Site {s.value} doesn't require setup (no data files to download)") + continue + + progress("SITE", f"Setting up {s.value}") + + # Ensure data directory exists + data_dir.mkdir(parents=True, exist_ok=True) + + # Download data files + for url in config.setup.data_urls: + filename = url.rsplit("/", 1)[-1] + file_path = data_dir / filename + + if file_path.exists(): + progress("SKIP", f"File exists: {filename}") + result["skipped"].append(f"download:{filename}") + else: + if dry_run: + progress("DRY-RUN", f"Would download: {url}") + else: + progress("DOWNLOAD", f"Downloading {filename}...") + download_file(url, data_dir, filename) + result["downloads"].append(filename) + + # Set up volumes + for vol_spec in config.setup.volumes: + vol_name = vol_spec.volume_name + + # Create volume if needed + if not volume_exists(vol_name): + if dry_run: + progress("DRY-RUN", f"Would create volume: {vol_name}") + else: + progress("CREATE", f"Creating volume: {vol_name}") + create_volume(vol_name) + result["volumes_created"].append(vol_name) + else: + progress("SKIP", f"Volume exists: {vol_name}") + result["skipped"].append(f"create:{vol_name}") + + # Populate volume if empty + if not dry_run and volume_exists(vol_name) and volume_is_empty(vol_name): + if vol_spec.source_tar: + # Extract from tar file + tar_path = data_dir / vol_spec.source_tar + if tar_path.exists(): + progress("EXTRACT", f"Extracting to {vol_name}...") + extract_tar_to_volume( + tar_path, + vol_name, + extract_path=vol_spec.tar_extract_path, + strip_components=vol_spec.strip_components, + ) + result["volumes_populated"].append(vol_name) + else: + progress("WARN", f"Tar file not found: {vol_spec.source_tar}") + elif dry_run and vol_spec.source_tar: + if volume_exists(vol_name) and not volume_is_empty(vol_name): + progress("DRY-RUN", f"Would skip (has data): {vol_name}") + else: + progress("DRY-RUN", f"Would extract to: {vol_name}") + + if dry_run: + progress("DONE", "Dry run complete - no changes made") + else: + progress("DONE", "Setup complete") + + return result + + +def setup_clean( # noqa: C901 + *, + site: WebArenaSite | None, + force: bool = False, + progress: ProgressCallback | None = None, +) -> dict[str, list[str]]: + """Remove Docker volumes for a site (or all sites). + + Args: + site: Site to clean, or None for all sites. + force: If False, prompt for confirmation before removing. + progress: Optional callback for progress messages. + + Returns: + Dict with keys: + - "removed": List of volumes removed + - "failed": List of volumes that failed to remove (e.g., in use) + + Raises: + SystemExit: If user declines confirmation (only when force=False). + """ + if progress is None: + progress = _print_default + + # Determine which sites to clean + if site is not None: + sites = [site] + else: + sites = get_sites_with_setup() + + # Collect volumes to remove + volumes_to_remove: list[str] = [] + for s in sites: + config = get_container_config(site=s) + if config.setup is None: + continue + + for vol_spec in config.setup.volumes: + if volume_exists(vol_spec.volume_name): + volumes_to_remove.append(vol_spec.volume_name) + + if not volumes_to_remove: + progress("INFO", "No volumes found to remove") + return {"removed": [], "failed": []} + + # Confirmation prompt + if not force: + print(f"\nThis will remove {len(volumes_to_remove)} volume(s):") + for vol in volumes_to_remove: + print(f" - {vol}") + try: + response = input("\nContinue? [y/N]: ").strip().lower() + except EOFError: + response = "n" + + if response not in ("y", "yes"): + progress("ABORT", "Cleanup aborted") + raise SystemExit(0) + + # Remove volumes + result: dict[str, list[str]] = {"removed": [], "failed": []} + for vol_name in volumes_to_remove: + try: + progress("REMOVE", f"Removing {vol_name}...") + remove_volume(vol_name) + result["removed"].append(vol_name) + except RuntimeError as e: + progress("ERROR", f"Failed to remove {vol_name}: {e}") + result["failed"].append(vol_name) + + progress("DONE", f"Removed {len(result['removed'])} volume(s)") + return result + + +def list_site_volumes(*, site: WebArenaSite | None = None) -> dict[str, list[str]]: + """List Docker volumes for a site (or all sites). + + Args: + site: Site to list volumes for, or None for all. + + Returns: + Dict mapping site name to list of existing volume names. + """ + sites = [site] if site is not None else get_sites_with_setup() + result: dict[str, list[str]] = {} + + for s in sites: + config = get_container_config(site=s) + if config.setup is None: + continue + + volumes: list[str] = [] + for vol_spec in config.setup.volumes: + if volume_exists(vol_spec.volume_name): + volumes.append(vol_spec.volume_name) + + if volumes: + result[s.value] = volumes + + return result + + +__all__ = [ + "list_site_volumes", + "setup_clean", + "setup_init", +] diff --git a/src/webarena_verified/environments/setup/docker_ops.py b/src/webarena_verified/environments/setup/docker_ops.py new file mode 100644 index 0000000..4d3629b --- /dev/null +++ b/src/webarena_verified/environments/setup/docker_ops.py @@ -0,0 +1,310 @@ +"""Docker operations for volume setup and data management. + +This module provides low-level operations for Docker volume management, +file downloads, and tar extraction used during site setup. +""" + +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path # noqa: TC003 - Path is used at runtime + + +def volume_exists(name: str) -> bool: + """Check if Docker volume exists. + + Args: + name: Volume name to check. + + Returns: + True if volume exists, False otherwise. + """ + result = subprocess.run( + ["docker", "volume", "ls", "-q", "-f", f"name=^{name}$"], + capture_output=True, + text=True, + ) + return bool(result.stdout.strip()) + + +def volume_is_empty(name: str) -> bool: + """Check if Docker volume is empty (has no files). + + Args: + name: Volume name to check. + + Returns: + True if volume is empty, False if it has content. + """ + result = subprocess.run( + [ + "docker", + "run", + "--rm", + "-v", + f"{name}:/vol:ro", + "alpine", + "sh", + "-c", + "ls -A /vol | head -1", + ], + capture_output=True, + text=True, + ) + return not bool(result.stdout.strip()) + + +def create_volume(name: str) -> bool: + """Create Docker volume if it doesn't exist. + + Args: + name: Volume name to create. + + Returns: + True if volume was created, False if it already existed. + """ + if volume_exists(name): + return False + + result = subprocess.run( + ["docker", "volume", "create", name], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"Failed to create volume {name}: {result.stderr}") + return True + + +def remove_volume(name: str) -> bool: + """Remove Docker volume if it exists. + + Args: + name: Volume name to remove. + + Returns: + True if volume was removed, False if it didn't exist. + + Raises: + RuntimeError: If volume is in use or removal fails. + """ + if not volume_exists(name): + return False + + result = subprocess.run( + ["docker", "volume", "rm", name], + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError(f"Failed to remove volume {name}: {result.stderr}") + return True + + +def list_volumes(prefix: str) -> list[str]: + """List all Docker volumes with the given prefix. + + Args: + prefix: Volume name prefix to filter by. + + Returns: + List of volume names matching the prefix. + """ + result = subprocess.run( + ["docker", "volume", "ls", "-q", "-f", f"name={prefix}"], + capture_output=True, + text=True, + ) + if result.returncode != 0: + return [] + return [v for v in result.stdout.strip().split("\n") if v] + + +def download_file(url: str, output_dir: Path, filename: str | None = None) -> Path: + """Download a file using the best available tool. + + Prefers aria2c for parallel downloads, falls back to wget or curl. + + Args: + url: URL to download from. + output_dir: Directory to save the file. + filename: Optional filename override. If None, uses filename from URL. + + Returns: + Path to the downloaded file. + + Raises: + RuntimeError: If no download tool is available or download fails. + """ + if filename is None: + filename = url.rsplit("/", 1)[-1] + + output_path = output_dir / filename + output_dir.mkdir(parents=True, exist_ok=True) + + if shutil.which("aria2c"): + cmd = [ + "aria2c", + "-x", + "16", + "-s", + "16", + "--file-allocation=none", + "-d", + str(output_dir), + "-o", + filename, + url, + ] + tool = "aria2c" + elif shutil.which("wget"): + cmd = ["wget", "-O", str(output_path), url] + tool = "wget" + elif shutil.which("curl"): + cmd = ["curl", "-L", "-o", str(output_path), url] + tool = "curl" + else: + raise RuntimeError("No download tool available. Install aria2c, wget, or curl.") + + result = subprocess.run(cmd) + if result.returncode != 0: + raise RuntimeError(f"Download with {tool} failed for {url}") + + return output_path + + +def download_with_aria2c_container(urls: list[str], data_dir: Path) -> None: + """Download files using aria2c in an Alpine container. + + This is useful when aria2c is not installed on the host system. + Uses 16 parallel connections for faster downloads. + + Args: + urls: List of URLs to download. + data_dir: Directory to save downloaded files. + + Raises: + RuntimeError: If download fails. + """ + data_dir.mkdir(parents=True, exist_ok=True) + + for url in urls: + filename = url.rsplit("/", 1)[-1] + output_path = data_dir / filename + + if output_path.exists(): + continue + + cmd = [ + "docker", + "run", + "--rm", + "-v", + f"{data_dir}:/data", + "alpine", + "sh", + "-c", + f'apk add --no-cache aria2 && aria2c -x 16 -s 16 --file-allocation=none -d /data -o {filename} "{url}"', + ] + + result = subprocess.run(cmd) + if result.returncode != 0: + raise RuntimeError(f"Failed to download {url}") + + +def extract_tar_to_volume( + tar_file: Path, + volume_name: str, + extract_path: str | None = None, + strip_components: int = 0, +) -> None: + """Extract tar contents to a Docker volume. + + Uses an Alpine container to perform the extraction. + + Args: + tar_file: Path to the tar file on the host. + volume_name: Name of the target Docker volume. + extract_path: Optional path within tar to extract. If None, extracts all. + strip_components: Number of leading path components to strip. + + Raises: + RuntimeError: If extraction fails. + """ + tar_dir = tar_file.parent + tar_name = tar_file.name + + # Build tar extraction command + tar_cmd = f"tar -xf /tar/{tar_name}" + if strip_components > 0: + tar_cmd += f" --strip-components={strip_components}" + tar_cmd += " -C /vol" + if extract_path: + tar_cmd += f" {extract_path}" + + cmd = [ + "docker", + "run", + "--rm", + "-v", + f"{tar_dir}:/tar:ro", + "-v", + f"{volume_name}:/vol", + "alpine", + "sh", + "-c", + tar_cmd, + ] + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"Failed to extract {tar_file} to volume {volume_name}: {result.stderr}") + + +def copy_file_to_volume(file_path: Path, volume_name: str, dest_path: str = ".") -> None: + """Copy a file from host to a Docker volume. + + Uses an Alpine container to perform the copy. + + Args: + file_path: Path to the file on the host. + volume_name: Name of the target Docker volume. + dest_path: Destination path within the volume (default: root). + + Raises: + RuntimeError: If copy fails. + """ + file_dir = file_path.parent + file_name = file_path.name + + cmd = [ + "docker", + "run", + "--rm", + "-v", + f"{file_dir}:/src:ro", + "-v", + f"{volume_name}:/vol", + "alpine", + "cp", + f"/src/{file_name}", + f"/vol/{dest_path}", + ] + + result = subprocess.run(cmd, capture_output=True, text=True) + if result.returncode != 0: + raise RuntimeError(f"Failed to copy {file_path} to volume {volume_name}: {result.stderr}") + + +__all__ = [ + "copy_file_to_volume", + "create_volume", + "download_file", + "download_with_aria2c_container", + "extract_tar_to_volume", + "list_volumes", + "remove_volume", + "volume_exists", + "volume_is_empty", +] diff --git a/src/webarena_verified/environments/site_handler.py b/src/webarena_verified/environments/site_handler.py index df2f6fb..c6ea14b 100644 --- a/src/webarena_verified/environments/site_handler.py +++ b/src/webarena_verified/environments/site_handler.py @@ -11,7 +11,7 @@ class SiteInstanceHandler(Protocol): """Protocol for handling command execution and site operations. - This abstraction allows patches to execute commands without knowing + This abstraction allows site operations to execute commands without knowing the underlying implementation (Docker, SSH, Kubernetes, etc.). Implementations should provide: @@ -64,8 +64,8 @@ def start(self) -> str: """Start the site container and initialize it. Creates and starts the container, then calls init() to apply - configuration and patches. For containers that are already - running, this will recreate them. + configuration. For containers that are already running, this + will recreate them. Returns: The URL where the site is running @@ -78,7 +78,7 @@ def start(self) -> str: def init(self, *, url: str | None = None) -> None: """Initialize the site container. - Applies patches, configuration, and any site-specific setup. + Applies configuration and any site-specific setup. Called automatically by start() after container creation. Args: diff --git a/src/webarena_verified/types/__init__.py b/src/webarena_verified/types/__init__.py index 2cf01e2..96ebde9 100644 --- a/src/webarena_verified/types/__init__.py +++ b/src/webarena_verified/types/__init__.py @@ -1,6 +1,7 @@ """Type definitions for WebArena Verified.""" from .agent_response import FinalAgentResponse, MainObjectiveType, Status +from .container import ContainerStartResult, ContainerStatus, ContainerStatusResult from .environment import EnvCtrlResult from .task import ( AgentResponseEvaluatorCfg, @@ -12,6 +13,9 @@ __all__ = [ "AgentResponseEvaluatorCfg", + "ContainerStartResult", + "ContainerStatus", + "ContainerStatusResult", "EnvCtrlResult", "EvaluatorCfg", "FinalAgentResponse", diff --git a/src/webarena_verified/types/config.py b/src/webarena_verified/types/config.py index 9f76e1b..36d714d 100644 --- a/src/webarena_verified/types/config.py +++ b/src/webarena_verified/types/config.py @@ -4,13 +4,100 @@ from typing import Any, Self from urllib.parse import urlparse -from pydantic import BaseModel, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, model_validator from webarena_verified.core.utils import logger from webarena_verified.utils import get_package_assets_path from .task import WebArenaSite +# ============================================================================ +# Container Configuration Models +# ============================================================================ + + +class ContainerVolumeSpec(BaseModel): + """Specification for a Docker volume setup. + + Defines how to set up a single Docker volume, including where to get data + (from a tar file or empty), and how to mount it in the container. + + Attributes: + volume_name: Full Docker volume name (e.g., "webarena_verified_wikipedia_data") + mount_path: Path inside container where volume is mounted (e.g., "/data/database") + source_tar: Optional tar filename to extract data from (e.g., "osm_tile_server.tar") + tar_extract_path: Path inside tar to extract (e.g., "data/database"). If None, + extracts entire tar to volume root. + strip_components: Number of leading path components to strip when extracting + """ + + model_config = ConfigDict(frozen=True) + + volume_name: str + mount_path: str + source_tar: str | None = None + tar_extract_path: str | None = None + strip_components: int = 0 + + +class ContainerSetupConfig(BaseModel): + """Configuration for setting up Docker volumes before running a container. + + Used by sites that require external data files to be downloaded and + extracted into Docker volumes before the container can run. + + Attributes: + data_urls: URLs to download data files from (e.g., tar files, ZIM files) + volumes: Volume specifications defining how to set up each volume + """ + + model_config = ConfigDict(frozen=True) + + data_urls: tuple[str, ...] = () + volumes: tuple[ContainerVolumeSpec, ...] = () + + +class ContainerConfig(BaseModel): + """Container configuration for running a site. + + Defines Docker image, port mappings, and volume mounts for running + a WebArena site container. + + Attributes: + docker_img: Docker image name (e.g., "am1n3e/webarena-verified-shopping") + container_port: Port inside container where the web service listens (default: 80) + env_ctrl_port: Port inside container where env-ctrl API listens (default: 8877) + host_port: Port on host to expose the web service (default: same as container_port) + host_env_ctrl_port: Port on host to expose env-ctrl API (default: same as env_ctrl_port) + health_check_path: URL path to poll for external health check (e.g., "/login") + setup: Optional setup configuration for sites needing data volume preparation + data_dir_mount: If set, bind-mount data_dir to this path instead of using volumes. + Used for sites like Wikipedia where data should be read directly from disk. + """ + + model_config = ConfigDict(frozen=True) + + docker_img: str + container_port: int = 80 + env_ctrl_port: int = 8877 + host_port: int | None = None + host_env_ctrl_port: int | None = None + health_check_path: str = "/" + setup: ContainerSetupConfig | None = None + data_dir_mount: str | None = None + + @property + def volumes(self) -> dict[str, str]: + """Derive volume mounts from setup.volumes.""" + if self.setup is None: + return {} + return {v.volume_name: v.mount_path for v in self.setup.volumes} + + +# ============================================================================ +# Environment Configuration +# ============================================================================ + class EnvironmentConfig(BaseModel): """Configuration for a single environment/site. @@ -56,6 +143,13 @@ class EnvironmentConfig(BaseModel): extra: dict[str, Any] = Field(default_factory=dict) + container: ContainerConfig | None = None + """Optional container configuration for running this site via Docker. + + If provided, overrides the default container configuration for the site. + When None, the default container config from DEFAULT_CONTAINER_CONFIGS is used. + """ + @model_validator(mode="after") def set_default_active_url_idx(self) -> Self: """Set active_url_idx to 0 if None and urls list is not empty.""" diff --git a/src/webarena_verified/types/container.py b/src/webarena_verified/types/container.py new file mode 100644 index 0000000..310c05d --- /dev/null +++ b/src/webarena_verified/types/container.py @@ -0,0 +1,62 @@ +"""Container-related type definitions for WebArena environments.""" + +from __future__ import annotations + +from enum import StrEnum + +from pydantic import BaseModel, ConfigDict, Field + + +class ContainerStatus(StrEnum): + """Status of a Docker container.""" + + RUNNING = "running" + STOPPED = "stopped" + NOT_FOUND = "not_found" + + +class ContainerStartResult(BaseModel): + """Result from starting a container. + + Attributes: + container_name: Name of the started container. + url: URL to access the site (e.g., "http://localhost:8080"). + env_ctrl_url: URL to access the env-ctrl API (e.g., "http://localhost:8877"). + host_port: Host port mapped to the container's web service. + env_ctrl_host_port: Host port mapped to the container's env-ctrl port. + """ + + model_config = ConfigDict(frozen=True) + + container_name: str = Field(description="Name of the started container") + url: str = Field(description="URL to access the site") + env_ctrl_url: str = Field(description="URL to access the env-ctrl API") + host_port: int = Field(description="Host port mapped to the container's web service") + env_ctrl_host_port: int = Field(description="Host port mapped to the container's env-ctrl port") + + +class ContainerStatusResult(BaseModel): + """Result from checking container status. + + Attributes: + container_name: Name of the container. + status: Current status of the container. + url: URL to access the site (if running). + env_ctrl_url: URL to access the env-ctrl API (if running). + env_ctrl_status: Status from env-ctrl client (if running and reachable). + """ + + model_config = ConfigDict(frozen=True) + + container_name: str = Field(description="Name of the container") + status: ContainerStatus = Field(description="Current status of the container") + url: str | None = Field(default=None, description="URL to access the site (if running)") + env_ctrl_url: str | None = Field(default=None, description="URL to access the env-ctrl API (if running)") + env_ctrl_status: dict | None = Field(default=None, description="Status from env-ctrl client (if running)") + + +__all__ = [ + "ContainerStartResult", + "ContainerStatus", + "ContainerStatusResult", +] diff --git a/tasks.py b/tasks.py index 73db83c..4ef8a76 100644 --- a/tasks.py +++ b/tasks.py @@ -1,8 +1,141 @@ -"""Root invoke tasks file.""" +"""Main invoke tasks file. Use `inv --list` to see available tasks.""" -from invoke import Collection +import os -from dev import tasks as dev_tasks +from invoke import Collection, Context, task +from dev import ci_tasks, code_tasks, data_tasks, docs_tasks, env_tasks +from dev.environments import tasks as envs_tasks +from dev.utils import git_utils +from examples import tasks as demo_tasks + +# Service config: display name, port env var, default port +SERVICES = { + "shopping_admin": ("Shopping Admin", "WA_SHOPPING_ADMIN_PORT", 7780), + "shopping": ("Shopping", "WA_SHOPPING_PORT", 7770), + "gitlab": ("GitLab", "WA_GITLAB_PORT", 8023), + "reddit": ("Reddit", "WA_REDDIT_PORT", 9999), + "wikipedia": ("Wikipedia", "WA_WIKIPEDIA_PORT", 8888), + "map": ("Map", "WA_MAP_PORT", 3030), +} + +# Service descriptions for output +SERVICE_DESCRIPTIONS = { + "shopping_admin": "Magento admin panel", + "shopping": "Magento storefront", + "gitlab": "GitLab instance", + "reddit": "Reddit-like forum", + "wikipedia": "Wikipedia via Kiwix", + "map": "OpenStreetMap with tiles, routing, and geocoding", +} + + +def _get_service_url(service: str) -> str: + """Get the localhost URL for a service.""" + if service in SERVICES: + _, env_var, default_port = SERVICES[service] + port = os.environ.get(env_var, default_port) + return f"http://localhost:{port}" + return "" + + +@task( + help={ + "service": "Service(s) to start (can be specified multiple times). If not specified, starts all services.", + "foreground": "Run in foreground (default: detached)", + }, + iterable=["service"], +) +def up(ctx: Context, service: list[str] | None = None, foreground: bool = False) -> None: + """Start Docker Compose services.""" + services = list(service) if service else [] + + service_args = " ".join(services) if services else "" + detach_flag = "" if foreground else "-d" + + cmd = f"docker compose up {detach_flag} {service_args}".strip() + ctx.run(cmd, pty=foreground, hide=not foreground) + + if not foreground: + # Print URLs for started services + started = services if services else SERVICE_DESCRIPTIONS.keys() + print("Started services:") + for svc in started: + if svc in SERVICE_DESCRIPTIONS: + url = _get_service_url(svc) + description = SERVICE_DESCRIPTIONS[svc] + print(f" {svc:15} {url:30} - {description}") + + +@task( + help={ + "service": "Service(s) to stop (can be specified multiple times). If not specified, stops all services.", + }, + iterable=["service"], +) +def down(ctx: Context, service: list[str] | None = None) -> None: + """Stop Docker Compose services.""" + services = service or [] + if services: + # Stop specific services + service_args = " ".join(services) + ctx.run(f"docker compose stop {service_args}") + ctx.run(f"docker compose rm -f {service_args}") + else: + # Stop all services + ctx.run("docker compose down") + + +@task +def docker_build(ctx: Context, tag: str | None = None, publish: bool = False) -> None: + """Build the webarena-verified Docker image. + + Always tags with git short SHA. Optionally adds an additional tag. + + Args: + tag: Optional additional tag (e.g., "latest", "v1.0.0") + publish: Push image to Docker Hub after building + """ + image = "am1n3e/webarena-verified" + short_sha = git_utils.get_short_sha() + + image_tags = [f"{image}:{short_sha}"] + if tag: + image_tags.append(f"{image}:{tag}") + + tags_arg = " ".join(f"-t {t}" for t in image_tags) + ctx.run(f"docker build {tags_arg} .") + + if publish: + print(f"\nAbout to push: {', '.join(image_tags)}") + confirm = input("Push to Docker Hub? [y/N]: ").strip().lower() + if confirm == "y": + for t in image_tags: + ctx.run(f"docker push {t}") + print("Done.") + else: + print("Push cancelled.") + + +# Create compose namespace +compose_ns = Collection("compose") +compose_ns.add_task(up) +compose_ns.add_task(down) + +# Create the namespace ns = Collection() -ns.add_collection(Collection.from_module(dev_tasks), name="dev") + +# Add top-level tasks +ns.add_task(docker_build) + +# Add namespaces +dev_ns = Collection("dev") +dev_ns.add_collection(Collection.from_module(ci_tasks), name="ci") +dev_ns.add_collection(Collection.from_module(code_tasks), name="code") +dev_ns.add_collection(Collection.from_module(data_tasks), name="data") +dev_ns.add_collection(Collection.from_module(docs_tasks), name="docs") +dev_ns.add_collection(Collection.from_module(env_tasks), name="env") +ns.add_collection(dev_ns) +ns.add_collection(Collection.from_module(demo_tasks), name="demo") +ns.add_collection(envs_tasks.ns, name="envs") +ns.add_collection(compose_ns, name="compose") diff --git a/src/webarena_verified/environments/patches/shopping/__init__.py b/tests/__init__.py similarity index 100% rename from src/webarena_verified/environments/patches/shopping/__init__.py rename to tests/__init__.py diff --git a/tests/integration/environments/__init__.py b/tests/integration/environments/__init__.py new file mode 100644 index 0000000..cb9de88 --- /dev/null +++ b/tests/integration/environments/__init__.py @@ -0,0 +1 @@ +# Integration tests for Docker environments diff --git a/tests/integration/environments/conftest.py b/tests/integration/environments/conftest.py new file mode 100644 index 0000000..73d6ae0 --- /dev/null +++ b/tests/integration/environments/conftest.py @@ -0,0 +1,159 @@ +"""Pytest configuration for integration tests. + +These tests assume containers are already running. Use `inv envs.docker.start --site ` +to start containers before running tests. + +Example: + pytest tests/integration/environments/shopping_admin/ \ + --shopping_admin_url=http://localhost:7780 \ + --shopping_admin_env_ctrl_url=http://localhost:8877 +""" + +import pytest + +from webarena_verified.environments.env_ctrl_client import EnvCtrlDockerClient, HttpClient + + +def pytest_addoption(parser): + """Add CLI options for integration tests. + + All site-specific URL options are defined here because pytest parses + arguments before discovering subdirectory conftest files. + """ + # Playwright options + parser.addoption( + "--playwright-slow-mo", + action="store", + type=int, + default=1500, + help="Playwright slow motion in milliseconds", + ) + parser.addoption( + "--playwright-timeout-sec", + action="store", + type=int, + default=30, + help="Playwright timeout in seconds (default: 30)", + ) + + # Shopping Admin site + parser.addoption( + "--shopping_admin_url", + action="store", + default=None, + help="Base URL for shopping-admin site (e.g., http://localhost:7780)", + ) + parser.addoption( + "--shopping_admin_env_ctrl_url", + action="store", + default=None, + help="env-ctrl URL for shopping-admin site (e.g., http://localhost:8877)", + ) + + # Shopping (storefront) site + parser.addoption( + "--shopping_url", + action="store", + default=None, + help="Base URL for shopping site (e.g., http://localhost:7770)", + ) + parser.addoption( + "--shopping_env_ctrl_url", + action="store", + default=None, + help="env-ctrl URL for shopping site (e.g., http://localhost:8877)", + ) + + # Reddit site + parser.addoption( + "--reddit_url", + action="store", + default=None, + help="Base URL for reddit site (e.g., http://localhost:9999)", + ) + parser.addoption( + "--reddit_env_ctrl_url", + action="store", + default=None, + help="env-ctrl URL for reddit site (e.g., http://localhost:8877)", + ) + + # GitLab site + parser.addoption( + "--gitlab_url", + action="store", + default=None, + help="Base URL for gitlab site (e.g., http://localhost:8023)", + ) + parser.addoption( + "--gitlab_env_ctrl_url", + action="store", + default=None, + help="env-ctrl URL for gitlab site (e.g., http://localhost:8877)", + ) + + # Wikipedia site + parser.addoption( + "--wikipedia_url", + action="store", + default=None, + help="Base URL for wikipedia site (e.g., http://localhost:8888)", + ) + parser.addoption( + "--wikipedia_env_ctrl_url", + action="store", + default=None, + help="env-ctrl URL for wikipedia site (e.g., http://localhost:8877)", + ) + + # Map (OpenStreetMap) site + parser.addoption( + "--map_url", + action="store", + default=None, + help="Base URL for map site (e.g., http://localhost:3030)", + ) + parser.addoption( + "--map_env_ctrl_url", + action="store", + default=None, + help="env-ctrl URL for map site (e.g., http://localhost:8877)", + ) + parser.addoption( + "--map_tile_url", + action="store", + default=None, + help="Tile server URL for map site (e.g., http://localhost:8080)", + ) + + +@pytest.fixture(scope="session") +def playwright_slow_mo(request): + """Get playwright slow motion from CLI option.""" + return request.config.getoption("--playwright-slow-mo") + + +@pytest.fixture(scope="session") +def pw_timeout(request): + """Get playwright timeout in milliseconds from CLI option (specified in seconds).""" + return request.config.getoption("--playwright-timeout-sec") * 1000 + + +@pytest.fixture(scope="session") +def create_docker_client(): + """Factory fixture to create env-ctrl Docker clients.""" + + def _create(container_name: str, timeout: int = 30) -> EnvCtrlDockerClient: + return EnvCtrlDockerClient.create(container_name, timeout=timeout) + + return _create + + +@pytest.fixture(scope="session") +def create_http_client(): + """Factory fixture to create env-ctrl HTTP clients.""" + + def _create(base_url: str, timeout: int = 30) -> HttpClient: + return HttpClient(base_url=base_url, timeout=timeout) + + return _create diff --git a/tests/integration/environments/gitlab/__init__.py b/tests/integration/environments/gitlab/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/environments/gitlab/conftest.py b/tests/integration/environments/gitlab/conftest.py new file mode 100644 index 0000000..b81a412 --- /dev/null +++ b/tests/integration/environments/gitlab/conftest.py @@ -0,0 +1,65 @@ +"""GitLab site fixtures.""" + +import pytest + + +@pytest.fixture(scope="session") +def gitlab_credentials(): + """Default user credentials for GitLab. + + These are the pre-configured credentials in the Docker image, + used for both testing and production environments. + """ + return { + "username": "byteblaze", + "password": "hello1234", + } + + +@pytest.fixture(scope="session") +def gitlab_base_url(request): + """Base URL for GitLab from CLI arg.""" + url = request.config.getoption("--gitlab_url") + if url is None: + raise ValueError("--gitlab_url is required") + return url + + +@pytest.fixture(scope="session") +def gitlab_env_ctrl_url(request): + """env-ctrl URL for GitLab from CLI arg.""" + url = request.config.getoption("--gitlab_env_ctrl_url") + if url is None: + raise ValueError("--gitlab_env_ctrl_url is required") + return url + + +@pytest.fixture(scope="session") +def gitlab_container(): + """GitLab container name (assumes already running).""" + return "webarena-verified-gitlab" + + +@pytest.fixture(scope="session") +def gitlab_docker_client(gitlab_container, create_docker_client): + """Docker client for GitLab container (via docker exec).""" + return create_docker_client(gitlab_container) + + +@pytest.fixture(scope="session") +def gitlab_http_client(gitlab_env_ctrl_url, create_http_client): + """HTTP client for GitLab env-ctrl server.""" + return create_http_client(gitlab_env_ctrl_url) + + +@pytest.fixture +def gitlab_logged_in_page(page, gitlab_base_url, gitlab_credentials, pw_timeout): + """Page fixture that's already logged in to GitLab.""" + page.goto(f"{gitlab_base_url}/users/sign_in") + # Use test IDs for reliable selectors + page.get_by_test_id("username-field").fill(gitlab_credentials["username"]) + page.get_by_test_id("password-field").fill(gitlab_credentials["password"]) + page.get_by_test_id("sign-in-button").click() + # Wait for dashboard to confirm login + page.wait_for_url("**/", timeout=pw_timeout) + return page diff --git a/tests/integration/environments/gitlab/test_basic.py b/tests/integration/environments/gitlab/test_basic.py new file mode 100644 index 0000000..28c4529 --- /dev/null +++ b/tests/integration/environments/gitlab/test_basic.py @@ -0,0 +1,39 @@ +"""Basic HTTP health tests for GitLab Docker container. + +Tests basic connectivity to the GitLab instance without requiring Playwright. + +Usage: + pytest tests/integration/environments/gitlab/test_basic.py +""" + +import urllib.request + +import pytest + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_gitlab] + + +@pytest.mark.flaky(reruns=2) +def test_gitlab_homepage_responds(gitlab_container, gitlab_base_url): + """Test that homepage responds with 200.""" + req = urllib.request.Request(gitlab_base_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 + + +@pytest.mark.flaky(reruns=2) +def test_gitlab_login_page_accessible(gitlab_container, gitlab_base_url): + """Test that login page is accessible.""" + login_url = f"{gitlab_base_url}/users/sign_in" + req = urllib.request.Request(login_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 + + +@pytest.mark.flaky(reruns=2) +def test_gitlab_api_accessible(gitlab_container, gitlab_base_url): + """Test that GitLab API endpoint is accessible.""" + api_url = f"{gitlab_base_url}/api/v4/projects" + req = urllib.request.Request(api_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 diff --git a/tests/integration/environments/gitlab/test_playwright.py b/tests/integration/environments/gitlab/test_playwright.py new file mode 100644 index 0000000..6812009 --- /dev/null +++ b/tests/integration/environments/gitlab/test_playwright.py @@ -0,0 +1,191 @@ +"""Playwright UI tests for GitLab Docker container. + +Tests GitLab functionality using browser automation. + +Test categories: +- User Login: Authentication flow +- Issue Management: Create and update issues +- Project Management: Update project settings + +Usage: + pytest tests/integration/environments/gitlab/test_playwright.py + pytest tests/integration/environments/gitlab/test_playwright.py --headed +""" + +import pytest +from playwright.sync_api import expect + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_gitlab] + +# ============================================================================= +# User Login Tests +# ============================================================================= + + +@pytest.mark.flaky(reruns=2) +def test_gitlab_user_login(gitlab_container, gitlab_base_url, gitlab_credentials, page, pw_timeout): + """Test user login flow.""" + page.goto(f"{gitlab_base_url}/users/sign_in") + + # Wait for and fill login form using test IDs + page.get_by_test_id("username-field").fill(gitlab_credentials["username"]) + page.get_by_test_id("password-field").fill(gitlab_credentials["password"]) + page.get_by_test_id("sign-in-button").click() + + # Wait for redirect to dashboard and verify logged in + page.wait_for_url("**/", timeout=pw_timeout) + expect(page.get_by_role("heading", name="Projects")).to_be_visible(timeout=pw_timeout) + + +# ============================================================================= +# Issue Management Tests +# ============================================================================= + + +@pytest.mark.flaky(reruns=2) +def test_gitlab_create_issue(gitlab_container, gitlab_base_url, gitlab_logged_in_page, pw_timeout): + """Test creating a new issue.""" + page = gitlab_logged_in_page + project_path = "byteblaze/a11y-syntax-highlighting" + + # Navigate to new issue page + page.goto(f"{gitlab_base_url}/{project_path}/-/issues/new") + + # Fill in issue details using role-based selectors + issue_title = "Test Issue Created by Playwright" + issue_description = "This is a test issue created by automated Playwright tests." + + page.get_by_role("textbox", name="Title (required)").fill(issue_title) + page.get_by_role("textbox", name="Description").fill(issue_description) + page.get_by_role("button", name="Create issue").click() + + # Wait for issue to be created and verify + page.wait_for_url(f"**/{project_path}/-/issues/*", timeout=pw_timeout) + expect(page.get_by_role("heading", name=issue_title)).to_be_visible(timeout=pw_timeout) + + +@pytest.mark.flaky(reruns=2) +def test_gitlab_update_issue_title(gitlab_container, gitlab_base_url, gitlab_logged_in_page, pw_timeout): + """Test updating an existing issue's title.""" + page = gitlab_logged_in_page + project_path = "byteblaze/a11y-webring.club" + issue_number = "21" + + # Navigate to existing issue + page.goto(f"{gitlab_base_url}/{project_path}/-/issues/{issue_number}") + + # Get original title + original_title = page.locator("h1.title").first.inner_text() + + # Click edit button + page.get_by_role("button", name="Edit title and description").click() + + # Wait for edit form and update title + title_input = page.get_by_role("textbox", name="Title") + title_input.wait_for(timeout=pw_timeout) + updated_title = f"{original_title} [Updated]" + title_input.fill(updated_title) + + # Save changes + page.get_by_role("button", name="Save changes").click() + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Verify title was updated + expect(page.locator("h1.title").first).to_contain_text("[Updated]", timeout=pw_timeout) + + # Restore original title + page.get_by_role("button", name="Edit title and description").click() + title_input = page.get_by_role("textbox", name="Title") + title_input.wait_for(timeout=pw_timeout) + title_input.fill(original_title) + page.get_by_role("button", name="Save changes").click() + + +@pytest.mark.flaky(reruns=2) +def test_gitlab_close_and_reopen_issue(gitlab_container, gitlab_base_url, gitlab_logged_in_page, pw_timeout): + """Test closing and reopening an issue.""" + page = gitlab_logged_in_page + project_path = "byteblaze/a11y-webring.club" + issue_number = "21" + + # Navigate to existing issue + page.goto(f"{gitlab_base_url}/{project_path}/-/issues/{issue_number}") + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Define button locators + reopen_button = page.get_by_role("button", name="Reopen issue").first + close_button = page.get_by_role("button", name="Close issue").first + + # Check if issue is already closed and reopen it first if needed + if reopen_button.is_visible(): + # Issue is closed, reopen it first + reopen_button.click() + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Now close the issue + close_button.wait_for(state="visible", timeout=pw_timeout) + close_button.click() + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Verify issue is closed by checking "Reopen issue" button is now visible + expect(reopen_button).to_be_visible(timeout=pw_timeout) + + # Reopen the issue + reopen_button.click() + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Verify issue is open again by checking "Close issue" button is visible + expect(close_button).to_be_visible(timeout=pw_timeout) + + +# ============================================================================= +# Project Management Tests +# ============================================================================= + + +@pytest.mark.flaky(reruns=2) +def test_gitlab_update_project_description(gitlab_container, gitlab_base_url, gitlab_logged_in_page, pw_timeout): + """Test updating a project's description.""" + page = gitlab_logged_in_page + project_path = "byteblaze/a11y-syntax-highlighting" + + # Navigate to project settings + page.goto(f"{gitlab_base_url}/{project_path}/edit") + + # Get the description textarea + description_input = page.get_by_role("textbox", name="Project description") + description_input.wait_for(timeout=pw_timeout) + + # Get original description + original_description = description_input.input_value() + + # Update description + updated_description = f"{original_description} [Test Update]" + description_input.fill(updated_description) + + # Save changes + page.get_by_role("button", name="Save changes").first.click() + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Verify description was updated + expect(description_input).to_have_value(updated_description, timeout=pw_timeout) + + # Restore original description + description_input.fill(original_description) + page.get_by_role("button", name="Save changes").first.click() + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + +@pytest.mark.flaky(reruns=2) +def test_gitlab_navigate_project_repository(gitlab_container, gitlab_base_url, gitlab_logged_in_page, pw_timeout): + """Test navigating to project repository.""" + page = gitlab_logged_in_page + project_path = "byteblaze/a11y-syntax-highlighting" + + # Navigate to repository + page.goto(f"{gitlab_base_url}/{project_path}/-/tree/main") + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Verify we can see files in the repository (use .first() as README.md appears in both + # the file table and the readme preview section) + expect(page.get_by_role("link", name="README.md").first).to_be_visible(timeout=pw_timeout) diff --git a/tests/integration/environments/map/__init__.py b/tests/integration/environments/map/__init__.py new file mode 100644 index 0000000..2287bb4 --- /dev/null +++ b/tests/integration/environments/map/__init__.py @@ -0,0 +1 @@ +"""Map (OpenStreetMap) integration tests.""" diff --git a/tests/integration/environments/map/conftest.py b/tests/integration/environments/map/conftest.py new file mode 100644 index 0000000..7867b5e --- /dev/null +++ b/tests/integration/environments/map/conftest.py @@ -0,0 +1,27 @@ +"""Map (OpenStreetMap) site fixtures.""" + +import pytest + + +@pytest.fixture(scope="session") +def map_base_url(request): + """Base URL for OpenStreetMap website from CLI arg.""" + url = request.config.getoption("--map_url") + if url is None: + raise ValueError("--map_url is required") + return url + + +@pytest.fixture(scope="session") +def map_tile_url(request): + """Tile server URL for OpenStreetMap from CLI arg.""" + url = request.config.getoption("--map_tile_url") + if url is None: + raise ValueError("--map_tile_url is required") + return url + + +@pytest.fixture(scope="session") +def map_container(): + """Map container name (assumes already running).""" + return "osm-website" diff --git a/tests/integration/environments/map/test_basic.py b/tests/integration/environments/map/test_basic.py new file mode 100644 index 0000000..b4c63fb --- /dev/null +++ b/tests/integration/environments/map/test_basic.py @@ -0,0 +1,66 @@ +"""Basic HTTP health tests for OpenStreetMap Docker container. + +Tests basic connectivity to the OpenStreetMap website and tile server +without requiring Playwright. + +Usage: + pytest tests/integration/environments/map/test_basic.py \ + --map_url=http://localhost:3030 \ + --map_tile_url=http://localhost:8080 +""" + +import urllib.request + +import pytest + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_map] + + +@pytest.mark.flaky(reruns=2) +def test_map_homepage_responds(map_container, map_base_url): + """Test that OpenStreetMap homepage responds with 200.""" + req = urllib.request.Request(map_base_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 + + +@pytest.mark.flaky(reruns=2) +def test_map_login_page_accessible(map_container, map_base_url): + """Test that login page is accessible.""" + login_url = f"{map_base_url}/login" + req = urllib.request.Request(login_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 + + +@pytest.mark.flaky(reruns=2) +def test_map_api_capabilities(map_container, map_base_url): + """Test that OSM API capabilities endpoint is accessible.""" + api_url = f"{map_base_url}/api/capabilities" + req = urllib.request.Request(api_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 + content = response.read().decode("utf-8") + assert "api" in content.lower() + + +@pytest.mark.flaky(reruns=2) +def test_tile_server_responds(map_container, map_tile_url): + """Test that tile server responds with 200.""" + req = urllib.request.Request(map_tile_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 + + +@pytest.mark.flaky(reruns=2) +def test_tile_endpoint_accessible(map_container, map_tile_url): + """Test that tile endpoint returns a tile image. + + Requests zoom level 0, which should always exist (single world tile). + """ + tile_url = f"{map_tile_url}/tile/0/0/0.png" + req = urllib.request.Request(tile_url, method="GET") + with urllib.request.urlopen(req, timeout=30) as response: + assert response.status == 200 + content_type = response.headers.get("Content-Type", "") + assert "image" in content_type or response.read(4) == b"\x89PNG" diff --git a/tests/integration/environments/map/test_ci.py b/tests/integration/environments/map/test_ci.py new file mode 100644 index 0000000..60fb4ad --- /dev/null +++ b/tests/integration/environments/map/test_ci.py @@ -0,0 +1,140 @@ +"""CI-specific tests for Map Docker container with Monaco data. + +These tests use a tiny Monaco OSM dataset (~656KB source) for fast CI testing. +They verify basic functionality without requiring the full US-Northeast dataset. + +Monaco coordinates: ~43.73ยฐN, 7.42ยฐE + +Usage: + inv dev.ci.generate-map-data # Generate Monaco data first (or restore from cache) + pytest tests/integration/environments/map/test_ci.py -m integration_docker_map +""" + +import math +import urllib.error +import urllib.request + +import pytest + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_map] + +# Monaco bounds for testing +MONACO_LAT = 43.7384 +MONACO_LON = 7.4246 +MONACO_ZOOM = 14 + + +@pytest.mark.flaky(reruns=2) +def test_map_ci_homepage_loads(map_container, map_base_url, page, pw_timeout): + """Test that map homepage loads with Monaco data.""" + page.goto(map_base_url, timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Verify we're on the OpenStreetMap page + content = page.content().lower() + assert "openstreetmap" in content or "map" in content + + +@pytest.mark.flaky(reruns=2) +def test_map_ci_monaco_view(map_container, map_base_url, page, pw_timeout): + """Test that Monaco area loads correctly.""" + # Navigate to Monaco coordinates + monaco_url = f"{map_base_url}/#map={MONACO_ZOOM}/{MONACO_LAT}/{MONACO_LON}" + page.goto(monaco_url, timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Page should load without errors + content = page.content() + assert len(content) > 100, "Page should have content" + + +@pytest.mark.flaky(reruns=2) +def test_map_ci_tile_renders(map_container, map_tile_url): + """Test that tiles render for Monaco area. + + Requests a tile at zoom 14 covering Monaco. + Tile coordinates calculated for Monaco (43.73ยฐN, 7.42ยฐE). + """ + # Calculate tile coordinates for Monaco at zoom 14 + # Using standard Web Mercator tile calculation + zoom = MONACO_ZOOM + lat_rad = math.radians(MONACO_LAT) + n = 2**zoom + x = int((MONACO_LON + 180) / 360 * n) + y = int((1 - math.asinh(math.tan(lat_rad)) / math.pi) / 2 * n) + + tile_url = f"{map_tile_url}/tile/{zoom}/{x}/{y}.png" + req = urllib.request.Request(tile_url, method="GET") + + with urllib.request.urlopen(req, timeout=60) as response: + assert response.status == 200 + content_type = response.headers.get("Content-Type", "") + data = response.read() + # Verify it's a PNG image + assert "image" in content_type or data[:4] == b"\x89PNG", "Response should be a PNG tile" + # Verify tile has actual content (not just blank) + assert len(data) > 100, "Tile should have content" + + +@pytest.mark.flaky(reruns=2) +def test_map_ci_search_monaco(map_container, map_base_url, page, pw_timeout): + """Test that search finds Monaco location.""" + page.goto(map_base_url, timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Find search input + search_input = page.locator('input[name="query"]').first + if search_input.is_visible(timeout=5000): + search_input.fill("Monaco") + search_input.press("Enter") + + # Wait for search results + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Check URL or content for search results + content = page.content().lower() + # Search should show Monaco results + assert "monaco" in content or "search" in page.url.lower() + + +@pytest.mark.flaky(reruns=2) +def test_map_ci_routing_endpoint(map_container, map_tile_url): + """Test that OSRM routing endpoint responds. + + Tests routing between two points in Monaco. + """ + # Two points in Monaco for routing test + start_lon, start_lat = 7.4167, 43.7333 # Monaco-Ville + end_lon, end_lat = 7.4286, 43.7396 # Monte Carlo + + # OSRM route request + route_url = f"{map_tile_url}/osrm/car/route/v1/driving/{start_lon},{start_lat};{end_lon},{end_lat}?overview=false" + + req = urllib.request.Request(route_url, method="GET") + req.add_header("Accept", "application/json") + + with urllib.request.urlopen(req, timeout=30) as response: + assert response.status == 200 + content = response.read().decode("utf-8") + # OSRM returns JSON with "code": "Ok" on success + assert '"code"' in content + + +@pytest.mark.flaky(reruns=2) +def test_map_ci_nominatim_search(map_container, map_tile_url): + """Test that Nominatim geocoding endpoint responds.""" + search_url = f"{map_tile_url}/nominatim/search?q=Monaco&format=json&limit=1" + + req = urllib.request.Request(search_url, method="GET") + req.add_header("Accept", "application/json") + + try: + with urllib.request.urlopen(req, timeout=30) as response: + assert response.status == 200 + content = response.read().decode("utf-8") + # Should return JSON array + assert content.startswith("[") + except urllib.error.HTTPError as e: + # Accept endpoint existing even if search fails + if e.code >= 500: + raise diff --git a/tests/integration/environments/map/test_playwright.py b/tests/integration/environments/map/test_playwright.py new file mode 100644 index 0000000..1730d9b --- /dev/null +++ b/tests/integration/environments/map/test_playwright.py @@ -0,0 +1,111 @@ +"""Playwright UI tests for OpenStreetMap Docker container. + +Tests OpenStreetMap website functionality using browser automation. + +Test categories: +- Page Navigation: Basic map viewing +- User Registration: Account creation flow +- Map Interaction: Pan, zoom operations + +Usage: + pytest tests/integration/environments/map/test_playwright.py + pytest tests/integration/environments/map/test_playwright.py --headed +""" + +import pytest +from playwright.sync_api import expect + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_map] + +# ============================================================================= +# Page Navigation Tests +# ============================================================================= + + +@pytest.mark.flaky(reruns=2) +def test_map_homepage_loads(map_container, map_base_url, page, pw_timeout): + """Test that the map homepage loads correctly.""" + page.goto(map_base_url, timeout=pw_timeout) + + # Wait for the map to be visible + expect(page.locator("#map")).to_be_visible(timeout=pw_timeout) + + +@pytest.mark.flaky(reruns=2) +def test_map_login_page_loads(map_container, map_base_url, page, pw_timeout): + """Test that the login page loads correctly.""" + page.goto(f"{map_base_url}/login", timeout=pw_timeout) + + # Verify login form is present + expect(page.get_by_role("heading", name="Log In")).to_be_visible(timeout=pw_timeout) + + +@pytest.mark.flaky(reruns=2) +def test_map_signup_page_loads(map_container, map_base_url, page, pw_timeout): + """Test that the signup page loads correctly.""" + page.goto(f"{map_base_url}/user/new", timeout=pw_timeout) + + # Verify signup form is present + expect(page.get_by_role("heading", name="Sign Up")).to_be_visible(timeout=pw_timeout) + + +# ============================================================================= +# Map Interaction Tests +# ============================================================================= + + +@pytest.mark.flaky(reruns=2) +def test_map_zoom_controls_visible(map_container, map_base_url, page, pw_timeout): + """Test that map zoom controls are visible.""" + page.goto(map_base_url, timeout=pw_timeout) + + # Wait for map to load + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Check zoom controls are present + expect(page.locator(".leaflet-control-zoom-in")).to_be_visible(timeout=pw_timeout) + expect(page.locator(".leaflet-control-zoom-out")).to_be_visible(timeout=pw_timeout) + + +@pytest.mark.flaky(reruns=2) +def test_map_layers_control_visible(map_container, map_base_url, page, pw_timeout): + """Test that map layers control is visible.""" + page.goto(map_base_url, timeout=pw_timeout) + + # Wait for map to load + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Check layers control is present + expect(page.locator(".leaflet-control-layers")).to_be_visible(timeout=pw_timeout) + + +# ============================================================================= +# Search Tests +# ============================================================================= + + +@pytest.mark.flaky(reruns=2) +def test_map_search_box_visible(map_container, map_base_url, page, pw_timeout): + """Test that the search box is visible on the map page.""" + page.goto(map_base_url, timeout=pw_timeout) + + # Wait for map to load + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Check search box is present + search_box = page.locator("input[name='query']") + expect(search_box).to_be_visible(timeout=pw_timeout) + + +# ============================================================================= +# History/Changeset Tests +# ============================================================================= + + +@pytest.mark.flaky(reruns=2) +def test_map_history_page_loads(map_container, map_base_url, page, pw_timeout): + """Test that the history/changesets page loads.""" + page.goto(f"{map_base_url}/history", timeout=pw_timeout) + + # Verify history page loaded + expect(page.get_by_role("heading", name="Changesets")).to_be_visible(timeout=pw_timeout) diff --git a/tests/integration/environments/reddit/__init__.py b/tests/integration/environments/reddit/__init__.py new file mode 100644 index 0000000..3a59fc0 --- /dev/null +++ b/tests/integration/environments/reddit/__init__.py @@ -0,0 +1 @@ +# Reddit (Postmill) integration tests diff --git a/tests/integration/environments/reddit/conftest.py b/tests/integration/environments/reddit/conftest.py new file mode 100644 index 0000000..862416b --- /dev/null +++ b/tests/integration/environments/reddit/conftest.py @@ -0,0 +1,104 @@ +"""Reddit (Postmill) site fixtures.""" + +import pytest + +# Header name for Postmill's autologin feature +AUTOLOGIN_HEADER = "X-Postmill-Auto-Login" + + +@pytest.fixture(scope="session") +def reddit_container(): + """Reddit container name (assumes already running).""" + return "webarena-verified-reddit" + + +@pytest.fixture(scope="session") +def reddit_credentials(): + """Default user credentials for Reddit (Postmill). + + These are the pre-configured credentials in the Docker image, + used for both testing and production environments. + """ + return { + "username": "MarvelsGrantMan136", + "password": "test1234", + } + + +@pytest.fixture(scope="session") +def reddit_base_url(request): + """Base URL for Reddit from CLI arg.""" + url = request.config.getoption("--reddit_url") + if url is None: + raise ValueError("--reddit_url is required") + return url + + +@pytest.fixture(scope="session") +def reddit_env_ctrl_url(request): + """env-ctrl URL for Reddit from CLI arg.""" + url = request.config.getoption("--reddit_env_ctrl_url") + if url is None: + raise ValueError("--reddit_env_ctrl_url is required") + return url + + +@pytest.fixture(scope="session") +def reddit_docker_client(reddit_container, create_docker_client): + """Docker client for Reddit container (via docker exec).""" + return create_docker_client(reddit_container) + + +@pytest.fixture(scope="session") +def reddit_http_client(reddit_env_ctrl_url, create_http_client): + """HTTP client for Reddit env-ctrl server.""" + return create_http_client(reddit_env_ctrl_url) + + +@pytest.fixture(params=[False, True], ids=["form_login", "autologin"]) +def use_autologin(request): + """Parametrized fixture to test with both form login and autologin.""" + return request.param + + +@pytest.fixture +def reddit_get_logged_page(browser, reddit_base_url, reddit_credentials, pw_timeout, use_autologin): + """Factory fixture that returns a function to get a logged-in page. + + Supports both traditional form login and header-based autologin via the + X-Postmill-Auto-Login header, controlled by the use_autologin parameter. + """ + contexts = [] + pages = [] + + def _get_logged_page(): + if use_autologin: + # Create context with autologin header + credentials_header = f"{reddit_credentials['username']}:{reddit_credentials['password']}" + context = browser.new_context(extra_http_headers={AUTOLOGIN_HEADER: credentials_header}) + contexts.append(context) + page = context.new_page() + pages.append(page) + page.goto(reddit_base_url, timeout=pw_timeout) + page.wait_for_selector(f'button:has-text("{reddit_credentials["username"]}")', timeout=pw_timeout) + else: + # Traditional form login + context = browser.new_context() + contexts.append(context) + page = context.new_page() + pages.append(page) + page.goto(f"{reddit_base_url}/login", timeout=pw_timeout) + page.wait_for_selector('input[name="_username"]', timeout=pw_timeout) + page.fill('input[name="_username"]', reddit_credentials["username"]) + page.fill('input[name="_password"]', reddit_credentials["password"]) + page.click('button[type="submit"]') + page.wait_for_selector(f'button:has-text("{reddit_credentials["username"]}")', timeout=pw_timeout) + return page + + yield _get_logged_page + + # Cleanup + for page in pages: + page.close() + for context in contexts: + context.close() diff --git a/tests/integration/environments/reddit/test_basic.py b/tests/integration/environments/reddit/test_basic.py new file mode 100644 index 0000000..1df854b --- /dev/null +++ b/tests/integration/environments/reddit/test_basic.py @@ -0,0 +1,52 @@ +"""Basic HTTP health tests for Reddit (Postmill) Docker container. + +Tests basic connectivity to the Reddit forum without requiring Playwright. + +Usage: + pytest tests/integration/environments/reddit/test_basic.py +""" + +import http.cookiejar +import urllib.request + +import pytest + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_reddit] + + +@pytest.mark.flaky(reruns=2) +def test_reddit_homepage_responds(reddit_container, reddit_base_url): + """Test that homepage responds with 200.""" + req = urllib.request.Request(reddit_base_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 + + +@pytest.mark.flaky(reruns=2) +def test_reddit_homepage_contains_postmill(reddit_container, reddit_base_url): + """Test that homepage contains 'Postmill' branding.""" + req = urllib.request.Request(reddit_base_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + content = response.read().decode("utf-8") + assert "Postmill" in content + + +@pytest.mark.flaky(reruns=2) +def test_reddit_login_page_accessible(reddit_container, reddit_base_url): + """Test that login page is accessible.""" + login_url = f"{reddit_base_url}/login" + # Login page does a cookie check redirect, so we need a cookie jar + cookie_jar = http.cookiejar.CookieJar() + opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cookie_jar)) + req = urllib.request.Request(login_url, method="GET") + with opener.open(req, timeout=10) as response: + assert response.status == 200 + + +@pytest.mark.flaky(reruns=2) +def test_reddit_forums_page_accessible(reddit_container, reddit_base_url): + """Test that forums listing page is accessible.""" + forums_url = f"{reddit_base_url}/forums" + req = urllib.request.Request(forums_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 diff --git a/tests/integration/environments/reddit/test_playwright.py b/tests/integration/environments/reddit/test_playwright.py new file mode 100644 index 0000000..ea527ab --- /dev/null +++ b/tests/integration/environments/reddit/test_playwright.py @@ -0,0 +1,388 @@ +"""Playwright UI tests for Reddit (Postmill) Docker container. + +Tests the Reddit forum functionality using browser automation. + +Test categories: +- User Login: Authentication flow +- Forum Navigation: Navigate to subreddit pages +- Content Verification: Verify forum content exists +- Logged-in User: Profile access +- Image Loading: Post image verification +- Docker Override Tests: Verify fixes from docker_overrides/README.md + - Vote System Fix: Voting increments/decrements scores correctly + - Rate Limit Removal: Multiple submissions can be created quickly + - URL Rewriting: Link submissions with external URLs work correctly + +Usage: + pytest tests/integration/environments/reddit/test_playwright.py + pytest tests/integration/environments/reddit/test_playwright.py --playwright-timeout-sec=60 +""" + +import urllib.request +import uuid + +import pytest +from playwright.sync_api import expect + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_reddit] + +# User Login Test + + +@pytest.mark.flaky(reruns=2) +def test_reddit_user_login(reddit_container, reddit_base_url, reddit_credentials, page, pw_timeout): + """Test user login flow.""" + login_url = f"{reddit_base_url}/login" + + page.goto(login_url) + + # Wait for login form (Postmill uses _username/_password field names) + page.wait_for_selector('input[name="_username"]', timeout=pw_timeout) + + # Fill in credentials + page.fill('input[name="_username"]', reddit_credentials["username"]) + page.fill('input[name="_password"]', reddit_credentials["password"]) + + # Click login button + page.click('button[type="submit"]') + + # Wait for username button in navbar (indicates successful login) + page.wait_for_selector(f'button:has-text("{reddit_credentials["username"]}")', timeout=pw_timeout) + + # Verify logged in (username should appear somewhere) + assert reddit_credentials["username"] in page.content() + + +# Forum Navigation Tests + + +@pytest.mark.flaky(reruns=2) +def test_reddit_navigate_to_askreddit(reddit_container, reddit_base_url, page, pw_timeout): + """Test navigation to AskReddit forum.""" + page.goto(reddit_base_url) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Navigate to AskReddit forum + page.goto(f"{reddit_base_url}/f/AskReddit") + page.wait_for_selector("h1, .forum-title, .submission-list", timeout=pw_timeout) + + # Verify we're on the AskReddit page + content = page.content().lower() + assert "askreddit" in content + + +@pytest.mark.flaky(reruns=2) +def test_reddit_forums_list_has_content(reddit_container, reddit_base_url, page, pw_timeout): + """Test that forums list page has forum entries.""" + page.goto(f"{reddit_base_url}/forums") + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Should have multiple forum links + forum_links = page.locator('a[href*="/f/"]') + assert forum_links.count() > 10, "Expected at least 10 forums" + + +# Content Verification Tests + + +@pytest.mark.flaky(reruns=2) +def test_reddit_has_submissions(reddit_container, reddit_base_url, page, pw_timeout): + """Test that /all page has submission posts.""" + # Homepage defaults to "Featured" filter with no content, so use /all instead + page.goto(f"{reddit_base_url}/all") + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Look for submission entries + submissions = page.locator("article, .submission, .submission-row") + assert submissions.count() > 0, "Expected submissions on /all page" + + +@pytest.mark.flaky(reruns=2) +def test_reddit_askreddit_has_posts(reddit_container, reddit_base_url, page, pw_timeout): + """Test that AskReddit forum has posts.""" + page.goto(f"{reddit_base_url}/f/AskReddit") + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Look for submission entries + submissions = page.locator("article, .submission, .submission-row") + assert submissions.count() > 0, "Expected posts in AskReddit" + + +# Logged-in User Tests + + +@pytest.mark.flaky(reruns=2) +def test_reddit_logged_in_user_profile( + reddit_container, reddit_base_url, reddit_credentials, reddit_get_logged_page, pw_timeout +): + """Test that logged-in user can access their profile.""" + page = reddit_get_logged_page() + + # Navigate to user profile + page.goto(f"{reddit_base_url}/user/{reddit_credentials['username']}") + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Verify username is displayed in the page heading (sidebar) + expect(page.locator("h1.page-heading")).to_contain_text(reddit_credentials["username"]) + + +# Image Loading Tests + +# Posts from different forums to test image loading +# Format: pytest.param(post_url, forum_name, id="image_N") +TEST_POSTS_FOR_IMAGES = [ + pytest.param("/f/aww/58888/lovely-eyes-full-of-love", "aww", id="image_1"), + pytest.param("/f/memes/127582/it-s-out-of-control", "memes", id="image_2"), + pytest.param("/f/Art/10111/a-big-eater-me-pumpkin-carving-2022", "Art", id="image_3"), + pytest.param("/f/photoshopbattles/45340/psbattle-halloween-costume", "photoshopbattles", id="image_4"), + pytest.param("/f/gifs/19936/ok-time-for-you-to-go-to-bed", "gifs", id="image_5"), + pytest.param( + "/f/EarthPorn/98297/2-years-later-this-is-still-one-of-the-most-incredible", "EarthPorn", id="image_6" + ), + pytest.param("/f/OldSchoolCool/35802/twins-photographed-in-1937-and-2012", "OldSchoolCool", id="image_7"), + pytest.param("/f/GetMotivated/55022/image-experience-never-goes-wasted", "GetMotivated", id="image_8"), + pytest.param("/f/pics/45604/a-trejo-thanksgiving", "pics", id="image_9"), + pytest.param("/f/pics/110715/amazing-shot-of-a-blue-jay-pestering-a-bald-eagle", "pics_2", id="image_10"), +] + + +@pytest.mark.flaky(reruns=2) +@pytest.mark.parametrize(("post_url", "forum"), TEST_POSTS_FOR_IMAGES) +def test_post_images_load(reddit_container, reddit_base_url, page, pw_timeout, post_url, forum): + """Test that post images load correctly. + + Verifies that images in Reddit posts are accessible and not broken. + """ + page.goto(f"{reddit_base_url}{post_url}", timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Find post images (Postmill uses .submission__image class) + images = page.locator(".submission__image") + image_count = images.count() + + assert image_count > 0, f"No images found in post from {forum}" + + # Collect image URLs and verify they are accessible + broken_images = [] + for i in range(image_count): + src = images.nth(i).get_attribute("src") + if src and src.startswith("http"): + try: + req = urllib.request.Request(src, method="HEAD") + with urllib.request.urlopen(req, timeout=10) as response: + if response.status != 200: + broken_images.append(f"{src} (status: {response.status})") + else: + # Verify image has content (not empty/placeholder) + content_length = int(response.headers.get("Content-Length", 0)) + if content_length == 0: + broken_images.append(f"{src} (empty content)") + except Exception as e: + broken_images.append(f"{src} (error: {e})") + + assert len(broken_images) == 0, f"Found broken images in {forum}: {broken_images}" + + +# Docker Override Tests +# These tests verify the fixes documented in: +# dev/environments/docker/sites/reddit/docker_overrides/README.md + + +@pytest.mark.flaky(reruns=2) +def test_reddit_vote_increments_score(reddit_container, reddit_base_url, reddit_get_logged_page, pw_timeout): + """Test that voting increments/decrements score correctly. + + Verifies the vote system fix that uses increment/decrement instead of + recalculating from the votes table (which would reset imported scores). + + The original Postmill code recalculates netScore from the votes collection + whenever a vote is added. However, the imported database has net_score values + but no corresponding records in submission_votes tables. This fix ensures: + - First vote on a post increments/decrements the score (not resets to ยฑ1) + - Example: A post with score 38 becomes 39 after upvote (not 1) + """ + page = reddit_get_logged_page() + + # Navigate to a specific post (not the forum listing) to avoid reordering issues + # This post has score 38 in the original dataset + post_url = f"{reddit_base_url}/f/AskReddit/10224/why-bleaching-your-butthole-isn-t-called-changing-your-ring" + page.goto(post_url, timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Find the vote section (there's only one on a single post page) + score_element = page.locator(".vote__net-score").first + upvote_button = page.locator(".vote__up").first + + # Get initial score + initial_score_text = score_element.inner_text(timeout=pw_timeout) + initial_score = int(initial_score_text) + + # Score should be > 1 (imported data has real scores) + assert initial_score > 1, f"Expected score > 1 for imported data, got {initial_score}" + + # Click upvote - this submits a form via POST + upvote_button.click() + + # Wait for score to actually change (the page may reload or use AJAX) + # Use expect to wait for the text to NOT be the initial value + expect(score_element).not_to_have_text(str(initial_score), timeout=pw_timeout) + + # Get new score + new_score_text = score_element.inner_text(timeout=pw_timeout) + new_score = int(new_score_text) + + # The key test: score should change by exactly ยฑ1 or ยฑ2 (if changing vote direction) + # NOT reset to 1/-1 which would indicate the bug + score_diff = abs(new_score - initial_score) + assert score_diff in (1, 2), ( + f"Vote should change score by 1 or 2, not {score_diff}. Initial: {initial_score}, New: {new_score}." + ) + + # Critical: if initial score was > 5 and new score is 1 or -1, the fix is broken + # This catches the bug where voting recalculates from empty votes table + if initial_score > 5: + assert abs(new_score) > 1, ( + f"Score was reset to {new_score} instead of incrementing from {initial_score}. " + "This indicates the vote system fix is not applied - votes are being " + "recalculated from the (empty) votes table instead of incremented." + ) + + +@pytest.mark.flaky(reruns=2) +def test_reddit_no_rate_limit_on_submissions(reddit_container, reddit_base_url, reddit_get_logged_page, pw_timeout): + """Test that submissions can be created without rate limiting. + + Verifies that the @RateLimit annotation was removed from SubmissionData, + allowing multiple submissions to be created in quick succession. + """ + page = reddit_get_logged_page() + created_submissions = [] + + # Create 2 submissions quickly back-to-back + for i in range(2): + page.goto(f"{reddit_base_url}/submit", timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Generate unique title + unique_id = uuid.uuid4().hex[:8] + title = f"Test submission {i + 1} - {unique_id}" + + # Fill in title (Postmill uses textarea for title) + page.fill('textarea[name="submission[title]"]', title) + + # Fill in body (optional but good practice) + page.fill('textarea[name="submission[body]"]', f"This is test submission {i + 1} for rate limit testing.") + + # Select forum using select2 widget + # Click the select2 container to open dropdown + page.click(".select2-selection") + page.wait_for_selector(".select2-search__field", timeout=pw_timeout) + # Type to search for AskReddit + page.fill(".select2-search__field", "AskReddit") + page.wait_for_timeout(300) # Wait for search results + # Click the matching option + page.click(".select2-results__option:has-text('AskReddit')") + + # Submit + page.click('button:has-text("Create submission")') + + # Wait for navigation (successful submission redirects to the new post) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Check we're not on the submit page anymore (no rate limit error) + current_url = page.url + assert "/submit" not in current_url, ( + f"Submission {i + 1} failed - still on submit page. This may indicate rate limiting is still active." + ) + + # Verify the title appears on the page (we're on the new submission) + page_content = page.content() + assert title in page_content or unique_id in page_content, ( + f"Submission {i + 1} title not found on page after submit. URL: {current_url}" + ) + + created_submissions.append(title) + + # Both submissions should have been created successfully + assert len(created_submissions) == 2, "Expected 2 submissions to be created without rate limiting" + + +@pytest.mark.flaky(reruns=2) +def test_reddit_link_submission_with_external_url( + reddit_container, reddit_base_url, reddit_get_logged_page, pw_timeout +): + """Test that link submissions referencing the external port work correctly. + + Verifies the UrlRewritingHttpClient that rewrites external URLs (e.g., + http://localhost:9999/path) to internal localhost (http://localhost/path), + allowing link submissions that would otherwise be blocked by Postmill's + NoPrivateNetworkHttpClient. + + Without this fix, submitting a link to the same site on the external port + would fail because Postmill blocks requests to localhost/private IPs. + """ + page = reddit_get_logged_page() + + page.goto(f"{reddit_base_url}/submit", timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Generate unique title + unique_id = uuid.uuid4().hex[:8] + title = f"Link test - {unique_id}" + + # Ensure URL radio is selected (should be default) + url_radio = page.locator('input[type="radio"][value="url"]') + if url_radio.count() > 0 and not url_radio.is_checked(): + url_radio.click() + + # Fill in URL pointing to the external port (this tests the URL rewriting) + # We use a URL to an existing post on the same site + external_url = f"{reddit_base_url}/f/AskReddit/10224/why-bleaching-your-butthole-isn-t-called-changing-your-ring" + page.fill('input[name="submission[url]"]', external_url) + + # Fill in title (Postmill uses textarea for title) + page.fill('textarea[name="submission[title]"]', title) + + # Select forum using select2 widget + page.click(".select2-selection") + page.wait_for_selector(".select2-search__field", timeout=pw_timeout) + page.fill(".select2-search__field", "AskReddit") + page.wait_for_timeout(300) # Wait for search results + page.click(".select2-results__option:has-text('AskReddit')") + + # Submit + page.click('button:has-text("Create submission")') + + # Wait for result + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + current_url = page.url + page_content = page.content() + + # Check for common error messages that would indicate the URL rewriting failed + error_indicators = [ + "could not fetch", + "private network", + "blocked", + "invalid url", + "connection refused", + ] + page_content_lower = page_content.lower() + for error in error_indicators: + assert error not in page_content_lower, ( + f"Found error indicator '{error}' in page. URL rewriting may not be working correctly." + ) + + # Verify we're not still on the submit page with an error + # A successful submission redirects to the new post + if "/submit" in current_url: + # Still on submit page - check if there's a form error + error_elements = page.locator(".form-error, .error, .alert-danger, .flash-error") + if error_elements.count() > 0: + error_text = error_elements.first.inner_text() + pytest.fail(f"Submission failed with error: {error_text}. This may indicate URL rewriting is not working.") + + # Verify the submission was created (title should appear on page) + assert title in page_content or unique_id in page_content, ( + f"Link submission title not found. URL: {current_url}. The submission may have failed due to URL blocking." + ) diff --git a/tests/integration/environments/shopping/__init__.py b/tests/integration/environments/shopping/__init__.py new file mode 100644 index 0000000..4f26ae8 --- /dev/null +++ b/tests/integration/environments/shopping/__init__.py @@ -0,0 +1 @@ +# Shopping (storefront) integration tests diff --git a/tests/integration/environments/shopping/conftest.py b/tests/integration/environments/shopping/conftest.py new file mode 100644 index 0000000..ae38a08 --- /dev/null +++ b/tests/integration/environments/shopping/conftest.py @@ -0,0 +1,104 @@ +"""Shopping (storefront) site fixtures.""" + +import pytest + +# Constants +AUTOLOGIN_HEADER = "X-M2-Customer-Auto-Login" + + +@pytest.fixture(scope="session") +def shopping_credentials(): + """Default customer credentials for shopping storefront. + + These are the pre-configured credentials in the Docker image, + used for both testing and production environments. + """ + return { + "email": "emma.lopez@gmail.com", + "password": "Password.123", + } + + +@pytest.fixture(scope="session") +def shopping_container(): + """Shopping container name (assumes already running).""" + return "webarena-verified-shopping" + + +@pytest.fixture(scope="session") +def shopping_base_url(request): + """Base URL for shopping storefront from CLI arg.""" + url = request.config.getoption("--shopping_url") + if url is None: + raise ValueError("--shopping_url is required") + return url + + +@pytest.fixture(scope="session") +def shopping_env_ctrl_url(request): + """env-ctrl URL for shopping from CLI arg.""" + url = request.config.getoption("--shopping_env_ctrl_url") + if url is None: + raise ValueError("--shopping_env_ctrl_url is required") + return url + + +@pytest.fixture(scope="session") +def shopping_docker_client(shopping_container, create_docker_client): + """Docker client for shopping container (via docker exec).""" + return create_docker_client(shopping_container) + + +@pytest.fixture(scope="session") +def shopping_http_client(shopping_env_ctrl_url, create_http_client): + """HTTP client for shopping env-ctrl server.""" + return create_http_client(shopping_env_ctrl_url) + + +@pytest.fixture(params=[False, True], ids=["form_login", "autologin"]) +def use_autologin(request): + """Parametrized fixture to test with both form login and autologin.""" + return request.param + + +@pytest.fixture +def shopping_get_logged_page(browser, shopping_base_url, shopping_credentials, pw_timeout, use_autologin): + """Factory fixture that returns a function to get a logged-in customer page. + + Supports both traditional form login and header-based autologin via the + X-M2-Customer-Auto-Login header, controlled by the use_autologin parameter. + """ + contexts = [] + pages = [] + + def _get_logged_page(): + if use_autologin: + # Create context with autologin header + credentials_header = f"{shopping_credentials['email']}:{shopping_credentials['password']}" + context = browser.new_context(extra_http_headers={AUTOLOGIN_HEADER: credentials_header}) + contexts.append(context) + page = context.new_page() + pages.append(page) + page.goto(f"{shopping_base_url}/customer/account", timeout=pw_timeout) + page.wait_for_selector(".block-dashboard-info, .box-information", timeout=pw_timeout) + else: + # Traditional form login + context = browser.new_context() + contexts.append(context) + page = context.new_page() + pages.append(page) + page.goto(f"{shopping_base_url}/customer/account/login", timeout=pw_timeout) + page.wait_for_selector('input[name="login[username]"]', timeout=pw_timeout) + page.fill('input[name="login[username]"]', shopping_credentials["email"]) + page.fill('input[name="login[password]"]', shopping_credentials["password"]) + page.click('button[type="submit"].action.login') + page.wait_for_selector(".block-dashboard-info, .box-information", timeout=pw_timeout) + return page + + yield _get_logged_page + + # Cleanup + for page in pages: + page.close() + for context in contexts: + context.close() diff --git a/tests/integration/environments/shopping/test_basic.py b/tests/integration/environments/shopping/test_basic.py new file mode 100644 index 0000000..f3fe0df --- /dev/null +++ b/tests/integration/environments/shopping/test_basic.py @@ -0,0 +1,43 @@ +"""Basic HTTP health tests for Shopping Docker container. + +Tests basic connectivity to the Magento storefront without requiring Playwright. + +Usage: + pytest tests/integration/environments/shopping/test_basic.py +""" + +import urllib.request + +import pytest + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_shopping] + +# Known URLs in the shopping dataset (One Stop Market) +TEST_PRODUCT_URL = "toothbrush-head-cover-toothbrush-protective-case-toothbrush-head-cap-for-home-travel-camping-lightweight-safe-protecting-toothbrush-head-light-blue.html" +TEST_CATEGORY_URL = "beauty-personal-care.html" + + +@pytest.mark.flaky(reruns=2) +def test_shopping_homepage_responds(shopping_container, shopping_base_url): + """Test that homepage responds with 200.""" + req = urllib.request.Request(shopping_base_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 + + +@pytest.mark.flaky(reruns=2) +def test_shopping_product_page_responds(shopping_container, shopping_base_url): + """Test that a product page responds.""" + product_url = f"{shopping_base_url}/{TEST_PRODUCT_URL}" + req = urllib.request.Request(product_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 + + +@pytest.mark.flaky(reruns=2) +def test_shopping_category_page_responds(shopping_container, shopping_base_url): + """Test that a category page responds.""" + category_url = f"{shopping_base_url}/{TEST_CATEGORY_URL}" + req = urllib.request.Request(category_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 diff --git a/tests/integration/environments/shopping/test_playwright.py b/tests/integration/environments/shopping/test_playwright.py new file mode 100644 index 0000000..189f521 --- /dev/null +++ b/tests/integration/environments/shopping/test_playwright.py @@ -0,0 +1,241 @@ +"""Playwright UI tests for Shopping Docker container. + +Tests the Magento storefront functionality using browser automation. + +Test categories: +- Customer Login: Authentication flow +- Storefront Navigation: Category and product browsing +- Search: Product search functionality +- Cart: Add to cart functionality +- Image Loading: Product image verification + +Usage: + pytest tests/integration/environments/shopping/test_playwright.py + pytest tests/integration/environments/shopping/test_playwright.py --playwright-timeout-sec=60 +""" + +import urllib.request + +import pytest +from playwright.sync_api import expect + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_shopping] + +# Magento placeholder image size in bytes (used to detect missing images) +PLACEHOLDER_IMAGE_SIZE = 1692 + +# Known products in the shopping dataset (One Stop Market) +TEST_PRODUCT_URL = "toothbrush-head-cover-toothbrush-protective-case-toothbrush-head-cap-for-home-travel-camping-lightweight-safe-protecting-toothbrush-head-light-blue.html" +TEST_PRODUCT_NAME = "Toothbrush Head Cover" +TEST_CATEGORY_URL = "beauty-personal-care.html" +TEST_SEARCH_TERM = "toothbrush" + + +# Customer Login Tests + + +@pytest.mark.flaky(reruns=2) +def test_customer_login(shopping_container, shopping_base_url, shopping_credentials, page, pw_timeout): + """Test customer login flow on storefront.""" + login_url = f"{shopping_base_url}/customer/account/login" + + page.goto(login_url, timeout=pw_timeout) + + # Wait for login form + page.wait_for_selector('input[name="login[username]"]', timeout=pw_timeout) + + # Fill in credentials + page.fill('input[name="login[username]"]', shopping_credentials["email"]) + page.fill('input[name="login[password]"]', shopping_credentials["password"]) + + # Click login button + page.click('button[type="submit"].action.login') + + # Wait for account page to load + page.wait_for_selector(".block-dashboard-info, .box-information", timeout=pw_timeout) + + # Verify we're on the account page + assert "My Account" in page.content() or "Account Information" in page.content() + + +@pytest.mark.flaky(reruns=2) +def test_customer_account_page(shopping_container, shopping_base_url, shopping_get_logged_page, pw_timeout): + """Test customer account page shows correct information.""" + page = shopping_get_logged_page() + + # Navigate to account page + page.goto(f"{shopping_base_url}/customer/account/", timeout=pw_timeout) + + # Verify account information is visible + account_info = page.locator(".box-information, .block-dashboard-info") + expect(account_info.first).to_be_visible(timeout=pw_timeout) + + +# Storefront Navigation Tests + + +@pytest.mark.flaky(reruns=2) +def test_shopping_homepage_loads(shopping_container, shopping_base_url, shopping_get_logged_page, pw_timeout): + """Test that homepage loads with header (logged in).""" + page = shopping_get_logged_page() + page.goto(shopping_base_url, timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Check that the page has loaded (look for header or logo) + header = page.locator("header.page-header") + expect(header).to_be_visible(timeout=pw_timeout) + + +@pytest.mark.flaky(reruns=2) +def test_shopping_category_page_loads(shopping_container, shopping_base_url, shopping_get_logged_page, pw_timeout): + """Test that category page loads (logged in).""" + page = shopping_get_logged_page() + page.goto(f"{shopping_base_url}/{TEST_CATEGORY_URL}", timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Category heading should be visible + heading = page.locator("h1.page-title span, h1.page-title") + expect(heading.first).to_be_visible(timeout=pw_timeout) + + +@pytest.mark.flaky(reruns=2) +def test_shopping_product_detail_page(shopping_container, shopping_base_url, shopping_get_logged_page, pw_timeout): + """Test product detail page loads correctly (logged in).""" + page = shopping_get_logged_page() + page.goto(f"{shopping_base_url}/{TEST_PRODUCT_URL}", timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Verify product title is visible + title = page.locator("h1.page-title span") + expect(title).to_be_visible(timeout=pw_timeout) + expect(title).to_contain_text(TEST_PRODUCT_NAME) + + # Verify price is visible + price = page.locator(".product-info-price .price") + expect(price).to_be_visible(timeout=pw_timeout) + + +# Search Tests + + +@pytest.mark.flaky(reruns=2) +def test_shopping_search_page_loads(shopping_container, shopping_base_url, shopping_get_logged_page, pw_timeout): + """Test that search page loads (logged in).""" + page = shopping_get_logged_page() + page.goto(f"{shopping_base_url}/catalogsearch/result/?q={TEST_SEARCH_TERM}", timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Search page title should be visible + title = page.locator("h1.page-title span, .page-title-wrapper h1") + expect(title.first).to_be_visible(timeout=pw_timeout) + + +# Cart Tests + + +@pytest.mark.flaky(reruns=2) +def test_shopping_product_has_add_to_cart(shopping_container, shopping_base_url, shopping_get_logged_page, pw_timeout): + """Test that in-stock product page has add to cart button (logged in).""" + page = shopping_get_logged_page() + page.goto(f"{shopping_base_url}/{TEST_PRODUCT_URL}", timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Verify product is in stock + availability = page.locator('[data-th="Availability"], .stock.available, .availability') + expect(availability.first).to_contain_text("In stock", timeout=pw_timeout) + + # Verify add to cart button is present (selector varies by theme) + add_to_cart = page.locator("button.tocart, button:has-text('Add to Cart')") + expect(add_to_cart.first).to_be_visible(timeout=pw_timeout) + + # Verify quantity input is present + qty_input = page.locator("input#qty, input[name='qty']") + expect(qty_input.first).to_be_visible(timeout=pw_timeout) + + +# Image Loading Tests + +# Products from different categories to test image loading +# Format: pytest.param(product_url, category_name, id="image_N") +TEST_PRODUCTS_FOR_IMAGES = [ + pytest.param(TEST_PRODUCT_URL, "Beauty & Personal Care", id="image_1"), + pytest.param("toolworx-precision-cut-pro-toenail-nipper.html", "Foot, Hand & Nail Care", id="image_2"), + pytest.param( + "hazel-nintendo-animal-crossing-happy-home-designer-amiibo-card-248.html", "Nintendo Systems", id="image_3" + ), + pytest.param( + "powered-toothbrush-sets-with-your-favorite-star-wars-characters-3-piece-chewbacca.html", + "Children's Dental Care", + id="image_4", + ), + pytest.param( + "stainless-steel-tongue-scraper-portable-freshen-breath-tongue-cleaner-oral-care-tool-tongue-scraper-stainless-steel-tongue-scraper-tongue-cleaner-freshen-breath.html", + "Tongue Cleaners", + id="image_5", + ), + pytest.param("de-chanceny-cremant-de-loire-rose-375ml.html", "Alcoholic Beverages", id="image_6"), + pytest.param( + "afuower-nail-polish-organizer-bag-with-handles-holds-30-bottles-15ml-0-5-fl-oz-travel-case-portable-storage-bag-for-manicure-set-pink.html", + "Nail Art & Polish", + id="image_7", + ), + pytest.param( + "dimj-closet-organizer-storage-washable-storage-basket-for-shelves-large-closet-storage-bins-collapsible-rectangular-fabric-basket-for-clothes-toys-home-beige-grey-large.html", + "Baskets, Bins & Containers", + id="image_8", + ), + pytest.param( + "charger-cable-for-ns-switch-and-switch-lite-2-pack-10ft-nylon-braided-usb-c-to-usb-a-type-c-fast-data-sync-power-charging-cord-accessories-for-samsung-galaxy-s9-s8-note-9-pixel-lg-v30-g6-oneplus.html", + "Nintendo Switch", + id="image_9", + ), + pytest.param( + "emergency-solar-power-radio-with-4000mah-battery-hand-crank-survival-radio-with-flashlight-cell-phone-charger-sos-alarm-survival-whistle-for-snowstorm-hiking-picnic-camping-outdoor-activities-light.html", + "Radios", + id="image_10", + ), +] + + +@pytest.mark.flaky(reruns=2) +@pytest.mark.parametrize(("product_url", "category"), TEST_PRODUCTS_FOR_IMAGES) +def test_product_images_load( + shopping_container, shopping_base_url, shopping_get_logged_page, pw_timeout, product_url, category +): + """Test that product images load correctly (not placeholders). + + Magento serves a 1692-byte placeholder image when the actual image is missing. + This test verifies that product images are real images, not placeholders. + """ + page = shopping_get_logged_page() + page.goto(f"{shopping_base_url}/{product_url}", timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Get all product gallery images + images = page.locator(".product.media img, .gallery-placeholder img, .fotorama__img") + image_count = images.count() + + assert image_count > 0, f"No product images found on page for {category}" + + # Collect image URLs (filter out data URIs and empty src) + image_urls = [] + for i in range(image_count): + src = images.nth(i).get_attribute("src") + if src and src.startswith("http") and "placeholder" not in src.lower(): + image_urls.append(src) + + assert len(image_urls) > 0, f"No valid product image URLs found for {category}" + + # Verify each image is not a placeholder (check Content-Length) + placeholder_images = [] + for url in image_urls: + try: + req = urllib.request.Request(url, method="HEAD") + with urllib.request.urlopen(req, timeout=10) as response: + content_length = int(response.headers.get("Content-Length", 0)) + if content_length == PLACEHOLDER_IMAGE_SIZE: + placeholder_images.append(url) + except Exception as e: + placeholder_images.append(f"{url} (error: {e})") + + assert len(placeholder_images) == 0, f"Found placeholder/missing images in {category}: {placeholder_images}" diff --git a/tests/integration/environments/shopping_admin/__init__.py b/tests/integration/environments/shopping_admin/__init__.py new file mode 100644 index 0000000..022853c --- /dev/null +++ b/tests/integration/environments/shopping_admin/__init__.py @@ -0,0 +1 @@ +# Shopping Admin integration tests diff --git a/tests/integration/environments/shopping_admin/conftest.py b/tests/integration/environments/shopping_admin/conftest.py new file mode 100644 index 0000000..667d763 --- /dev/null +++ b/tests/integration/environments/shopping_admin/conftest.py @@ -0,0 +1,166 @@ +"""Shopping Admin site fixtures.""" + +import pytest + +# Constants +AUTOLOGIN_HEADER = "X-M2-Admin-Auto-Login" + + +@pytest.fixture(scope="session") +def shopping_admin_credentials(): + """Default admin credentials for Magento backend. + + These are the pre-configured credentials in the Docker image, + used for both testing and production environments. + """ + return { + "username": "admin", + "password": "admin1234", + } + + +@pytest.fixture(scope="session") +def shopping_admin_container(): + """Shopping admin container name (assumes already running).""" + return "webarena-verified-shopping_admin" + + +@pytest.fixture(scope="session") +def shopping_admin_base_url(request): + """Base URL for admin panel from CLI arg.""" + url = request.config.getoption("--shopping_admin_url") + if url is None: + raise ValueError("--shopping_admin_url is required") + return url + + +@pytest.fixture(scope="session") +def shopping_admin_env_ctrl_url(request): + """env-ctrl URL for shopping admin from CLI arg.""" + url = request.config.getoption("--shopping_admin_env_ctrl_url") + if url is None: + raise ValueError("--shopping_admin_env_ctrl_url is required") + return url + + +@pytest.fixture(scope="session") +def shopping_admin_docker_client(shopping_admin_container, create_docker_client): + """Docker client for shopping admin container (via docker exec).""" + return create_docker_client(shopping_admin_container) + + +@pytest.fixture(scope="session") +def shopping_admin_http_client(shopping_admin_env_ctrl_url, create_http_client): + """HTTP client for shopping admin env-ctrl server.""" + return create_http_client(shopping_admin_env_ctrl_url) + + +@pytest.fixture(params=[False, True], ids=["form_login", "autologin"]) +def use_autologin(request): + """Parametrized fixture to test with both form login and autologin.""" + return request.param + + +@pytest.fixture +def shopping_admin_get_logged_page( + browser, shopping_admin_base_url, shopping_admin_credentials, pw_timeout, use_autologin +): + """Factory fixture that returns a function to get a logged-in admin page. + + Supports both traditional form login and header-based autologin via the + X-M2-Admin-Auto-Login header, controlled by the use_autologin parameter. + """ + contexts = [] + pages = [] + + def _get_logged_page(): + if use_autologin: + # Create context with autologin header + credentials_header = f"{shopping_admin_credentials['username']}:{shopping_admin_credentials['password']}" + context = browser.new_context(extra_http_headers={AUTOLOGIN_HEADER: credentials_header}) + contexts.append(context) + page = context.new_page() + pages.append(page) + page.goto(f"{shopping_admin_base_url}/admin") + page.wait_for_selector(".page-title", timeout=pw_timeout) + else: + # Traditional form login + context = browser.new_context() + contexts.append(context) + page = context.new_page() + pages.append(page) + page.goto(f"{shopping_admin_base_url}/admin") + page.wait_for_selector('input[name="login[username]"]', timeout=pw_timeout) + page.fill('input[name="login[username]"]', shopping_admin_credentials["username"]) + page.fill('input[name="login[password]"]', shopping_admin_credentials["password"]) + page.click("button.action-login") + page.wait_for_selector(".page-title", timeout=pw_timeout) + return page + + yield _get_logged_page + + # Cleanup + for page in pages: + page.close() + for context in contexts: + context.close() + + +# Default product ID for modification tests +PRODUCT_ID = 1 + + +@pytest.fixture +def get_test_product(shopping_admin_base_url, pw_timeout): + """Factory fixture to get test product info with optional reset. + + Captures the original product name when called and resets to it after the test. + + Usage: + def test_something(shopping_admin_get_logged_page, get_test_product): + page = shopping_admin_get_logged_page() + product = get_test_product(page) # returns {"id": 1, "name": "...", "original_name": "..."} + # ... test logic ... + # product name auto-resets to original_name after test + + def test_readonly(shopping_admin_get_logged_page, get_test_product): + page = shopping_admin_get_logged_page() + product = get_test_product(page, no_reset=True) # no reset needed + """ + products_to_reset = [] # Track (page, product_id, original_name) for teardown + + def _get_test_product(page, product_id=PRODUCT_ID, no_reset=False): + # Navigate to product edit page + page.goto(f"{shopping_admin_base_url}/admin/catalog/product/edit/id/{product_id}/", timeout=pw_timeout) + page.wait_for_selector('input[name="product[name]"]', timeout=pw_timeout) + + # Capture original name dynamically + name_input = page.locator('input[name="product[name]"]') + original_name = name_input.input_value() + + if not no_reset: + products_to_reset.append((page, product_id, original_name)) + + return { + "id": product_id, + "name": original_name, + "original_name": original_name, + } + + yield _get_test_product + + # Teardown: reset all products to their original names + for page, product_id, original_name in products_to_reset: + try: + page.goto(f"{shopping_admin_base_url}/admin/catalog/product/edit/id/{product_id}/", timeout=pw_timeout) + page.wait_for_selector('input[name="product[name]"]', timeout=pw_timeout) + + name_input = page.locator('input[name="product[name]"]') + current_name = name_input.input_value() + + if current_name != original_name: + name_input.fill(original_name) + page.click("button#save-button, button:has-text('Save')") + page.wait_for_selector("text=You saved the product", timeout=pw_timeout) + except Exception: + pass # Best effort cleanup diff --git a/tests/integration/environments/shopping_admin/test_basic.py b/tests/integration/environments/shopping_admin/test_basic.py new file mode 100644 index 0000000..467da71 --- /dev/null +++ b/tests/integration/environments/shopping_admin/test_basic.py @@ -0,0 +1,31 @@ +"""Basic HTTP health tests for Shopping Admin Docker container. + +Tests basic connectivity to the Magento admin panel without requiring Playwright. + +Usage: + pytest tests/integration/environments/shopping_admin/test_basic.py +""" + +import urllib.request + +import pytest + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_shopping_admin] + + +@pytest.mark.flaky(reruns=2) +def test_homepage_responds(shopping_admin_container, shopping_admin_base_url): + """Test that homepage responds with 200.""" + req = urllib.request.Request(shopping_admin_base_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 + + +@pytest.mark.flaky(reruns=2) +def test_admin_panel_accessible(shopping_admin_container, shopping_admin_base_url): + """Test that admin panel is accessible.""" + admin_url = f"{shopping_admin_base_url}/admin" + req = urllib.request.Request(admin_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + # Admin panel may redirect to login, but should respond + assert response.status == 200 diff --git a/tests/integration/environments/shopping_admin/test_playwright.py b/tests/integration/environments/shopping_admin/test_playwright.py new file mode 100644 index 0000000..7a59ed2 --- /dev/null +++ b/tests/integration/environments/shopping_admin/test_playwright.py @@ -0,0 +1,153 @@ +"""Playwright UI tests for Shopping Admin Docker container. + +Tests the Magento admin panel functionality using browser automation. + +Test categories: +- Admin Login: Authentication flow +- Admin Navigation: Menu navigation to Products and Customers pages +- Admin Reports: Sales report generation with date filtering +- Product Modification: Edit product name and verify persistence in admin + +Usage: + pytest tests/integration/environments/shopping_admin/test_playwright.py + pytest tests/integration/environments/shopping_admin/test_playwright.py --playwright-timeout-sec=60 +""" + +import time + +import pytest +from playwright.sync_api import expect + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_shopping_admin] + +# Admin Login Test + + +@pytest.mark.flaky(reruns=2) +def test_admin_login(shopping_admin_container, shopping_admin_base_url, shopping_admin_credentials, page, pw_timeout): + """Test admin login flow.""" + admin_url = f"{shopping_admin_base_url}/admin" + + page.goto(admin_url) + + # Wait for login form + page.wait_for_selector('input[name="login[username]"]', timeout=pw_timeout) + + # Fill in credentials + page.fill('input[name="login[username]"]', shopping_admin_credentials["username"]) + page.fill('input[name="login[password]"]', shopping_admin_credentials["password"]) + + # Click login button + page.click("button.action-login") + + # Wait for dashboard to load + page.wait_for_selector(".page-title", timeout=pw_timeout) + + # Verify we're on the dashboard + assert "Dashboard" in page.content() + + +# Admin Navigation Tests + + +@pytest.mark.flaky(reruns=2) +def test_deep_link_navigation( + shopping_admin_container, shopping_admin_base_url, shopping_admin_get_logged_page, pw_timeout +): + """Test navigation to a deep link (Products page).""" + page = shopping_admin_get_logged_page() + + # Navigate to Products (Catalog > Products) + page.click("#menu-magento-catalog-catalog") + page.click('a[href*="catalog/product"]') + + # Wait for products page + page.wait_for_selector(".page-title", timeout=pw_timeout) + + # Verify we're on the products page + assert "Products" in page.content() + + +@pytest.mark.flaky(reruns=2) +def test_navigate_to_customers( + shopping_admin_container, shopping_admin_base_url, shopping_admin_get_logged_page, pw_timeout +): + """Test navigation to Customers page.""" + page = shopping_admin_get_logged_page() + + # Navigate to Customers > All Customers + page.click("#menu-magento-customer-customer") + page.click('a[href*="customer/index"]') + + # Wait for customers page + page.wait_for_selector(".page-title", timeout=pw_timeout) + + # Verify we're on the customers page + assert "Customers" in page.content() + + +# Admin Reports Tests + + +@pytest.mark.flaky(reruns=2) +def test_generate_report(shopping_admin_container, shopping_admin_base_url, shopping_admin_get_logged_page, pw_timeout): + """Test generating a sales report with date range.""" + page = shopping_admin_get_logged_page() + + # Navigate directly to Orders Report page + page.goto(f"{shopping_admin_base_url}/admin/reports/report_sales/sales/", timeout=pw_timeout) + page.wait_for_selector("h1", timeout=pw_timeout) + + expect(page.locator("h1")).to_contain_text("Orders Report") + + # Fill date range + page.fill('input[name="from"]', "01/01/2023") + page.fill('input[name="to"]', "12/31/2023") + + # Generate report + page.click("button:has-text('Show Report')") + + # Wait for table with data rows + page.wait_for_selector("table tbody tr", timeout=pw_timeout) + + # Verify report was generated (check for records text or table presence) + records_el = page.locator("text=/\\d+ records found/") + if records_el.count() > 0: + assert records_el.first.is_visible() + else: + # At minimum, table should be loaded + assert page.locator("table tbody tr").count() > 0 + + +# Admin Product Modification Tests + + +@pytest.mark.flaky(reruns=2) +def test_modify_product( + shopping_admin_container, shopping_admin_base_url, shopping_admin_get_logged_page, get_test_product, pw_timeout +): + """Test modifying product name and verifying persistence in admin.""" + page = shopping_admin_get_logged_page() + product = get_test_product(page) # Navigates to product, captures original name, will reset after + + # Update name with timestamp + timestamp = int(time.time()) + test_name = f"{timestamp} {product['original_name']}" + name_input = page.locator('input[name="product[name]"]') + name_input.fill(test_name) + + # Save + page.click("button#save-button, button:has-text('Save')") + page.wait_for_selector("text=You saved the product", timeout=pw_timeout) + + # Verify in admin by opening product in new tab + verify_page = page.context.new_page() + verify_page.goto(f"{shopping_admin_base_url}/admin/catalog/product/edit/id/{product['id']}/", timeout=pw_timeout) + verify_page.wait_for_selector('input[name="product[name]"]', timeout=pw_timeout) + + # Check the product name field contains the timestamp + saved_name = verify_page.locator('input[name="product[name]"]').input_value() + assert str(timestamp) in saved_name, f"Expected timestamp {timestamp} in saved name '{saved_name}'" + verify_page.close() + + # No manual revert needed - fixture resets to original_name automatically diff --git a/tests/integration/environments/test_env_ctrl.py b/tests/integration/environments/test_env_ctrl.py new file mode 100644 index 0000000..7f24851 --- /dev/null +++ b/tests/integration/environments/test_env_ctrl.py @@ -0,0 +1,72 @@ +"""Consolidated env-ctrl tests for all sites. + +Tests both Docker exec and HTTP API interfaces for each site's env-ctrl. + +Usage: + # Run all env-ctrl tests + pytest tests/integration/environments/test_env_ctrl.py + + # Run env-ctrl tests for specific site only + pytest tests/integration/environments/test_env_ctrl.py -m integration_docker_shopping + + # Run with verbose output + pytest tests/integration/environments/test_env_ctrl.py -v +""" + +import pytest + +pytestmark = pytest.mark.docker + +# Site configurations for parametrization +# Each pytest.param attaches a marker so tests can be filtered by site +SITES = [ + pytest.param("shopping", marks=pytest.mark.integration_docker_shopping), + pytest.param("shopping_admin", marks=pytest.mark.integration_docker_shopping_admin), + pytest.param("reddit", marks=pytest.mark.integration_docker_reddit), + pytest.param("gitlab", marks=pytest.mark.integration_docker_gitlab), + pytest.param("wikipedia", marks=pytest.mark.integration_docker_wikipedia), + pytest.param("map", marks=pytest.mark.integration_docker_map), +] + + +@pytest.fixture +def docker_client(request, create_docker_client): + """Create Docker client for the parametrized site.""" + site = request.param + container_name = f"webarena-verified-{site}" + return create_docker_client(container_name) + + +@pytest.fixture +def http_client(request, create_http_client): + """Create HTTP client for the parametrized site. + + Uses --{site}_env_ctrl_url CLI argument (e.g., --shopping_env_ctrl_url). + """ + site = request.param + env_ctrl_url = request.config.getoption(f"--{site}_env_ctrl_url") + if env_ctrl_url is None: + raise ValueError(f"--{site}_env_ctrl_url not specified") + return create_http_client(env_ctrl_url) + + +# --- Docker Client Tests --- + + +@pytest.mark.flaky(reruns=2) +@pytest.mark.parametrize("docker_client", SITES, indirect=True) +def test_env_ctrl_docker_status(docker_client): + """Test env-ctrl status via docker exec.""" + result = docker_client.status() + assert result.success, f"Docker client status failed: {result.message}" + + +# --- HTTP Client Tests --- + + +@pytest.mark.flaky(reruns=2) +@pytest.mark.parametrize("http_client", SITES, indirect=True) +def test_env_ctrl_http_status(http_client): + """Test env-ctrl status via HTTP API.""" + result = http_client.status() + assert result.get("success"), f"HTTP client status failed: {result.get('message')}" diff --git a/tests/integration/environments/wikipedia/__init__.py b/tests/integration/environments/wikipedia/__init__.py new file mode 100644 index 0000000..7a2d312 --- /dev/null +++ b/tests/integration/environments/wikipedia/__init__.py @@ -0,0 +1 @@ +# Wikipedia integration tests diff --git a/tests/integration/environments/wikipedia/conftest.py b/tests/integration/environments/wikipedia/conftest.py new file mode 100644 index 0000000..b64ad43 --- /dev/null +++ b/tests/integration/environments/wikipedia/conftest.py @@ -0,0 +1,39 @@ +"""Wikipedia site fixtures.""" + +import pytest + + +@pytest.fixture(scope="session") +def wikipedia_base_url(request): + """Base URL for Wikipedia from CLI arg.""" + url = request.config.getoption("--wikipedia_url") + if url is None: + raise ValueError("--wikipedia_url is required") + return url + + +@pytest.fixture(scope="session") +def wikipedia_container(): + """Wikipedia container name (assumes already running).""" + return "webarena-verified-wikipedia" + + +@pytest.fixture(scope="session") +def wikipedia_docker_client(wikipedia_container, create_docker_client): + """Docker client for Wikipedia container (via docker exec).""" + return create_docker_client(wikipedia_container) + + +@pytest.fixture(scope="session") +def wikipedia_env_ctrl_url(request): + """env-ctrl URL for Wikipedia from CLI arg.""" + url = request.config.getoption("--wikipedia_env_ctrl_url") + if url is None: + raise ValueError("--wikipedia_env_ctrl_url is required") + return url + + +@pytest.fixture(scope="session") +def wikipedia_http_client(wikipedia_env_ctrl_url, create_http_client): + """HTTP client for Wikipedia env-ctrl server.""" + return create_http_client(wikipedia_env_ctrl_url) diff --git a/tests/integration/environments/wikipedia/test_basic.py b/tests/integration/environments/wikipedia/test_basic.py new file mode 100644 index 0000000..f999e6c --- /dev/null +++ b/tests/integration/environments/wikipedia/test_basic.py @@ -0,0 +1,21 @@ +"""Basic HTTP health tests for Wikipedia Docker container. + +Tests basic connectivity to the kiwix-serve Wikipedia without requiring Playwright. + +Usage: + pytest tests/integration/environments/wikipedia/test_basic.py +""" + +import urllib.request + +import pytest + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_wikipedia] + + +@pytest.mark.flaky(reruns=2) +def test_wikipedia_homepage_responds(wikipedia_container, wikipedia_base_url): + """Test that Wikipedia homepage responds with 200 (auto-redirects to landing).""" + req = urllib.request.Request(wikipedia_base_url, method="GET") + with urllib.request.urlopen(req, timeout=10) as response: + assert response.status == 200 diff --git a/tests/integration/environments/wikipedia/test_playwright.py b/tests/integration/environments/wikipedia/test_playwright.py new file mode 100644 index 0000000..f4929b2 --- /dev/null +++ b/tests/integration/environments/wikipedia/test_playwright.py @@ -0,0 +1,66 @@ +"""Playwright UI tests for Wikipedia Docker container. + +Tests the kiwix-serve Wikipedia functionality using browser automation. +Works with both the full Wikipedia ZIM and the small Ray Charles CI ZIM. + +Usage: + pytest tests/integration/environments/wikipedia/test_playwright.py + pytest tests/integration/environments/wikipedia/test_playwright.py --playwright-timeout-sec=60 +""" + +import pytest + +pytestmark = [pytest.mark.docker, pytest.mark.integration_docker_wikipedia] + + +@pytest.mark.flaky(reruns=2) +def test_wikipedia_landing_loads(wikipedia_container, wikipedia_base_url, page, pw_timeout): + """Test that Wikipedia/Kiwix landing page loads.""" + page.goto(wikipedia_base_url, timeout=pw_timeout) + + # Verify we're on the kiwix landing page (works with any ZIM) + content = page.content().lower() + assert "kiwix" in content or "wikipedia" in content or "ray charles" in content + + +@pytest.mark.flaky(reruns=2) +def test_wikipedia_article_accessible(wikipedia_container, wikipedia_base_url, page, pw_timeout): + """Test that we can access article content.""" + page.goto(wikipedia_base_url, timeout=pw_timeout) + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Should have some content loaded + content = page.content() + assert len(content) > 100, "Page should have content" + + +@pytest.mark.flaky(reruns=2) +def test_wikipedia_search_interface(wikipedia_container, wikipedia_base_url, page, pw_timeout): + """Test that the search interface exists.""" + page.goto(wikipedia_base_url, timeout=pw_timeout) + + # Look for search input - kiwix reader has a search box + search_input = page.get_by_role("textbox", name="Search") + + # Verify search input exists + assert search_input.is_visible(timeout=5000), "Search input should be visible" + + +@pytest.mark.flaky(reruns=2) +def test_wikipedia_search_returns_results(wikipedia_container, wikipedia_base_url, page, pw_timeout): + """Test that searching returns results (uses Ray Charles - exists in all ZIMs).""" + page.goto(wikipedia_base_url, timeout=pw_timeout) + + # Find and use search input + search_input = page.get_by_role("textbox", name="Search") + assert search_input.is_visible(timeout=5000), "Search input should be visible" + + search_input.fill("Ray Charles") + search_input.press("Enter") + + # Wait for results + page.wait_for_load_state("networkidle", timeout=pw_timeout) + + # Should show Ray Charles content + content = page.content().lower() + assert "ray" in content or "charles" in content, "Search should show Ray Charles content" diff --git a/uv.lock b/uv.lock index cf425f9..8121161 100644 --- a/uv.lock +++ b/uv.lock @@ -707,6 +707,30 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] +[[package]] +name = "docker" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "requests" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, +] + +[[package]] +name = "docker-squash" +version = "1.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "docker" }, + { name = "packaging" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/77/ef/45dc63f3f0f3fba18306dac34d9be26f9dd0d1a0742a7f0ca39cf1a98b15/docker_squash-1.2.2.tar.gz", hash = "sha256:74dd70412ddcf570bcd322567929062342793d1ec70c63b5371cc9e2808ae2ef", size = 38619, upload-time = "2024-11-13T19:15:51.224Z" } + [[package]] name = "docstring-parser" version = "0.17.0" @@ -6949,6 +6973,7 @@ examples = [ [package.dev-dependencies] dev = [ { name = "click" }, + { name = "docker-squash" }, { name = "environment-control" }, { name = "invoke" }, { name = "litellm" }, @@ -7003,6 +7028,7 @@ provides-extras = ["examples"] [package.metadata.requires-dev] dev = [ { name = "click", specifier = ">=8.3.0" }, + { name = "docker-squash", specifier = ">=1.2.2" }, { name = "environment-control", editable = "packages/environment_control" }, { name = "invoke", specifier = ">=2.2.0" }, { name = "litellm", specifier = ">=1.80.5" },