Add Docker site environments and integration tests

.github/workflows/test-docker-gitlab.yml

@@ -0,0 +1,43 @@
+name: Test Docker GitLab
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/gitlab/**"
+      - "tests/integration/environments/gitlab/**"
+      - "tests/integration/environments/conftest.py"
+  push:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/gitlab/**"
+      - "tests/integration/environments/gitlab/**"
+      - "tests/integration/environments/conftest.py"
+
+jobs:
+  test:
+    name: Test gitlab
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install 3.11
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Install Playwright browsers
+        run: uv run playwright install chromium --with-deps
+
+      - name: Build Docker image
+        run: uv run invoke envs.docker.build --site=gitlab --tag=test
+
+      - name: Run integration tests
+        run: uv run invoke envs.docker.test --site=gitlab --tag=test

.github/workflows/test-docker-map.yml.disabled

@@ -0,0 +1,54 @@
+name: Test Docker Map
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/map/**"
+      - "tests/integration/environments/map/**"
+      - "tests/integration/environments/conftest.py"
+  push:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/map/**"
+      - "tests/integration/environments/map/**"
+      - "tests/integration/environments/conftest.py"
+
+jobs:
+  test:
+    name: Test map
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install 3.11
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Install Playwright browsers
+        run: uv run playwright install chromium --with-deps
+
+      - name: Restore map test data cache
+        id: map-data-cache
+        uses: actions/cache@v4
+        with:
+          path: data/map
+          key: map-monaco-data-v1
+
+      - name: Generate map test data
+        if: steps.map-data-cache.outputs.cache-hit != 'true'
+        run: uv run invoke dev.ci.generate-map-data
+
+      - name: Build Docker image
+        run: uv run invoke envs.docker.build --site=map --tag=test
+
+      - name: Run integration tests
+        run: uv run invoke envs.docker.test --site=map --tag=test --data-dir=data/map

.github/workflows/test-docker-reddit.yml

@@ -0,0 +1,43 @@
+name: Test Docker Reddit
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/reddit/**"
+      - "tests/integration/environments/reddit/**"
+      - "tests/integration/environments/conftest.py"
+  push:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/reddit/**"
+      - "tests/integration/environments/reddit/**"
+      - "tests/integration/environments/conftest.py"
+
+jobs:
+  test:
+    name: Test reddit
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install 3.11
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Install Playwright browsers
+        run: uv run playwright install chromium --with-deps
+
+      - name: Build Docker image
+        run: uv run invoke envs.docker.build --site=reddit --tag=test
+
+      - name: Run integration tests
+        run: uv run invoke envs.docker.test --site=reddit --tag=test

.github/workflows/test-docker-shopping-admin.yml

@@ -0,0 +1,43 @@
+name: Test Docker Shopping Admin
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/shopping_admin/**"
+      - "tests/integration/environments/shopping_admin/**"
+      - "tests/integration/environments/conftest.py"
+  push:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/shopping_admin/**"
+      - "tests/integration/environments/shopping_admin/**"
+      - "tests/integration/environments/conftest.py"
+
+jobs:
+  test:
+    name: Test shopping_admin
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install 3.11
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Install Playwright browsers
+        run: uv run playwright install chromium --with-deps
+
+      - name: Build Docker image
+        run: uv run invoke envs.docker.build --site=shopping_admin --tag=test
+
+      - name: Run integration tests
+        run: uv run invoke envs.docker.test --site=shopping_admin --tag=test

.github/workflows/test-docker-shopping.yml

@@ -0,0 +1,43 @@
+name: Test Docker Shopping
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/shopping/**"
+      - "tests/integration/environments/shopping/**"
+      - "tests/integration/environments/conftest.py"
+  push:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/shopping/**"
+      - "tests/integration/environments/shopping/**"
+      - "tests/integration/environments/conftest.py"
+
+jobs:
+  test:
+    name: Test shopping
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install 3.11
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Install Playwright browsers
+        run: uv run playwright install chromium --with-deps
+
+      - name: Build Docker image
+        run: uv run invoke envs.docker.build --site=shopping --tag=test
+
+      - name: Run integration tests
+        run: uv run invoke envs.docker.test --site=shopping --tag=test

.github/workflows/test-docker-wikipedia.yml

@@ -0,0 +1,46 @@
+name: Test Docker Wikipedia
+
+on:
+  pull_request:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/wikipedia/**"
+      - "tests/integration/environments/wikipedia/**"
+      - "tests/integration/environments/conftest.py"
+  push:
+    branches: [main]
+    paths:
+      - "dev/environments/docker/sites/wikipedia/**"
+      - "tests/integration/environments/wikipedia/**"
+      - "tests/integration/environments/conftest.py"
+
+jobs:
+  test:
+    name: Test wikipedia
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+        with:
+          enable-cache: true
+
+      - name: Set up Python
+        run: uv python install 3.11
+
+      - name: Install dependencies
+        run: uv sync --all-extras
+
+      - name: Install Playwright browsers
+        run: uv run playwright install chromium --with-deps
+
+      - name: Download CI ZIM
+        run: uv run invoke dev.ci.setup-wikipedia
+
+      - name: Build Docker image
+        run: uv run invoke envs.docker.build --site=wikipedia --tag=test
+
+      - name: Run integration tests
+        run: uv run invoke envs.docker.test --site=wikipedia --tag=test --data-dir=data/wikipedia

.github/workflows/test.yml

@@ -32,4 +32,4 @@ jobs:
         run: docker build -t webarena-verified:test .
 
       - name: Run tests
-        run: uv run pytest --webarena-verified-docker-img webarena-verified:test --ignore=tests/integration/environment_control/
+        run: uv run pytest --webarena-verified-docker-img webarena-verified:test --ignore=tests/integration/environment_control/ --ignore=tests/integration/environments/

.gitignore

@@ -241,5 +241,5 @@ demo_config.json
 tmp/
 eval_log.txt
 
-# Generated Gatus config
-dev/environments/docker/monitoring/gatus/config.yaml
+# CI test data (downloaded/generated by inv dev.ci.* tasks)
+data/