feat(BA-2327): dockerize agent with DooD (Docker-out-of-Docker)

changes/9596.feature.md

@@ -0,0 +1 @@
+Dockerize the agent component with DooD (Docker-out-of-Docker) support, including krunner path sharing, REPL port binding fixes, and comprehensive Docker deployment documentation for all six components

docker-compose.monorepo.yml

@@ -20,6 +20,31 @@ services:
       - "host.docker.internal:host-gateway"
     restart: unless-stopped
 
+  backend-ai-agent:
+    image: backend.ai/agent:26.1.0rc1
+    container_name: backend-ai-agent
+    networks:
+      - half
+    ports:
+      - "6001:6001"       # RPC (Manager -> Agent)
+      - "6003:6003"       # Metrics/Internal API
+      # NOTE: Do NOT map service ports (30000-31000) here.
+      # In DooD mode, session containers bind these ports directly on the host.
+      # Mapping them to the agent container would cause port conflicts.
+    volumes:
+      - ./agent.toml:/etc/backend.ai/agent.toml:ro
+      - /var/run/docker.sock:/var/run/docker.sock     # DooD: host Docker daemon
+      - /tmp/backend.ai/ipc:/tmp/backend.ai/ipc       # IPC with session containers
+      - /tmp/scratches:/tmp/scratches                  # Scratch storage (must use host path)
+      - /tmp/backend-ai-krunner:/tmp/backend-ai-krunner  # Shared krunner files for DooD
+    environment:
+      - PYTHONUNBUFFERED=1
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    restart: unless-stopped
+    depends_on:
+      - backend-ai-manager
+
   backend-ai-webserver:
     image: backend.ai/webserver:26.1.0rc1
     container_name: backend-ai-webserver
@@ -35,6 +60,51 @@ services:
     depends_on:
       - backend-ai-manager
 
+  backend-ai-storage-proxy:
+    image: backend.ai/storage-proxy:26.1.0rc1
+    container_name: backend-ai-storage-proxy
+    networks:
+      - half
+    ports:
+      - "6021:6021"       # Client API
+      - "6022:6022"       # Manager API
+    volumes:
+      - ./storage-proxy.toml:/etc/backend.ai/storage-proxy.toml:ro
+      - ./vfroot:/vfroot
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    restart: unless-stopped
+
+  backend-ai-appproxy-coordinator:
+    image: backend.ai/appproxy-coordinator:26.1.0rc1
+    container_name: backend-ai-appproxy-coordinator
+    networks:
+      - half
+    ports:
+      - "10200:10200"
+    volumes:
+      - ./proxy-coordinator.toml:/etc/backend.ai/proxy-coordinator.toml:ro
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    restart: unless-stopped
+    depends_on:
+      - backend-ai-manager
+
+  backend-ai-appproxy-worker:
+    image: backend.ai/appproxy-worker:26.1.0rc1
+    container_name: backend-ai-appproxy-worker
+    networks:
+      - half
+    ports:
+      - "10205-10300:10205-10300"   # Port proxy range
+    volumes:
+      - ./proxy-worker.toml:/etc/backend.ai/proxy-worker.toml:ro
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    restart: unless-stopped
+    depends_on:
+      - backend-ai-appproxy-coordinator
+
 networks:
   half:
     external: true

docker/agent-entrypoint.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+# DooD (Docker-out-of-Docker) krunner path setup
+#
+# In DooD mode, the agent creates session containers via the host Docker daemon.
+# When the agent bind-mounts krunner files (runner, kernel, helpers) into session
+# containers, Docker resolves the source paths on the HOST filesystem, not inside
+# the agent container. To make these paths accessible from the host, we:
+# 1. Copy krunner packages to a shared volume path (/tmp/backend-ai-krunner/)
+# 2. Replace the originals with symlinks so importlib.resources.files() resolves
+#    to the shared path (via Path.resolve() in resolve_krunner_filepath)
+
+KRUNNER_SHARED="/tmp/backend-ai-krunner"
+
+if [ -d "$KRUNNER_SHARED" ]; then
+    SITE_PKG=$(python3 -c "import site; print(site.getsitepackages()[0])")
+    for pkg in runner kernel helpers; do
+        src="$SITE_PKG/ai/backend/$pkg"
+        dst="$KRUNNER_SHARED/$pkg"
+        if [ -d "$src" ] && [ ! -L "$src" ]; then
+            # First run: copy packages to shared volume and create symlinks
+            cp -r "$src" "$dst"
+            rm -rf "$src"
+            ln -s "$dst" "$src"
+        elif [ -L "$src" ]; then
+            # Already symlinked (container restart)
+            :
+        fi
+    done
+fi
+
+exec "$@"

docker/backend.ai-agent.dockerfile

@@ -0,0 +1,37 @@
+ARG PYTHON_VERSION
+FROM python:${PYTHON_VERSION} AS builder
+ARG PKGVER
+COPY dist /dist
+COPY requirements.txt /requirements.txt
+# Install dependencies from requirements.txt to respect version constraints
+RUN pip wheel --wheel-dir=/wheels --no-cache-dir -r /requirements.txt
+# Install backend.ai packages from /dist (these are not in requirements.txt or PyPI)
+RUN pip wheel --wheel-dir=/wheels --no-cache-dir backend.ai-agent==${PKGVER} --find-links=/dist --no-deps
+
+FROM python:${PYTHON_VERSION}
+COPY --from=builder /wheels /wheels
+COPY dist /dist
+# Install all wheels and also look in /dist for backend.ai packages
+RUN pip install --no-cache-dir --find-links=/dist /wheels/*.whl
+
+# Install Docker CLI for DooD (Docker-out-of-Docker) operations
+RUN install -m 0755 -d /etc/apt/keyrings \
+    && curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc \
+    && chmod a+r /etc/apt/keyrings/docker.asc \
+    && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends docker-ce-cli \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create necessary directories
+RUN mkdir -p /tmp/backend.ai/ipc /var/log/backend.ai /etc/backend.ai \
+    /var/lib/backend.ai
+
+# Set working directory
+WORKDIR /app
+
+COPY docker/agent-entrypoint.sh /usr/local/bin/agent-entrypoint.sh
+RUN chmod +x /usr/local/bin/agent-entrypoint.sh
+
+ENTRYPOINT ["/usr/local/bin/agent-entrypoint.sh"]
+CMD ["backend.ai", "ag", "start-server", "-f", "/etc/backend.ai/agent.toml"]