fix(ci): use docker container in integration-test

.github/workflows/integration-test.yml

@@ -50,6 +50,8 @@ jobs:
               - 'logger_config_em.yaml'
               - 'supervisord.conf'
               - '.github/workflows/integration-test.yml'
+              - 'Dockerfile'
+              - 'dockerfile_scripts/**'
 
   integration:
     runs-on: ubuntu-24.04
@@ -218,39 +220,48 @@ jobs:
         run: |
           ./occ files:scan --all
 
-      - name: Setup python 3.11
-        uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5
-        with:
-          python-version: '3.11'
-          cache: 'pip'
-          cache-dependency-path: |
-            requirements.txt
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3
 
-      - name: Install and init backend
+      - name: Build the context_chat_backend cpu image
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6
+        with:
+          context: context_chat_backend
+          push: false
+          platforms: linux/amd64
+          # use local tag so image is not pulled from remote
+          tags: ghcr.io/ccb-cpu:local
+          target: runtime-cpu
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Start backend container
         run: |
           cd context_chat_backend
-          pip install --upgrade pip setuptools wheel
-          # use the cpu version of torch to not run out of space
-          pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
-          sed -i '/torch(vision)?/d' requirements.txt
-          pip install -r requirements.txt
-          cp example.env .env
-          echo "NEXTCLOUD_URL=http://localhost:8080" >> .env
-          python3 -u ./main_em.py > em_backend_logs 2>&1 &
-          python3 -u ./main.py > backend_logs 2>&1 &
-          echo $! > ../pid.txt          # Save the process ID (PID)
-          sleep 60 # Wait for the backend to get ready
+          echo "NEXTCLOUD_URL=http://localhost:8080" >> example.env
+          cd ..
+          # create it up front so the bind mount is not root-owned-only and the log
+          # files stay readable from the runner after the job
+          mkdir -p context_chat_backend/persistent_storage/logs
+          chmod -R 777 context_chat_backend/persistent_storage
+          docker run --net host --name context_chat_backend \
+            --env-file context_chat_backend/example.env \
+            -v $(pwd)/context_chat_backend/persistent_storage:/app/persistent_storage \
+            -e APP_PERSISTENT_STORAGE=/app/persistent_storage \
+            -e EXTERNAL_DB="$CCB_DB_URL" \
+            -d ghcr.io/ccb-cpu:local
 
       - name: Register backend
         run: |
           timeout 10 ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
-          timeout 120 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
+          timeout 180 ./occ app_api:app:register context_chat_backend manual_install --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" --force-scopes --wait-finish
           ls -la context_chat_backend/persistent_storage/*
 
       - name: Initial memory usage check
         run: |
-          ps -p $(cat pid.txt) -o pid,cmd,%mem,rss --sort=-%mem
-          ps -p $(cat pid.txt) -o %mem --no-headers > initial_mem.txt
+          docker stats --no-stream context_chat_backend
+          docker stats --no-stream --format '{{.MemPerc}}' context_chat_backend | tr -d ' %' > initial_mem.txt
 
       - name: Run cron jobs
         run: |
@@ -316,10 +327,12 @@ jobs:
               echo "Outside 3% tolerance: diff=$diff (${progress}%), threshold=$threshold"
             fi
 
-            # Check if backend is still alive
-            ccb_alive=$(ps -p $(cat pid.txt) -o cmd= | grep -c "main.py" || echo "0")
-            if [ "$ccb_alive" -eq 0 ]; then
-              echo "Error: Context Chat Backend process is not running. Exiting."
+            # Check if backend is still alive; inspect prints exactly "true"/"false"
+            # and nothing at all when the container is gone
+            ccb_alive=$(docker inspect -f '{{.State.Running}}' context_chat_backend 2>/dev/null || true)
+            if [ "$ccb_alive" != "true" ]; then
+              echo "Error: Context Chat Backend container is not running. Exiting."
+              docker logs --tail 50 context_chat_backend 2>&1 || true
               exit 1
             fi
 
@@ -461,20 +474,20 @@ jobs:
 
       - name: Check python memory usage
         run: |
-          ps -p $(cat pid.txt) -o pid,cmd,%mem,rss --sort=-%mem
-          ps -p $(cat pid.txt) -o %mem --no-headers > after_prompt_mem.txt
+          docker stats --no-stream context_chat_backend
+          docker stats --no-stream --format '{{.MemPerc}}' context_chat_backend | tr -d ' %' > after_prompt_mem.txt
 
       - name: Compare memory usage and detect leak
         run: |
           initial_mem=$(cat initial_mem.txt | tr -d ' ')
-          final_mem=$(cat after_scan_mem.txt | tr -d ' ')
+          final_mem=$(cat after_prompt_mem.txt | tr -d ' ')
           echo "Initial Memory Usage: $initial_mem%"
-          echo "Memory Usage after scan: $final_mem%"
+          echo "Memory Usage after prompt: $final_mem%"
 
           if (( $(echo "$final_mem > $initial_mem" | bc -l) )); then
-            echo "Memory usage has increased during scan. Possible memory leak detected!"
+            echo "Memory usage has increased during the run. Possible memory leak detected!"
           else
-            echo "Memory usage during scan is stable. No memory leak detected."
+            echo "Memory usage during the run is stable. No memory leak detected."
           fi
 
       - name: Final dump of DB with vectordb populated
@@ -497,25 +510,22 @@ jobs:
         run: |
           tail -v -n +1 worker?_logs || echo "No worker logs"
 
-      - name: Show main app logs
+      - name: Show container logs
         if: always()
         run: |
-          cat context_chat_backend/backend_logs || echo "No main backend logs"
+          # supervisord routes both the main app and the embedding server to the
+          # container's stdout/stderr, so this covers each of them
+          docker logs context_chat_backend || echo "No container logs"
 
       - name: Show main app JSON logs
         if: always()
         run: |
-          tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory"
-
-      - name: Show embedding server logs
-        if: always()
-        run: |
-          cat context_chat_backend/em_backend_logs || echo "No main backend logs"
+          sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory"
 
       - name: Show embedding server JSON logs
         if: always()
         run: |
-          tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory"
+          sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory"
 
       - name: Upload database dumps
         uses: actions/upload-artifact@v4