feat(ci): add RAG benchmarks workflow

.github/workflows/rag-benchmark.yml

@@ -0,0 +1,326 @@
+# SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
+# SPDX-License-Identifier: AGPL-3.0-or-later
+
+name: RAG Benchmark
+
+on:
+  workflow_dispatch:
+  schedule:
+    # cron day-of-month steps restart every month, so this runs on the 1st, 15th and 29th at 04:00 UTC
+    - cron: '0 4 */14 * *'
+
+permissions:
+  contents: read
+
+concurrency:
+  group: rag-benchmark-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+
+    outputs:
+      src: ${{ steps.changes.outputs.src }}
+
+    steps:
+      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
+        id: changes
+        continue-on-error: true
+        with:
+          filters: |
+            src:
+              - 'main.py'
+              - 'main_em.py'
+              - 'config.cpu.yaml'
+              - 'config.gpu.yaml'
+              - 'context_chat_backend/**'
+              - 'appinfo/**'
+              - 'example.env'
+              - 'requirements.txt'
+              - 'benchmark/**'
+              - '.github/workflows/rag-benchmark.yml'
+              - 'Dockerfile'
+              - 'dockerfile_scripts/**'
+
+  benchmark:
+    runs-on: ubuntu-24.04
+
+    needs: changes
+    if: needs.changes.outputs.src != 'false'
+
+    # hard cap so a stuck run fails fast instead of burning the 6h default
+    timeout-minutes: 180
+
+    strategy:
+      fail-fast: false
+      matrix:
+        benchmark: [ 'multihop-rag', 'frames' ]
+
+    name: RAG Benchmark - ${{ matrix.benchmark }}
+
+    env:
+      PGSQL_PORT: 4445
+      CCB_DB_URL: postgresql+psycopg://root:rootpassword@localhost:4445/nextcloud
+      NC_SERVER_VERSION: stable34
+      NC_PHP_VERSION: '8.2'
+      # queries per benchmark leg, stratified by question type; 0 = all of them
+      # (2255 for multihop-rag after null queries are dropped, 824 for frames)
+      BENCHMARK_QUERY_LIMIT: 0
+      # wall-clock cap for the query loop, leaving room for setup and indexing
+      # within the 180 min job timeout; at ~2.3s/query the full multihop-rag set
+      # needs ~86 min
+      BENCHMARK_BUDGET_SECONDS: 7200
+      # indexing attempts, 10s apart (45 min); FRAMES indexes ~2.5k documents
+      BENCHMARK_INDEX_ATTEMPTS: 270
+
+    services:
+      postgres:
+        image: pgvector/pgvector:pg17
+        ports:
+          - 4445:5432/tcp
+        env:
+          POSTGRES_USER: root
+          POSTGRES_PASSWORD: rootpassword
+          POSTGRES_DB: nextcloud
+        options: --health-cmd pg_isready --health-interval 5s --health-timeout 2s --health-retries 5 --name postgres --hostname postgres
+
+    steps:
+      - name: Checkout server
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          repository: nextcloud/server
+          ref: ${{ env.NC_SERVER_VERSION }}
+          submodules: 'recursive'
+          persist-credentials: false
+
+      - name: Set up php ${{ env.NC_PHP_VERSION }}
+        uses: shivammathur/setup-php@9e72090525849c5e82e596468b86eb55e9cc5401 # v2
+        with:
+          php-version: ${{ env.NC_PHP_VERSION }}
+          tools: phpunit
+          extensions: mbstring, iconv, fileinfo, intl, sqlite, pdo_mysql, pdo_sqlite, pgsql, pdo_pgsql, gd, zip
+
+      - name: Checkout context_chat php app
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          repository: nextcloud/context_chat
+          path: apps/context_chat
+          persist-credentials: false
+
+      - name: Checkout backend
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          path: context_chat_backend/
+          persist-credentials: false
+
+      - name: Checkout app_api
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          repository: nextcloud/app_api
+          ref: ${{ env.NC_SERVER_VERSION }}
+          path: apps/app_api
+          persist-credentials: false
+
+      - name: Get app version
+        id: appinfo
+        uses: skjnldsv/xpath-action@7e6a7c379d0e9abc8acaef43df403ab4fc4f770c # master
+        with:
+          filename: context_chat_backend/appinfo/info.xml
+          expression: "/info/version/text()"
+
+      - name: Set up Nextcloud
+        run: |
+          sleep 25
+          mkdir data
+          ./occ maintenance:install \
+            --verbose \
+            --database=pgsql \
+            --database-name=nextcloud \
+            --database-host=127.0.0.1 \
+            --database-port=$PGSQL_PORT \
+            --database-user=root \
+            --database-pass=rootpassword \
+            --admin-user admin \
+            --admin-pass password
+          composer run serve &
+
+      - name: Enable context_chat, app_api and testing
+        run: ./occ app:enable -vvv -f context_chat app_api testing
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3
+
+      - name: Build the context_chat_backend cpu image
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 # v6
+        with:
+          context: context_chat_backend
+          push: false
+          platforms: linux/amd64
+          # use local tag so image is not pulled from remote
+          tags: ghcr.io/ccb-benchmark:local
+          target: runtime-cpu
+          load: true
+          # the gha backend keys on scope, not on the tag, and the default scope
+          # ("buildkit") is shared with the integration tests, whose builds would
+          # otherwise overwrite this one's cache on every run
+          cache-from: type=gha,scope=rag-benchmark
+          cache-to: type=gha,mode=max,scope=rag-benchmark
+
+      # the preparation scripts only use the standard library, so the runner's
+      # default python is enough and no backend dependencies are needed here
+      - name: Download and prepare corpus and queries
+        run: |
+          python3 context_chat_backend/benchmark/prepare_dataset.py \
+            ${{ matrix.benchmark }} \
+            --out benchmark_data \
+            --limit ${{ env.BENCHMARK_QUERY_LIMIT }}
+
+      - name: Upload benchmark corpus to Nextcloud
+        run: |
+          mkdir -p data/admin/files/benchmark
+          cp -r benchmark_data/docs/. data/admin/files/benchmark/
+          ./occ files:scan admin
+
+      - name: Start backend container
+        run: |
+          cd context_chat_backend
+          echo "NEXTCLOUD_URL=http://localhost:8080" >> example.env
+          cd ..
+          # create it up front so the bind mount is not root-owned-only and the log
+          # files stay readable from the runner after the job
+          mkdir -p context_chat_backend/persistent_storage/logs
+          chmod -R 777 context_chat_backend/persistent_storage
+          docker run --net host --name context_chat_backend \
+            --env-file context_chat_backend/example.env \
+            -v $(pwd)/context_chat_backend/persistent_storage:/app/persistent_storage \
+            -e APP_PERSISTENT_STORAGE=/app/persistent_storage \
+            -e EXTERNAL_DB="$CCB_DB_URL" \
+            -d ghcr.io/ccb-benchmark:local
+
+      - name: Register backend
+        run: |
+          timeout 10 ./occ app_api:daemon:register --net host manual_install "Manual Install" manual-install http localhost http://localhost:8080
+          timeout 180 ./occ app_api:app:register context_chat_backend manual_install \
+            --json-info "{\"appid\":\"context_chat_backend\",\"name\":\"Context Chat Backend\",\"daemon_config_name\":\"manual_install\",\"version\":\"${{ fromJson(steps.appinfo.outputs.result).version }}\",\"secret\":\"12345\",\"port\":10034,\"scopes\":[],\"system_app\":0}" \
+            --force-scopes --wait-finish
+
+      - name: Run cron jobs to populate indexing queue
+        run: |
+          while true; do php cron.php; sleep 10; done &
+          sleep 30
+
+      - name: Wait for indexing to complete
+        run: |
+          # eligible_files_count is cached in appconfig for an hour, so a value read
+          # before the corpus upload would let the gate pass on a partial index
+          ./occ config:app:delete context_chat eligible_files_count || true
+          ./occ config:app:delete context_chat eligible_files_count_time || true
+
+          success=0
+          for i in $(seq 1 ${{ env.BENCHMARK_INDEX_ATTEMPTS }}); do
+            stats=$(./occ context_chat:stats --json 2>/dev/null || echo '{}')
+            # jq -e fails on null/false, so malformed stats are reported instead of
+            # silently becoming empty strings that bash then treats as 0
+            if ! total=$(echo "$stats" | jq -e '.eligible_files_count // 0') \
+              || ! indexed=$(echo "$stats" | jq -e '.vectordb_document_counts.files__default // 0') \
+              || ! queued=$(echo "$stats" | jq -e '[(.queued_documents_counts // {}) | to_entries[].value] | add // 0'); then
+              echo "Attempt $i: could not read stats, retrying. Raw output: $stats"
+              sleep 10
+              continue
+            fi
+            echo "Attempt $i: eligible=$total indexed=$indexed queued=$queued"
+
+            diff=$((total - indexed))
+            threshold=$((total * 3 / 100))
+            if [ "$total" -gt 0 ] && [ "$queued" -eq 0 ] && [ "$diff" -le "$threshold" ]; then
+              echo "Indexing complete (diff=$diff, threshold=$threshold)"
+              success=1
+              break
+            fi
+
+            # Check if backend is still alive; inspect prints exactly "true"/"false"
+            # and nothing at all when the container is gone
+            ccb_alive=$(docker inspect -f '{{.State.Running}}' context_chat_backend 2>/dev/null || true)
+            if [ "$ccb_alive" != "true" ]; then
+              echo "Error: Context Chat Backend container is not running. Exiting."
+              docker logs --tail 50 context_chat_backend 2>&1 || true
+              exit 1
+            fi
+
+            sleep 10
+          done
+
+          if [ $success -ne 1 ]; then
+            echo "Indexing did not complete within the allotted time"
+            exit 1
+          fi
+
+      # Retrieval-only: context_chat:search runs the vector search without invoking a
+      # text2text provider, which is what the retrieval metrics actually measure.
+      # maybe add LLM-as-judge answer correctness eval using llm2 + a very small model
+      # prompt it with the query + retrieved context + ground-truth answer, score pass/fail
+      - name: Run benchmark queries and compute retrieval metrics
+        run: |
+          ./occ background-job:worker 'OC\TaskProcessing\SynchronousBackgroundJob' > worker_logs 2>&1 &
+
+          python3 context_chat_backend/benchmark/run_benchmark.py \
+            --queries benchmark_data/queries.json \
+            --results benchmark_results.json \
+            --occ ./occ \
+            --user admin \
+            --budget-seconds ${{ env.BENCHMARK_BUDGET_SECONDS }}
+
+      - name: Show benchmark results
+        if: always()
+        run: |
+          echo "=== ${{ matrix.benchmark }} results ==="
+          cat benchmark_results.json 2>/dev/null || echo "No results file"
+
+      - name: Upload benchmark results
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: always()
+        with:
+          name: benchmark-results-${{ matrix.benchmark }}
+          path: benchmark_results.json
+          if-no-files-found: warn
+
+      - name: Show container logs
+        if: always()
+        run: |
+          # supervisord routes both the main app and the embedding server to the
+          # container's stdout/stderr, so this covers each of them
+          docker logs context_chat_backend || echo "No container logs"
+
+      - name: Show main app JSON logs
+        if: always()
+        run: |
+          sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/ccb.log* || echo "No logs in logs directory"
+
+      - name: Show embedding server JSON logs
+        if: always()
+        run: |
+          sudo tail -v -n +1 context_chat_backend/persistent_storage/logs/em_server.log* || echo "No logs in logs directory"
+
+      - name: Show task processing worker logs
+        if: always()
+        run: |
+          cat worker_logs || echo "No worker logs"
+
+  summary:
+    permissions:
+      contents: none
+    runs-on: ubuntu-latest-low
+    needs: [changes, benchmark]
+
+    if: always()
+
+    name: rag-benchmark
+
+    steps:
+      - name: Summary status
+        run: if ${{ needs.changes.outputs.src != 'false' && needs.benchmark.result != 'success' }}; then exit 1; fi

.nextcloudignore

@@ -25,6 +25,7 @@
 /README.md
 /readme.markdown
 /screenshots
+/benchmark
 /examples
 /docs
 /src