diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..b6b83a9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,65 @@ +name: Bug report +description: Report a reproducible defect in Vectorless RAG. +title: "[Bug]: " +labels: + - bug +body: + - type: markdown + attributes: + value: | + Remove API keys, provider prompts, document text, and other secrets before submitting. + - type: input + id: version + attributes: + label: Version or commit + description: Provide the v0.2.x version or full commit SHA. + placeholder: v0.2.0 + validations: + required: true + - type: dropdown + id: surface + attributes: + label: Surface + options: + - API or worker + - Operator console + - Docker Compose + - Helm + - Documentation + - Evaluation + validations: + required: true + - type: textarea + id: reproduction + attributes: + label: Reproduction + description: List the smallest provider-safe steps that reproduce the defect. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected behavior + validations: + required: true + - type: textarea + id: actual + attributes: + label: Actual behavior + validations: + required: true + - type: textarea + id: diagnostics + attributes: + label: Sanitized diagnostics + description: Include relevant logs or command output after removing secrets and document content. + render: shell + - type: checkboxes + id: checks + attributes: + label: Submission checks + options: + - label: I searched existing issues and Discussions. + required: true + - label: I removed credentials, private documents, prompts, and provider response bodies. + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..d6ae4f9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Questions and operator support + url: https://github.com/ProofOfTechOrg/vectorless-rag/discussions + about: Ask usage and troubleshooting questions in Discussions. + - name: Report a security vulnerability + url: https://github.com/ProofOfTechOrg/vectorless-rag/security/advisories/new + about: Send sensitive security reports through a private advisory. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..7b7e3c7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,32 @@ +name: Feature request +description: Propose a scoped change to Vectorless RAG. +title: "[Feature]: " +labels: + - enhancement +body: + - type: textarea + id: problem + attributes: + label: Problem + description: Describe the operator or research problem without prescribing an implementation. + validations: + required: true + - type: textarea + id: outcome + attributes: + label: Desired outcome + description: State the observable result and who benefits. + validations: + required: true + - type: textarea + id: alternatives + attributes: + label: Alternatives considered + description: List existing commands, workflows, or external tools you evaluated. + - type: textarea + id: constraints + attributes: + label: Security and provider-cost impact + description: Note any credential, document-data, model-call, or deployment implications. + validations: + required: true diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..01923dc --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,19 @@ + + + +## Change + +Describe the problem, the implemented change, and deliberate exclusions. + +## Verification + +List each command you ran and its result. + +- [ ] Tests cover the changed behavior +- [ ] Documentation matches the changed behavior +- [ ] No API key, provider prompt, document text, or other secret appears in the diff or test artifacts +- [ ] Provider-backed tests and their cost were explicitly authorized, or this change used only provider-free tests + +## AI-use disclosure + +Name any AI tool used, the work it performed, and the review you completed. Write “None” if no AI tool contributed. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 140d495..7f92b0b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,25 +3,29 @@ on: push: pull_request: +permissions: + contents: read + jobs: api: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8.3.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: {version: "0.11.29", enable-cache: true} - run: make api-install - run: make api-lint - run: make api-typecheck - run: make api-test - run: make migration-check + - run: make test-integration-stack web: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - - uses: pnpm/action-setup@v6 - - uses: actions/setup-node@v7 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6 + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 with: {node-version: 22, cache: pnpm} - run: make web-install - run: make playwright-install @@ -38,11 +42,13 @@ jobs: deployment: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v7 - - uses: astral-sh/setup-uv@v8.3.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 with: {version: "0.11.29"} - - uses: azure/setup-helm@v5 + - uses: Azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5 + with: {version: "v4.2.3"} - run: make api-install + - run: make docs-check - run: make workflow-check - run: make compose-check ENV_FILE=.env.example - run: make helm-check diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1d300a8 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,32 @@ + + + +# Track Vectorless RAG releases + +This changelog records user-visible changes. Vectorless RAG follows semantic version labels, but prerelease status does not promise production stability. + +## Unreleased + +No user-visible changes have been recorded after v0.2.0. + +## 0.2.0, 2026-07-27 + +Version 0.2.0 publishes the repository under the MIT License as a source-only research preview. + +### Added + +- TanStack Start operator console for chat, documents, ingestion, settings, scoped connection, appearance, and logout +- Encrypted HttpOnly browser session backed by FastAPI API-key validation +- Durable PDF ingestion, PageIndex v2 artifacts, catalog routing, page citations, usage accounting, and provider-state handling +- Docker Compose and Helm deployment surfaces +- Deterministic API, PostgreSQL integration, browser, deployment, image, and evaluation checks +- Atomic local environment initialization and indexed public documentation + +### Research-preview limitations + +- The default gate permits 25 registered documents and keeps full-corpus ingestion disabled +- The frozen pilot has a $10 provider-cost ceiling +- DeepSeek-backed ingestion and chat incur external provider cost +- Optical character recognition, API-key lifecycle management, and an embedded PDF viewer are unavailable +- Full 537-document ingestion and a final vector-versus-PageIndex quality verdict remain outside this release +- The release contains source archives only diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..c3ce301 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,84 @@ + + + +# Contributor Covenant Code of Conduct + +This policy adopts Contributor Covenant version 2.1 for all Vectorless RAG community spaces. + +## Our pledge + +We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community. + +## Our standards + +Examples of behavior that contributes to a positive environment for our community include: + +- Demonstrating empathy and kindness toward other people +- Respecting differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, and learning from the experience +- Focusing on what is best not only for us as individuals, but for the overall community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances of any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others’ private information, such as a physical or email address, without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a professional setting + +## Enforcement responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of acceptable behavior. They will take appropriate and fair corrective action in response to behavior that they deem inappropriate, threatening, offensive, or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that do not align with this Code of Conduct. They will communicate reasons for moderation decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces. It also applies when an individual officially represents the community in public spaces. + +Examples of representing our community include using an official email address, posting through an official social media account, or acting as an appointed representative at an online or offline event. + +## Enforcement + +Report abusive, harassing, or otherwise unacceptable behavior privately to [hello@proofoftech.org](mailto:hello@proofoftech.org). Community leaders will review and investigate complaints promptly and fairly. + +Community leaders must respect the privacy and security of anyone who reports an incident. + +## Enforcement guidelines + +Community leaders will follow these Community Impact Guidelines when determining the consequences for any action they deem in violation of this Code of Conduct. + +### 1. Correction + +**Community impact**: Use of inappropriate language or other behavior deemed unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders that explains the violation and the nature of why the behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community impact**: A violation through a single incident or series of actions. + +**Consequence**: A warning with consequences for continued behavior. The person must not interact with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, for a specified period. This restriction includes avoiding interactions in community spaces and external channels such as social media. Violating these terms may lead to a temporary or permanent ban. + +### 3. Temporary ban + +**Community impact**: A serious violation of community standards, including sustained inappropriate behavior. + +**Consequence**: A temporary ban from any interaction or public communication with the community for a specified period. The person must not interact publicly or privately with the people involved, including unsolicited interaction with those enforcing the Code of Conduct, during this period. Violating these terms may lead to a permanent ban. + +### 4. Permanent ban + +**Community impact**: A pattern of violating community standards, including sustained inappropriate behavior, harassment of an individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any public interaction within the community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant, version 2.1](https://www.contributor-covenant.org/version/2/1/code_of_conduct.html). + +Community Impact Guidelines were inspired by [Mozilla’s code of conduct enforcement ladder](https://github.com/mozilla/diversity). + +For answers to common questions about this code of conduct, see the [Contributor Covenant frequently asked questions](https://www.contributor-covenant.org/faq). Translations are available from the [Contributor Covenant translations index](https://www.contributor-covenant.org/translations). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4b0bad6 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,75 @@ + + + +# Contribute to Vectorless RAG + +This guide explains how to prepare, verify, and submit a change to the research preview. Keep each pull request focused on one defect, feature, or documentation outcome. + +## Prepare the development environment + +You need Python 3.12, `uv`, Node.js 22, pnpm, Docker Compose 2.22 or later, GNU Make, and Helm 4.2.3. + +Clone your fork and install the frozen dependency graphs: + +```bash +git clone https://github.com/your_github_name/vectorless-rag.git +cd vectorless-rag +make install +``` + +The repository quarantines newly published dependencies for seven days through pnpm and uv configuration. Do not remove or bypass those settings to resolve an install failure. + +Run `make init` only when you need a local live stack. The command refuses to replace an existing `.env`. + +## Develop on a focused branch + +Create a branch from current `master`: + +```bash +git fetch origin +git switch master +git pull --ff-only +git switch -c fix/describe-the-defect +``` + +Match the surrounding code style. Fix the root cause and every code path with the same defect. Avoid unrelated formatting, new abstractions, and dependency changes. + +## Verify the affected surfaces + +Run the smallest relevant command while developing, then run the complete provider-free gates before opening a pull request: + +```bash +make docs-check +make api-check +make test-integration-stack +make web-check +make playwright +make deploy-check ENV_FILE=.env.example +``` + +Changes to images or release behavior also require `make docker-build`. See [choose a test tier](docs/testing.md) for prerequisites and expected results. + +Do not use DeepSeek credentials in continuous integration (CI). Do not run provider-backed tests unless you have explicit cost authorization. + +## Document behavior and decisions + +Update user documentation in the same pull request as the behavior change. Add a content-type and content-plan comment to every new Markdown page, use sentence-case headings, tag code fences, and add the page to `docs/README.md`. + +Record user-visible changes under `Unreleased` in `CHANGELOG.md`. Preserve dated design records instead of rewriting their historical measurements with current assumptions. + +## Protect credentials and research data + +Never commit or post: + +- API keys, session secrets, database passwords, or provider credentials +- Provider prompts, response bodies, or private document text +- Playwright traces, screenshots, videos, or storage state from credentialed tests +- Files from the Git-ignored `arxiv-pdfs` corpus + +Use sanitized diagnostics in issues and pull requests. Report exploitable security defects through the [private vulnerability reporting channel](https://github.com/ProofOfTechOrg/vectorless-rag/security/advisories/new). + +## Open the pull request + +Push your branch and open a pull request against `master`. Complete the template with scope, verification evidence, security and provider-cost impact, and documentation changes. + +Disclose AI-assisted work by naming the tool, describing its contribution, and stating how you reviewed the result. You remain responsible for every submitted line. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..8c8174b --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 ProofOfTechOrg + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile index 5f91289..336e9dd 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,7 @@ SHELL := /usr/bin/env bash UV ?= uv UV_CACHE_DIR ?= .uv-cache +PYTHON ?= python3 PNPM ?= pnpm HELM ?= helm DOCKER ?= docker @@ -29,8 +30,9 @@ PILOT_OUTPUT ?= apps/api/pilots/run/result.json export UV_CACHE_DIR -.PHONY: help env install api-install web-install format api-format web-format lint api-lint web-lint \ +.PHONY: help env init install api-install web-install format api-format web-format lint api-lint web-lint \ typecheck api-typecheck web-typecheck test api-test web-test test-unit test-integration \ + test-integration-stack docs-check \ api-build web-build web-production-smoke playwright-install playwright playwright-live react-doctor react-doctor-diff astryx-doctor workflow-check migration-check \ compose-check helm-lint helm-template helm-check docker-build docker-build-api docker-build-web \ check api-check web-check deploy-check ci up rebuild dev dev-build dev-health down stop restart restart-app restart-web \ @@ -45,6 +47,9 @@ env: ## Create .env from .env.example if absent and restrict its permissions. @if [[ -e .env ]]; then echo ".env already exists; leaving it unchanged"; else cp .env.example .env; echo "created .env"; fi @chmod 600 .env +init: ## Create a mode-600 .env with generated local secrets without overwriting. + $(PYTHON) scripts/init_env.py + install: api-install web-install ## Install all frozen development dependencies. api-install: ## Install the frozen Python environment under apps/api. $(UV) sync --project $(API_DIR) --frozen @@ -82,6 +87,11 @@ test-integration: ## Run PostgreSQL integration tests with TEST_*_DATABASE_URL. @test -n "$${TEST_OWNER_DATABASE_URL:-}" || { echo "TEST_OWNER_DATABASE_URL is required" >&2; exit 2; } @test -n "$${TEST_DATABASE_URL:-}" || { echo "TEST_DATABASE_URL is required" >&2; exit 2; } cd $(API_DIR) && $(UV) run pytest tests -m integration +test-integration-stack: ## Run integration tests against an isolated disposable PostgreSQL project. + UV="$(UV)" DOCKER="$(DOCKER)" MAKE="$(MAKE)" scripts/test-integration-stack.sh + +docs-check: ## Validate community files, Markdown links, anchors, fences, and the documentation index. + $(PYTHON) scripts/docs_check.py api-build: ## Build the Python distribution. $(UV) build --project $(API_DIR) @@ -90,7 +100,11 @@ web-build: ## Build the TanStack Start production bundle. web-production-smoke: web-build ## Verify the built SSR document serves all referenced CSS and JavaScript assets. $(PNPM) --filter @vectorless-rag/web production-smoke playwright-install: ## Install Chromium and its operating-system dependencies for Playwright. - $(PNPM) --filter @vectorless-rag/web exec playwright install --with-deps chromium + $(PNPM) --filter @vectorless-rag/web exec playwright install chromium + @if ! $(PNPM) --filter @vectorless-rag/web exec node scripts/verify-playwright-install.mjs; then \ + $(PNPM) --filter @vectorless-rag/web exec playwright install-deps chromium; \ + $(PNPM) --filter @vectorless-rag/web exec node scripts/verify-playwright-install.mjs; \ + fi playwright: ## Run deterministic Playwright against the local mock upstream. $(PNPM) --filter @vectorless-rag/web test:e2e playwright-live: ## Run Playwright against a live stack with an explicit URL and process-only key. @@ -126,9 +140,9 @@ docker-build: docker-build-api docker-build-web ## Build both production images. api-check: api-lint api-typecheck api-test migration-check ## Validate the API package. web-check: web-lint web-typecheck web-test react-doctor web-production-smoke astryx-doctor ## Validate the web package. deploy-check: workflow-check compose-check helm-check ## Validate CI and deployment manifests. -check: api-check web-check deploy-check ## Validate the full repository without browser or image builds. +check: docs-check api-check web-check deploy-check ## Validate the full repository without browser or image builds. ci: ENV_FILE := .env.example -ci: check playwright-install playwright docker-build ## Run the complete CI-equivalent validation. +ci: check test-integration-stack playwright-install playwright docker-build ## Run the complete CI-equivalent validation. up: ## Start the complete Compose stack in the background without forcing rebuilds. $(COMPOSE) up -d diff --git a/README.md b/README.md index 208fa7a..ee70376 100644 --- a/README.md +++ b/README.md @@ -1,55 +1,49 @@ + # Vectorless RAG -A self-hosted research application for asking grounded questions across a local PDF corpus without embeddings or a vector database. The browser-first Vectorless RAG Operator Console uses ASTRYX and TanStack Start. Its server-side BFF stores the API key in an encrypted HttpOnly cookie and calls FastAPI. +[![CI](https://github.com/ProofOfTechOrg/vectorless-rag/actions/workflows/ci.yml/badge.svg)](https://github.com/ProofOfTechOrg/vectorless-rag/actions/workflows/ci.yml) +[![License: MIT](https://img.shields.io/github/license/ProofOfTechOrg/vectorless-rag)](LICENSE) +[![Release](https://img.shields.io/github/v/release/ProofOfTechOrg/vectorless-rag?include_prereleases)](https://github.com/ProofOfTechOrg/vectorless-rag/releases) -PostgreSQL applies explicit metadata constraints and exact-match signals, DeepSeek reasons over the complete eligible catalog in bounded batches, and PageIndex retrieves from validated hierarchical trees. Responses include inline, resolvable page citations. +Vectorless RAG is a self-hosted research preview for grounded question answering over local Portable Document Format (PDF) files. It uses PageIndex hierarchies instead of embeddings or a vector database, then returns inline citations that resolve to document pages. -## Quick start +Version 0.2.0 is a prerelease. It supports evaluation and bounded pilots, not production stability or unattended full-corpus ingestion. -Requirements: Docker Compose 2.22 or later and GNU Make. Local package development also uses Python 3.12, `uv`, Node 22, and pnpm. Helm 3 is required only to validate or deploy the Kubernetes chart. +## Choose this project for bounded research -Run `make download-corpus` before `make up` if you want the full test corpus. The download is optional and requires Bash, `curl`, and approximately 1.5 GB of free space. +Vectorless RAG fits these use cases: -```bash -make env -make secret-api-pepper -make secret-session -make secrets-langfuse -# Edit .env: replace change-me values and set DEEPSEEK_API_KEY. -make up -make bootstrap-key NAME=browser-operator -``` +- Compare hierarchical, vectorless retrieval with a conventional vector retrieval-augmented generation (RAG) system +- Ask cross-document questions over a small, controlled PDF corpus +- Inspect retrieval routes, durable ingestion jobs, citations, token use, and provider costs +- Run a self-hosted browser console with scoped API keys and server-side credential storage -Save the one-time `vrag_…` key, open [http://localhost:3001](http://localhost:3001), and enter it on `/connect`. Langfuse remains on port 3000 and FastAPI/OpenAPI on [http://localhost:8000/docs](http://localhost:8000/docs). +Do not use this preview for regulated workloads, scanned PDFs that require optical character recognition (OCR), or an unreviewed full-corpus migration. -The key is validated by FastAPI, encrypted with AES-256-GCM, and stored for eight hours in an HttpOnly, `SameSite=Lax`, `Path=/` cookie. Production uses `Secure` and a `__Host-` name. It is never placed in browser storage, a URL, an SSR payload, or a browser-readable cookie. +## Understand costs and limits before starting -## Download the test corpus - -The repository tracks the 537 arXiv IDs used for corpus testing, but it does not distribute the PDFs. This command downloads missing PDFs directly from arXiv into `arxiv-pdfs`: - -```bash -make download-corpus -``` +Ingestion and live chat call DeepSeek and can incur provider charges. The default safety controls cap registration at 25 documents, keep full-corpus indexing disabled, and enforce a $10 frozen-pilot ceiling. Provider prices can change, so check the [official DeepSeek model and pricing reference](https://api-docs.deepseek.com/quick_start/pricing/) before a paid run. -Set `ARXIV_PDF_HOST_DIR` when your Compose corpus mount uses another host directory: +The configured `deepseek-v4-pro` model remains available in that reference. Deterministic unit, integration, deployment, and browser tests do not require provider credentials. -```bash -make download-corpus ARXIV_PDF_HOST_DIR=/srv/vectorless/arxiv-pdfs -``` +Current limitations include: -The command validates each response before installing it as `arxiv_id.pdf`. Reruns skip valid existing PDFs without network requests, while invalid existing paths remain untouched and cause a nonzero exit status. +- Born-digital PDFs only, with no OCR +- No API-key lifecycle interface or embedded PDF viewer +- Source builds only, with no package-registry or container-registry release +- A pilot gate instead of a production-scale claim +- No final vector-versus-PageIndex quality verdict -Downloading only creates local files. It does not start the stack or queue ingestion. After the stack is running, `make sync-corpus` queues mounted files up to the configured 25-document safety gate. The frozen pilot uses `make pilot-run`, which selects only its tracked manifest; do not use ordinary full-directory sync for that run. See the [command reference](docs/commands.md#download-the-test-corpus) for failure handling and request-rate details. +## Follow the request path -## Architecture +The browser-facing frontend (BFF) keeps API keys out of browser-readable storage. FastAPI authorizes each API request, LangGraph coordinates retrieval, and a durable worker builds PageIndex artifacts. ```mermaid flowchart LR Browser -->|same origin| BFF[TanStack Start BFF] - BFF -->|X-API-Key, server only| API[FastAPI] + BFF -->|server-side API key| API[FastAPI] API --> Auth[Scoped API keys] API --> Graph[LangGraph] Graph --> Catalog[(PostgreSQL)] @@ -61,76 +55,54 @@ flowchart LR Worker --> Objects ``` -Browser requests never call FastAPI directly. TanStack server functions and routes authorize at their own boundary, validate runtime responses, enforce the configured full `APP_ORIGIN` on mutations, and clear a rejected session after an upstream `401`. Chat SSE is proxied without re-buffering and propagates cancellation. +Docker Compose runs the console, API, worker, PostgreSQL, MinIO, Langfuse, ClickHouse, and Redis. The Helm chart exposes the same application boundaries for Kubernetes. -## Console +## Start with one PDF -- `/connect`: API-key validation, safe errors, and bootstrap help. -- `/chat` and `/chat/$threadId`: recent threads, route/progress states, Markdown, citations, cancel, retry, and feedback. -- `/documents` and `/documents/$documentId`: cursor pagination, responsive table/list views, metadata, topics, status, diagnostics, and citation landing pages. -- `/ingestion`: PDF upload, durable lease-backed job polling, mounted-corpus sync modes, provider state, and dynamic pilot capacity. -- `/settings`: key identity/scopes, infrastructure readiness, sanitized provider state, system/light/dark appearance, setup details, and logout. +You need Docker Compose 2.22 or later, GNU Make, Python 3.12, and `uv`. Run these commands from the repository root: -The UI hides or disables actions outside the key's scopes, while FastAPI remains the authorization boundary. +```bash +make init +# Add DEEPSEEK_API_KEY to .env. +make up +make bootstrap-key NAME=browser-operator +``` -## Repository layout +Save the one-time `vrag_…` key. Open [the local operator console](http://localhost:3001/connect), enter the key, and upload one born-digital PDF from **Ingestion**. Ask a question from **Chat** after ingestion succeeds. -```text -apps/api/ Python package, tests, Alembic, evaluation, scripts, image -apps/web/ TanStack Start app, Vitest, Playwright, Node image -infra/docker/postgres-init/ PostgreSQL first-start initialization -infra/helm/vectorless-rag/ Full-stack Helm chart -docs/ Security, deployment, operator, command, evaluation, backup docs -compose.yaml Local full stack -Makefile Repository command surface -arxiv-pdfs/ Git-ignored, user-owned local corpus +The stack stores service data in named volumes. Stop containers without deleting data: + +```bash +make down ``` -## Development and verification +Read [get started with one cited answer](docs/getting-started.md) for prerequisites, expected states, and cleanup. + +## Run provider-free verification + +Install the frozen development dependencies before running the complete local gates: ```bash -make dev # full Compose stack with API, worker, and web reload -make dev-health # verify the web service is running through Vite -make dev-build # optional development image prebuild/troubleshooting -make web-dev # host web on :3001 against the Compose API on :8000 -make install # install host API and web dependencies +make install +make docs-check make api-check +make test-integration-stack make web-check +make playwright make deploy-check ENV_FILE=.env.example -make playwright-install # install Chromium and its operating-system dependencies -make playwright # deterministic mock-upstream browser suite make docker-build -make ci # local CI equivalent ``` -Use `make help` or the [command reference](docs/commands.md) for every target. Follow the [live verification workflow](docs/operator-guide.md#verify-a-disposable-live-stack) before running `make playwright-live`. - -`make dev` is the canonical active-development command. It runs in the foreground, builds the current images, and mounts local API and web source into the containers. Web source edits update through Vite hot module replacement. Changes to `apps/web/package.json`, either workspace manifest, `pnpm-lock.yaml`, or `apps/web/Dockerfile` automatically rebuild and recreate the web service. `make dev-build` remains available as an optional image prebuild or troubleshooting command. Docker Compose 2.22 or later is required for these watch rules. - -Changes to `.env`, `compose.yaml`, or `compose.dev.yaml` require stopping `make dev` with Ctrl+C and rerunning it so Compose can resolve the updated service configuration. - -`make up` starts an immutable deployment-style stack in the background and does not hot-reload or force image rebuilds. Run `make rebuild` to rebuild and recreate the complete stack. `make restart-app` rebuilds only API and worker, while `make restart-web` rebuilds only the production-style frontend. These commands preserve named volumes. +`make playwright` uses the existing mock upstream only for deterministic browser coverage. It is not a product demo and does not replace the paid release gate. -## Ingestion safety - -The default 25-document gate and $10 frozen-pilot ceiling remain enabled; full-corpus indexing remains disabled. Upload born-digital PDFs through the UI or use bounded sync. OCR and recursive/arbitrary server paths are out of scope. Use `make pilot-check` and the manifest-driven `make pilot-run` for the paid v2 pilot. See the [operator guide](docs/operator-guide.md). - -## Advanced API client path - -All `/v1` routes use `X-API-Key`; `/health/live` and `/health/ready` are public. - -```bash -export VECTORLESS_RAG_ADMIN_KEY=vrag_replace_me -curl --fail --silent --show-error http://localhost:8000/v1/auth/me \ - -H "X-API-Key: $VECTORLESS_RAG_ADMIN_KEY" -curl --fail --silent --show-error -X POST http://localhost:8000/v1/chat \ - -H "X-API-Key: $VECTORLESS_RAG_ADMIN_KEY" \ - -H 'Content-Type: application/json' \ - -d '{"message":"Compare retrieval quality in the indexed papers.","constraints":{}}' -``` +Read [choose a test tier](docs/testing.md) for test prerequisites and provider boundaries. -`GET /v1/auth/me` returns only key name and scopes. `GET /v1/threads?limit=&cursor=` returns key-isolated recent threads ordered by activity. OpenAPI documents the complete surface. +## Find the next task -## Deployment +The [documentation index](docs/README.md) routes setup, operation, deployment, security, evaluation, and historical design work. Start with: -`compose.yaml` runs web (3001), Langfuse (3000), API (8000), worker, PostgreSQL, ClickHouse, Redis, and MinIO while preserving the existing named volumes. The chart in `infra/helm/vectorless-rag` has separate API and optional frontend ingresses, services, probes, resources, and secret wiring. Read [deployment](docs/deployment.md) and [security](docs/security.md) before exposing the application. +- [Contributing](CONTRIBUTING.md) for development and pull requests +- [Security policy](SECURITY.md) for private vulnerability reports +- [Support](SUPPORT.md) for questions and reproducible defects +- [Changelog](CHANGELOG.md) for release status and limitations +- [MIT License](LICENSE) and [third-party notices](THIRD_PARTY_NOTICES.md) for redistribution terms diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..b6fa680 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,37 @@ + + + +# Report a Vectorless RAG security vulnerability + +Use this policy for exploitable defects in Vectorless RAG. Do not disclose sensitive vulnerabilities in public issues, Discussions, or pull requests. + +## Supported versions + +Security fixes target these versions: + +| Version | Supported | +| --- | --- | +| Current `master` | Yes | +| 0.2.x research previews | Yes | +| Earlier versions | No | + +Support means the maintainers will assess a report and may prepare a fix. This research preview has no guaranteed response or remediation service-level agreement. + +## Send a private report + +Open a [private GitHub security advisory](https://github.com/ProofOfTechOrg/vectorless-rag/security/advisories/new). Include the affected version or commit, impact, prerequisites, reproduction steps, and a minimal proof of concept. + +Remove real API keys, provider prompts, private PDF text, and credentials from the report. Use synthetic data whenever possible. + +If GitHub private reporting is unavailable, email [hello@proofoftech.org](mailto:hello@proofoftech.org) with a request for a private reporting channel. Do not send exploit details until the recipient confirms the channel. + +## Protect secrets during research + +Never place a Vectorless RAG key or deployment secret in: + +- `VITE_*` variables, URLs, logs, or committed files +- Browser local storage, session storage, or server-side rendering data +- Screenshots, traces, videos, Playwright storage state, or fixtures +- Public issues, Discussions, pull requests, or test output + +Do not attach private source PDFs, model prompts, or provider response bodies. The [security model](docs/security.md) documents the application’s trust boundaries and deployment controls. diff --git a/SUPPORT.md b/SUPPORT.md new file mode 100644 index 0000000..9185e15 --- /dev/null +++ b/SUPPORT.md @@ -0,0 +1,24 @@ + + + +# Get support for Vectorless RAG + +Vectorless RAG is a research preview maintained through public GitHub channels. It does not include a guaranteed support or response service-level agreement. + +## Ask usage questions in Discussions + +Use [GitHub Discussions](https://github.com/ProofOfTechOrg/vectorless-rag/discussions) for setup questions, operator guidance, architecture discussion, and research results. Search existing discussions and [troubleshooting](docs/troubleshooting.md) before posting. + +Remove keys, passwords, private document text, prompts, and provider response bodies from every post. + +## Report reproducible project defects in Issues + +Use the [bug report form](https://github.com/ProofOfTechOrg/vectorless-rag/issues/new?template=bug_report.yml) for a defect you can reproduce in this repository. Include the version or commit, affected surface, minimal reproduction, expected behavior, actual behavior, and sanitized diagnostics. + +Use the [feature request form](https://github.com/ProofOfTechOrg/vectorless-rag/issues/new?template=feature_request.yml) for a scoped product or research proposal. + +Send exploitable vulnerabilities through the [private security policy](SECURITY.md), not an issue. + +## Contact infrastructure and model providers directly + +Project maintainers cannot resolve provider account balances, model availability, rate limits, billing disputes, Docker host failures, Kubernetes platform outages, or third-party service incidents. Reproduce the problem at the project boundary, then contact the responsible provider when the failure occurs outside Vectorless RAG. diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000..64537c3 --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,26 @@ + + + +# Review third-party software notices + +Vectorless RAG includes or installs the following third-party software. Each component remains subject to its own license and copyright notices. + +## PageIndex + +The API image installs [VectifyAI PageIndex](https://github.com/VectifyAI/PageIndex) at revision `190f8b378be58199ca993566a9214dba72089c54` and applies the tracked runtime patch in `apps/api/patches/pageindex-runtime.patch`. + +PageIndex is licensed under the MIT License. + +## ASTRYX + +The web workspace depends on these ASTRYX packages at version 0.1.6: + +- `@astryxdesign/core` +- `@astryxdesign/theme-neutral` +- `@astryxdesign/cli` + +The ASTRYX packages are licensed under the MIT License. + +## Complete dependency records + +`apps/api/uv.lock` and `pnpm-lock.yaml` record the complete dependency graphs. Review each dependency’s source distribution for its copyright and license terms. diff --git a/apps/api/README.md b/apps/api/README.md index 6248a0a..1b52547 100644 --- a/apps/api/README.md +++ b/apps/api/README.md @@ -1,8 +1,8 @@ + + + # Vectorless RAG API -Python package for the Vectorless RAG FastAPI service and ingestion worker. -Repository-wide setup, operations, deployment, and browser usage are documented in -the [root README](../../README.md). +This package contains the Vectorless RAG FastAPI service, ingestion worker, Alembic migrations, evaluation tools, and Python tests. Repository-wide setup, operations, deployment, and browser usage are documented in the [project overview](../../README.md). -From the repository root, use `make api-install`, `make api-test`, and -`make api-check`. From this directory, the equivalent package commands use `uv`. +Run `make api-install`, `make api-test`, and `make api-check` from the repository root. Run equivalent package commands with `uv` from this directory. diff --git a/apps/api/evaluation/review-pack.md b/apps/api/evaluation/review-pack.md index bab3df9..51c67cf 100644 --- a/apps/api/evaluation/review-pack.md +++ b/apps/api/evaluation/review-pack.md @@ -1,3 +1,6 @@ + + + # PageIndex v2 evaluation review pack Mark every case approved, edited, or rejected in the JSONL review record before seeding. diff --git a/apps/api/pyproject.toml b/apps/api/pyproject.toml index 2cd9fd4..641589f 100644 --- a/apps/api/pyproject.toml +++ b/apps/api/pyproject.toml @@ -1,9 +1,16 @@ [project] name = "vectorless-rag" -version = "0.1.0" -description = "Self-hosted vectorless RAG over a local ArXiv corpus" +version = "0.2.0" +description = "Self-hosted vectorless RAG for grounded question answering over local PDFs" readme = "README.md" requires-python = ">=3.12,<3.13" +license = "MIT" +authors = [{ name = "ProofOfTechOrg", email = "hello@proofoftech.org" }] +classifiers = [ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 3.12", +] dependencies = [ "alembic==1.18.5", "boto3==1.43.51", @@ -36,6 +43,11 @@ dependencies = [ [project.scripts] vectorless-rag = "vectorless_rag.cli:app" +[project.urls] +Homepage = "https://github.com/ProofOfTechOrg/vectorless-rag" +Repository = "https://github.com/ProofOfTechOrg/vectorless-rag" +Issues = "https://github.com/ProofOfTechOrg/vectorless-rag/issues" + [dependency-groups] dev = [ "basedpyright==1.39.9", diff --git a/apps/api/scripts/build_evaluation_review_pack.py b/apps/api/scripts/build_evaluation_review_pack.py index a09cece..25dafc6 100644 --- a/apps/api/scripts/build_evaluation_review_pack.py +++ b/apps/api/scripts/build_evaluation_review_pack.py @@ -12,6 +12,13 @@ def main() -> None: rows = json.loads((root / "pageindex-v2-review-source.json").read_text(encoding="utf-8")) output = root / "review-pack.md" lines = [ + "", + ( + "" + ), + "", "# PageIndex v2 evaluation review pack", "", "Mark every case approved, edited, or rejected in the JSONL review record before seeding.", diff --git a/apps/api/tests/test_documentation_contracts.py b/apps/api/tests/test_documentation_contracts.py index f950a38..bc2c2b1 100644 --- a/apps/api/tests/test_documentation_contracts.py +++ b/apps/api/tests/test_documentation_contracts.py @@ -1,8 +1,14 @@ from __future__ import annotations +import json import re +import subprocess +import sys +import tomllib from pathlib import Path +import yaml + ROOT = Path(__file__).resolve().parents[3] @@ -62,3 +68,80 @@ def test_helm_secret_references_match_the_deployment_inventory() -> None: assert not missing and not extra, ( f"Helm Secret inventory mismatch: missing={missing}, extra={extra}" ) + + +def test_public_documentation_contract_passes() -> None: + result = subprocess.run( # noqa: S603 + [sys.executable, str(ROOT / "scripts/docs_check.py")], + cwd=ROOT, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + + +def test_documentation_checker_reports_markdown_contract_failures(tmp_path: Path) -> None: + target = tmp_path / "target.md" + target.write_text( + "\n" + "\n\n" + "# Existing heading\n", + encoding="utf-8", + ) + page = tmp_path / "page.md" + page.write_text( + "# Missing declarations\n\n" + "[Missing file](absent.md)\n\n" + "[Missing anchor](target.md#absent-heading)\n\n" + "```\n" + "untagged\n" + "```\n", + encoding="utf-8", + ) + + result = subprocess.run( # noqa: S603 + [ + sys.executable, + str(ROOT / "scripts/docs_check.py"), + "--root", + str(tmp_path), + ], + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode == 1 + assert "missing or invalid content-type" in result.stderr + assert "missing or invalid content-plan" in result.stderr + assert "local link target does not exist" in result.stderr + assert "local link anchor does not exist" in result.stderr + assert "opening code fence has no language tag" in result.stderr + + +def test_license_and_release_metadata_are_consistent() -> None: + pyproject = tomllib.loads((ROOT / "apps/api/pyproject.toml").read_text(encoding="utf-8")) + root_package = json.loads((ROOT / "package.json").read_text(encoding="utf-8")) + web_package = json.loads((ROOT / "apps/web/package.json").read_text(encoding="utf-8")) + chart = yaml.safe_load( + (ROOT / "infra/helm/vectorless-rag/Chart.yaml").read_text(encoding="utf-8") + ) + uv_lock = (ROOT / "apps/api/uv.lock").read_text(encoding="utf-8") + license_text = (ROOT / "LICENSE").read_text(encoding="utf-8") + + assert { + pyproject["project"]["version"], + root_package["version"], + web_package["version"], + chart["version"], + chart["appVersion"], + } == {"0.2.0"} + assert pyproject["project"]["license"] == "MIT" + assert root_package["license"] == "MIT" + assert web_package["license"] == "MIT" + assert chart["annotations"]["artifacthub.io/license"] == "MIT" + assert 'name = "vectorless-rag"\nversion = "0.2.0"' in uv_lock + assert "Copyright (c) 2026 ProofOfTechOrg" in license_text diff --git a/apps/api/tests/test_operator_scripts.py b/apps/api/tests/test_operator_scripts.py index 3e1dcdd..7254547 100644 --- a/apps/api/tests/test_operator_scripts.py +++ b/apps/api/tests/test_operator_scripts.py @@ -1,6 +1,10 @@ from __future__ import annotations import json +import os +import re +import stat +import subprocess import sys from decimal import Decimal from pathlib import Path @@ -159,3 +163,113 @@ def load_manifest(path: Path) -> None: assert output.read_text(encoding="utf-8").endswith("\n") assert not (output.parent / ".result.json.tmp").exists() + + +def test_environment_initializer_is_atomic_private_and_non_overwriting(tmp_path: Path) -> None: + root = API_ROOT.parents[1] + template = root / ".env.example" + output = tmp_path / ".env" + command = [ + sys.executable, + str(root / "scripts/init_env.py"), + "--template", + str(template), + "--output", + str(output), + ] + + processes = [ + subprocess.Popen( # noqa: S603 + command, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + for _ in range(2) + ] + results = [process.communicate(timeout=10) for process in processes] + return_codes = sorted(process.wait() for process in processes) + + assert return_codes == [0, 2] + assert stat.S_IMODE(output.stat().st_mode) == 0o600 + assert not list(tmp_path.glob(".env.tmp-*")) + + assignments = dict( + line.split("=", maxsplit=1) + for line in output.read_text(encoding="utf-8").splitlines() + if line and not line.startswith("#") and "=" in line + ) + assert assignments["DEEPSEEK_API_KEY"] == "" + + hex_keys = { + "SESSION_SECRET", + "POSTGRES_PASSWORD", + "RAG_OWNER_PASSWORD", + "RAG_APP_PASSWORD", + "RAG_READER_PASSWORD", + "API_KEY_PEPPER", + "MINIO_ROOT_PASSWORD", + "CLICKHOUSE_PASSWORD", + "REDIS_PASSWORD", + "LANGFUSE_DB_PASSWORD", + "LANGFUSE_NEXTAUTH_SECRET", + "LANGFUSE_SALT", + "LANGFUSE_ENCRYPTION_KEY", + } + for key in hex_keys: + assert re.fullmatch(r"[0-9a-f]{64}", assignments[key]), key + assert re.fullmatch(r"lf_pk_[0-9a-f]{64}", assignments["LANGFUSE_INIT_PROJECT_PUBLIC_KEY"]) + assert re.fullmatch(r"lf_sk_[0-9a-f]{64}", assignments["LANGFUSE_INIT_PROJECT_SECRET_KEY"]) + assert assignments["LANGFUSE_INIT_USER_PASSWORD"] == "" + + combined_output = "\n".join(value for result in results for value in result) + for key in hex_keys: + assert assignments[key] not in combined_output + + original = output.read_bytes() + repeated = subprocess.run( # noqa: S603 + command, text=True, capture_output=True, check=False + ) + assert repeated.returncode == 2 + assert output.read_bytes() == original + assert "already exists; leaving it unchanged" in repeated.stderr + + +def test_integration_runner_cleans_only_its_project_after_failure(tmp_path: Path) -> None: + root = API_ROOT.parents[1] + call_log = tmp_path / "docker-calls.log" + fake_docker = tmp_path / "docker" + fake_docker.write_text( + '#!/bin/sh\nprintf "%s\\n" "$*" >> "$CALL_LOG"\n', + encoding="utf-8", + ) + fake_docker.chmod(0o755) + + environment = os.environ.copy() + environment.update( + { + "CALL_LOG": str(call_log), + "DOCKER": str(fake_docker), + "UV": "/bin/false", + "MAKE": "/bin/true", + "TEST_POSTGRES_PORT": "55439", + } + ) + result = subprocess.run( # noqa: S603 + [str(root / "scripts/test-integration-stack.sh")], + cwd=root, + env=environment, + text=True, + capture_output=True, + check=False, + ) + + assert result.returncode != 0 + calls = call_log.read_text(encoding="utf-8").splitlines() + assert len(calls) == 2 + project_matches = [ + re.search(r"--project-name (vectorless-rag-integration-[^ ]+)", call) for call in calls + ] + assert all(match is not None for match in project_matches) + project_names = [match.group(1) for match in project_matches if match is not None] + assert len(set(project_names)) == 1 + assert "up -d --wait postgres" in calls[0] + assert "down --volumes --remove-orphans" in calls[1] + assert "--project-name vectorless-rag " not in "\n".join(calls) diff --git a/apps/api/uv.lock b/apps/api/uv.lock index ae9c591..1f13352 100644 --- a/apps/api/uv.lock +++ b/apps/api/uv.lock @@ -1806,7 +1806,7 @@ wheels = [ [[package]] name = "vectorless-rag" -version = "0.1.0" +version = "0.2.0" source = { editable = "." } dependencies = [ { name = "alembic" }, diff --git a/apps/web/package.json b/apps/web/package.json index 5e24ae3..53f135e 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -1,6 +1,15 @@ { "name": "@vectorless-rag/web", + "version": "0.2.0", + "description": "Operator console for Vectorless RAG", "private": true, + "license": "MIT", + "homepage": "https://github.com/ProofOfTechOrg/vectorless-rag#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ProofOfTechOrg/vectorless-rag.git", + "directory": "apps/web" + }, "type": "module", "imports": { "#/*": "./src/*" diff --git a/apps/web/scripts/verify-playwright-install.mjs b/apps/web/scripts/verify-playwright-install.mjs new file mode 100644 index 0000000..1692f8b --- /dev/null +++ b/apps/web/scripts/verify-playwright-install.mjs @@ -0,0 +1,4 @@ +import { chromium } from '@playwright/test' + +const browser = await chromium.launch({ headless: true }) +await browser.close() diff --git a/docs/DEFERRED.md b/docs/DEFERRED.md index 8783fd6..06bf3a5 100644 --- a/docs/DEFERRED.md +++ b/docs/DEFERRED.md @@ -1,4 +1,5 @@ + # Deferred hardening @@ -6,10 +7,6 @@ This page records accepted hardening work that remains outside the current imple ## Adopt Basedpyright recommended mode -Move from Basedpyright's `strict` mode to its full `recommended` mode. Resolve every -additional warning and error directly rather than adding a baseline or broad rule -suppression. The work should cover explicit and propagated `Any`, unannotated class -attributes, unused call results and parameters, implicit string concatenation, and -Basedpyright-specific safety rules. +Move from Basedpyright’s `strict` mode to its full `recommended` mode. Resolve every additional warning and error directly instead of adding a baseline or broad rule suppression. Cover explicit and propagated `Any`, unannotated class attributes, unused call results and parameters, implicit string concatenation, and Basedpyright-specific safety rules. Completion requires `make api-typecheck` to finish with zero errors and zero warnings. diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..85c29db --- /dev/null +++ b/docs/README.md @@ -0,0 +1,47 @@ + + + +# Find Vectorless RAG documentation + +Use this index to choose the shortest document for your current task. The [project overview](../README.md) explains status, fit, architecture, and the first-run path. + +## Start and use the research preview + +These pages take you from installation to a cited answer: + +- [Get started with one cited answer](getting-started.md) +- [Troubleshoot the local stack](troubleshooting.md) +- [Operate Vectorless RAG](operator-guide.md) +- [Command reference](commands.md) +- [Vectorless RAG API package](../apps/api/README.md) + +## Contribute and get help + +These project policies define contribution and support channels: + +- [Contribute to Vectorless RAG](../CONTRIBUTING.md) +- [Contributor Covenant Code of Conduct](../CODE_OF_CONDUCT.md) +- [Get support for Vectorless RAG](../SUPPORT.md) +- [Report a Vectorless RAG security vulnerability](../SECURITY.md) +- [Track Vectorless RAG releases](../CHANGELOG.md) +- [Review third-party software notices](../THIRD_PARTY_NOTICES.md) + +## Test, deploy, and recover + +These pages cover provider-free gates and operating environments: + +- [Choose a test tier](testing.md) +- [Deploy Vectorless RAG](deployment.md) +- [Understand the security model](security.md) +- [Back up and restore Vectorless RAG](backup-restore.md) + +## Run research and evaluation + +These pages document evaluation workflows and frozen historical conclusions: + +- [Run evaluation workflows](evaluation.md) +- [Review the generated PageIndex evaluation cases](../apps/api/evaluation/review-pack.md) +- [Calculate provider costs for PageIndex and vector RAG](token-cost-analysis.md) +- [Compare vector RAG with the PageIndex system](vector-rag-vs-pageindex-analysis.md) +- [Harden PageIndex before comparing retrieval architectures](pageindex-improvement-analysis.md) +- [Review deferred hardening](DEFERRED.md) diff --git a/docs/backup-restore.md b/docs/backup-restore.md index dde1a1e..1338bad 100644 --- a/docs/backup-restore.md +++ b/docs/backup-restore.md @@ -1,4 +1,5 @@ + # Back up and restore Vectorless RAG diff --git a/docs/commands.md b/docs/commands.md index efff5a4..922241b 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -1,4 +1,5 @@ + # Command reference @@ -6,16 +7,16 @@ This page lists every public Make target and the variables that alter operator c | Area | Commands | | --- | --- | -| Discovery and environment | `make help`, `make env` | +| Discovery and environment | `make help`, `make env`, `make init` | | Dependencies | `make install`, `make api-install`, `make web-install` | | Formatting | `make format`, `make api-format`, `make web-format` | | Linting | `make lint`, `make api-lint`, `make web-lint` | | Type checking | `make typecheck`, `make api-typecheck`, `make web-typecheck` | -| Unit and integration tests | `make test`, `make api-test`, `make web-test`, `make test-unit`, `make test-integration` | +| Unit and integration tests | `make test`, `make api-test`, `make web-test`, `make test-unit`, `make test-integration`, `make test-integration-stack` | | Package builds | `make api-build`, `make web-build`, `make web-production-smoke` | | Browser tests | `make playwright-install`, `make playwright`, `make playwright-live PLAYWRIGHT_BASE_URL=http://127.0.0.1:3001` | | Web diagnostics | `make react-doctor`, `make react-doctor-diff`, `make astryx-doctor` | -| Manifest validation | `make workflow-check`, `make migration-check`, `make compose-check`, `make helm-lint`, `make helm-template`, `make helm-check` | +| Documentation and manifest validation | `make docs-check`, `make workflow-check`, `make migration-check`, `make compose-check`, `make helm-lint`, `make helm-template`, `make helm-check` | | Image builds | `make docker-build-api`, `make docker-build-web`, `make docker-build` | | Validation suites | `make api-check`, `make web-check`, `make deploy-check`, `make check ENV_FILE=.env.example`, `make ci` | | Lifecycle | `make up`, `make rebuild`, `make dev`, `make dev-build`, `make dev-health`, `make down`, `make stop`, `make restart`, `make restart-app`, `make restart-web`, `make restart-langfuse`, `make restart-observability`, `make ps`, `make logs SERVICE=worker`, `make health`, `make web-dev` | @@ -26,11 +27,17 @@ This page lists every public Make target and the variables that alter operator c `make pilot-run` writes its immutable result to `apps/api/pilots/run/result.json`. Set `PILOT_OUTPUT=path/to/result.json` to select another new output directory. The target fails before provider work when that directory already exists. +`make init` atomically creates a mode-600 `.env` with generated local infrastructure and session secrets. It leaves `DEEPSEEK_API_KEY` blank and refuses to overwrite an existing file. + +`make test-integration-stack` uses port `55432` and a unique disposable Compose project. Set `TEST_POSTGRES_PORT` to another unused port. The cleanup trap removes only that project and its volumes. + +`make docs-check` validates required community files, local Markdown targets and anchors, content declarations, language-tagged fences, and this documentation index. + `make evaluation-seed` streams the reviewed PageIndex suite into the API container with an explicit suite selector and corpus manifest. Direct script callers must pass `--suite pageindex-v2` when the dataset arrives through `/dev/stdin`. `make playwright-live` exits with status 2 unless both `PLAYWRIGHT_BASE_URL` and the process-only `PLAYWRIGHT_API_KEY` are set. Use the [disposable live-stack procedure](operator-guide.md#verify-a-disposable-live-stack) to create both values without saving the key. -`make ci` installs Playwright Chromium and its operating-system dependencies through `make playwright-install` before it runs browser tests. +`make ci` runs the isolated PostgreSQL integration gate and installs Playwright Chromium with its operating-system dependencies before browser tests. ## Run the local development stack diff --git a/docs/deployment.md b/docs/deployment.md index ff9f7c7..b90fd75 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -1,4 +1,5 @@ + # Deploy Vectorless RAG diff --git a/docs/evaluation.md b/docs/evaluation.md index 52db9b7..6498409 100644 --- a/docs/evaluation.md +++ b/docs/evaluation.md @@ -1,4 +1,5 @@ + # Run evaluation workflows diff --git a/docs/getting-started.md b/docs/getting-started.md new file mode 100644 index 0000000..c2d1b0b --- /dev/null +++ b/docs/getting-started.md @@ -0,0 +1,93 @@ + + + +# Get started with one cited answer + +This tutorial starts the local stack, ingests one born-digital PDF, and asks one grounded question. It preserves service data when you stop. + +## Check prerequisites + +You need: + +- Linux or macOS with Docker Engine and Docker Compose 2.22 or later +- GNU Make +- Python 3.12 and `uv` +- A DeepSeek API key with available credit +- One born-digital PDF whose text you may send to DeepSeek + +Ingestion and chat incur provider charges. The project defaults limit registration to 25 documents and cap the frozen pilot at $10. + +## Clone and initialize the repository + +Clone the public repository and create `.env` with generated local secrets: + +```bash +git clone https://github.com/ProofOfTechOrg/vectorless-rag.git +cd vectorless-rag +make init +``` + +`make init` writes `.env` with mode `600`, leaves `DEEPSEEK_API_KEY` blank, and refuses to overwrite an existing file. It never prints generated secrets. + +## Configure DeepSeek + +Open `.env` and assign your provider key: + +```dotenv +DEEPSEEK_API_KEY=your_deepseek_api_key_here +``` + +Keep `DEEPSEEK_MODEL=deepseek-v4-pro` and `PAGEINDEX_MODEL=deepseek/deepseek-v4-pro`. Confirm current availability and cost in the [official DeepSeek model and pricing reference](https://api-docs.deepseek.com/quick_start/pricing/). + +Do not commit `.env` or copy its values into logs, issues, shell history, or browser storage. + +## Start the complete stack + +Start the deployment-style Compose stack: + +```bash +make up +make health +``` + +The first start builds application images and downloads infrastructure images. `make health` succeeds after the console, API, and Langfuse respond. + +The local services use these addresses: + +| Service | Address | +| --- | --- | +| Operator console | [http://localhost:3001](http://localhost:3001) | +| FastAPI documentation | [http://localhost:8000/docs](http://localhost:8000/docs) | +| Langfuse | [http://localhost:3000](http://localhost:3000) | + +## Mint and use a scoped key + +Create a key with the scopes needed for chat and ingestion: + +```bash +make bootstrap-key NAME=browser-operator SCOPES=chat,documents:read,admin:ingest +``` + +Copy the one-time `vrag_…` value directly to the **Connect** page. The BFF validates it through FastAPI and stores it in an encrypted HttpOnly cookie for eight hours. + +## Ingest one PDF + +Open **Ingestion**, choose **Upload PDF**, and select one born-digital file. Wait until the durable job reports **Succeeded**. + +The worker sends document content to DeepSeek during PageIndex construction. Stop and read [troubleshooting](troubleshooting.md) if the job reports a provider wait, unsupported document, or terminal failure. + +## Ask a cited question + +Open **Chat** and ask a question whose answer appears in the uploaded document. A successful response streams to completion and includes a citation with the document and page number. + +Open the citation to verify that its page and quoted evidence support the answer. The console links to document detail because v0.2.0 does not include an embedded PDF viewer. + +## Stop without deleting data + +Stop and remove containers while preserving named volumes: + +```bash +make down +``` + +Run `make up` to use the same database and artifacts later. Do not add `-v` unless you intend to delete all local service data. diff --git a/docs/operator-guide.md b/docs/operator-guide.md index ccc411d..877002b 100644 --- a/docs/operator-guide.md +++ b/docs/operator-guide.md @@ -1,4 +1,5 @@ + # Operate Vectorless RAG diff --git a/docs/pageindex-improvement-analysis.md b/docs/pageindex-improvement-analysis.md index 12e792c..58f0483 100644 --- a/docs/pageindex-improvement-analysis.md +++ b/docs/pageindex-improvement-analysis.md @@ -1,7 +1,10 @@ - + + # Harden PageIndex before comparing retrieval architectures +> **Dated design record, July 2026:** This document preserves the decisions and evidence from its recorded repository baseline. Do not recalculate its historical conclusions with current dependencies or pricing. + This record defines the PageIndex v2 implementation baseline, architecture, rollout, and verification evidence. It is the durable handoff for the work that starts at commit `cfbea604c64d51c20ca1cea413b2bfa35be94471`. ## Baseline @@ -398,10 +401,7 @@ The implementation does not apply historical classifications, run the credential ### Verification results -All results below are from the final working-tree source state. The PageIndex -patch was regenerated as a canonical, ordered diff against the pinned upstream -tree after a clean-clone image build exposed whitespace-sensitive hunks in the -earlier patch file. +All results below are from the final working-tree source state. The PageIndex patch was regenerated as a canonical, ordered diff against the pinned upstream tree after a clean-clone image build exposed whitespace-sensitive hunks in the earlier patch file. | Command or gate | Result | | --- | --- | @@ -418,7 +418,4 @@ earlier patch file. | Disposable PostgreSQL migration and integration gate | Passed: both migrations applied at schema head and 4 integration tests passed. Validation databases and roles were then removed. | | `git diff --check` | Passed. Patch files explicitly disable end-of-line whitespace diagnostics because their context must preserve pinned upstream whitespace. | -The credential-safe live fixture, exact seven-event historical classification, -and paid 25-PDF pilot were not executed. Consequently there are no cleanup -event IDs, pilot run IDs, v2 provider spend, or v2 quality results to append. -Those are rollout operations, not missing deterministic verification. +The credential-safe live fixture, exact seven-event historical classification, and paid 25-PDF pilot were not executed. Consequently there are no cleanup event IDs, pilot run IDs, v2 provider spend, or v2 quality results to append. Those are rollout operations, not missing deterministic verification. diff --git a/docs/security.md b/docs/security.md index b57d840..da0f662 100644 --- a/docs/security.md +++ b/docs/security.md @@ -1,4 +1,5 @@ + # Understand the security model diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..d6692e5 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,67 @@ + + + +# Choose a test tier + +This reference maps each verification command to its dependencies, provider use, and completion signal. Continuous integration (CI) runs only provider-free tiers. + +## Install frozen dependencies + +Run the repository install gate once: + +```bash +make install +``` + +The pnpm and uv configurations delay newly published dependencies for seven days. Both package managers install from committed lockfiles. + +## Run provider-free tiers + +Use these tiers during development: + +| Tier | Command | External requirements | Done when | +| --- | --- | --- | --- | +| Documentation | `make docs-check` | None after install | Community files, links, anchors, declarations, fences, and index pass | +| API | `make api-check` | None | Lint, type checks, tests, and offline migrations pass | +| PostgreSQL | `make test-integration-stack` | Docker | All integration tests run against owner and reader roles, then the isolated volume is removed | +| Web | `make web-check` | None | Lint, type checks, unit tests, React Doctor, production smoke, and ASTRYX Doctor pass | +| Browser | `make playwright-install && make playwright` | Chromium installation | Deterministic browser tests pass against the mock upstream | +| Deployment | `make deploy-check ENV_FILE=.env.example` | Docker and Helm 4.2.3 | Workflow, Compose, and Helm validation pass | +| Images | `make docker-build` | Docker | API and web production images build | + +Override the isolated PostgreSQL port when `55432` is occupied: + +```bash +TEST_POSTGRES_PORT=55433 make test-integration-stack +``` + +The runner creates a unique Compose project. Its cleanup trap addresses only that project and its volumes. + +## Run deterministic evaluation checks + +The PageIndex evaluation schema and review-pack generator need no PDF, database, model, or Langfuse: + +```bash +make evaluation-check +make evaluation-review-pack +``` + +The generated review pack supports human source review. It does not produce a model-quality score. + +## Run the local CI equivalent + +Run every provider-free gate, browser suite, integration stack, and image build: + +```bash +make ci +``` + +The command uses `.env.example` only for manifest rendering. It does not start the normal Compose project or read `DEEPSEEK_API_KEY`. + +## Authorize paid live verification separately + +`make playwright-live` uses a real DeepSeek-backed stack and can incur cost. Run it only for an explicitly authorized release candidate. + +Follow [verify a disposable live stack](operator-guide.md#verify-a-disposable-live-stack). The release gate must cover upload, durable ingestion, Server-Sent Events (SSE) completion, document and page citation, thread history, feedback, document detail, identity and scopes, theme, responsive navigation, and logout. + +Stop the release if ingestion or chat fails. Never add provider credentials to CI. diff --git a/docs/token-cost-analysis.md b/docs/token-cost-analysis.md index dbaaa51..b9e57b6 100644 --- a/docs/token-cost-analysis.md +++ b/docs/token-cost-analysis.md @@ -3,6 +3,8 @@ # Calculate provider costs for PageIndex and vector RAG +> **Dated design record, July 2026:** This document preserves measurements, pricing, and projections from its recorded baselines. Check current provider prices before authorizing spend. + This quantitative appendix calculates provider cost for PageIndex-based and embedding-based retrieval-augmented generation (RAG) over 537 Portable Document Format (PDF) files. It records corpus measurements, token formulas, live ingestion accounting, historical query traces, sensitivity analysis, and volume projections. Read [Compare vector RAG with the PageIndex system](vector-rag-vs-pageindex-analysis.md) for retrieval quality, latency, scaling, security, and the product decision. ## Measurement baselines @@ -95,9 +97,9 @@ The durable report separates metadata and PageIndex calls across every recorded The report includes 10 completed and seven failed ingestion attempts. Four calls lack reported usage, and 15 older jobs predate durable attempt accounting. The measured cost is therefore known post-migration spend, not complete lifetime spend. -The ingestion path extracts up to 60,000 characters from the first three pages and asks DeepSeek for structured bibliographic metadata. See [`extract_metadata`](../apps/api/src/vectorless_rag/ingestion.py#L131-L153). +The ingestion path extracts up to 60,000 characters from the first three pages and asks DeepSeek for structured bibliographic metadata. See [`extract_metadata` in `ingestion.py`](../apps/api/src/vectorless_rag/ingestion.py). -PageIndex sends page and section text to DeepSeek while it builds structure, maps headings to pages, repairs mappings, summarizes nodes, and generates a document description. The adapter enables node summaries and document descriptions in [`PageIndexAdapter.index`](../apps/api/src/vectorless_rag/pageindex_adapter.py#L117-L231). +PageIndex sends page and section text to DeepSeek while it builds structure, maps headings to pages, repairs mappings, summarizes nodes, and generates a document description. The adapter enables node summaries and document descriptions in [`PageIndexAdapter.index` in `pageindex_adapter.py`](../apps/api/src/vectorless_rag/pageindex_adapter.py). ### Current `vr3` full-corpus projection @@ -218,7 +220,7 @@ The stages are: 5. Synthesize the answer with citations 6. Repeat node selection and sufficiency once when the first evidence set is insufficient -The route and synthesis calls live in [`GraphRunner.run`](../apps/api/src/vectorless_rag/graph.py#L227-L524). Candidate selection, tree-node selection, and sufficiency evaluation live in [`retrieval.py`](../apps/api/src/vectorless_rag/retrieval.py#L124-L246). +The route and synthesis calls live in [`GraphRunner.run` in `graph.py`](../apps/api/src/vectorless_rag/graph.py). Candidate selection, tree-node selection, and sufficiency evaluation live in [`retrieval.py`](../apps/api/src/vectorless_rag/retrieval.py). The configured retrieval limits are: @@ -232,7 +234,7 @@ The configured retrieval limits are: | Retrieval rounds | 2 | | Structured-output attempts per call | 3 | -See [`Settings`](../apps/api/src/vectorless_rag/config.py#L21-L59) for the limits except retrieval rounds, which are fixed in [`PageIndexRetriever.retrieve`](../apps/api/src/vectorless_rag/retrieval.py#L179-L246). +See [`Settings` in `config.py`](../apps/api/src/vectorless_rag/config.py) for the limits except retrieval rounds, which are fixed in [`PageIndexRetriever.retrieve` in `retrieval.py`](../apps/api/src/vectorless_rag/retrieval.py). ### Historical document-answer cost @@ -273,10 +275,7 @@ The successful document-answer sample is small. Treat its average as a measured ### Historical v1 candidate-selection effect -The v1 implementation filled a result set to `candidate_limit` and -hard-excluded documents below that lexical boundary. The ten-document vr3 -pilot’s candidate-selection calls consumed about 3,026 input tokens. Its -former linear projection to 50 documents was: +The v1 implementation filled a result set to `candidate_limit` and hard-excluded documents below that lexical boundary. The ten-document vr3 pilot’s candidate-selection calls consumed about 3,026 input tokens. Its former linear projection to 50 documents was: ```text pilot successful answer input ≈ 10,738 tokens @@ -286,16 +285,9 @@ observed output ≈ 1,678 tokens projected cache-miss answer cost ≈ $0.0114 ``` -Confidence: moderate-low. This is retained only as a vr3 historical baseline; -it no longer describes production retrieval. +Confidence: moderate-low. This is retained only as a vr3 historical baseline; it no longer describes production retrieval. -The v2 retriever reasons over every eligible document in deterministic, -token-bounded catalog batches and then reduces the batch selections. Lexical -rank is a signal, not a recall cutoff. That design removes the 50-document -blind spot but can issue multiple map calls as the catalog grows. No v2 -full-corpus cost is reported yet: the 25-document pilot must freeze one recipe, -verify every artifact, and record measured and reserved spend before a -projection is eligible. +The v2 retriever reasons over every eligible document in deterministic, token-bounded catalog batches and then reduces the batch selections. Lexical rank is a signal, not a recall cutoff. That design removes the 50-document blind spot but can issue multiple map calls as the catalog grows. No v2 full-corpus cost is reported yet: the 25-document pilot must freeze one recipe, verify every artifact, and record measured and reserved spend before a projection is eligible. The 40,000-token evidence limit is not a request-level billing cap. The system can submit that evidence to sufficiency evaluation more than once and then submit it again for synthesis. Structured-output retries can repeat any failed call up to three times. @@ -369,7 +361,7 @@ No token-cost break-even emerges from higher query volume under these assumption Index invalidation changes long-term cost even when the first index fits the budget. -The PageIndex artifact key includes the PDF hash, PageIndex version, metadata prompt version, and PageIndex model. See [`artifact_version_key`](../apps/api/src/vectorless_rag/pageindex_adapter.py#L264-L268) and its use in [`IngestionProcessor.process`](../apps/api/src/vectorless_rag/ingestion.py#L316-L328). +The PageIndex artifact key includes the PDF hash, PageIndex version, metadata prompt version, and PageIndex model. See [`artifact_version_key` in `pageindex_adapter.py`](../apps/api/src/vectorless_rag/pageindex_adapter.py) and its use in [`IngestionProcessor.process` in `ingestion.py`](../apps/api/src/vectorless_rag/ingestion.py). Changing any of these values creates a new artifact version and can trigger the full PageIndex cost: diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md new file mode 100644 index 0000000..3362f2c --- /dev/null +++ b/docs/troubleshooting.md @@ -0,0 +1,99 @@ + + + +# Troubleshoot the local stack + +Use the symptom headings below to diagnose local failures. Preserve `.env`, `arxiv-pdfs`, and named volumes until you identify the affected boundary. + +## A host port is already allocated + +Docker reports an address or port allocation error when another process uses a published port. + +Check Compose status and listening processes: + +```bash +docker compose ps +ss -ltn +``` + +The default host ports are console `3001`, Langfuse `3000`, FastAPI `8000`, PostgreSQL `5432`, MinIO console `9001`, ClickHouse HTTP `8123`, and ClickHouse native `9000`. + +Change the matching value in `.env` before startup. Use `TEST_POSTGRES_PORT=55433 make test-integration-stack` when only the disposable integration port is occupied. + +## Session or encryption secrets are rejected + +The web service rejects `SESSION_SECRET` unless it encodes exactly 32 random bytes as 64 hexadecimal characters. Langfuse applies the same 64-character format to `LANGFUSE_ENCRYPTION_KEY`. + +Create a fresh local environment only when `.env` does not exist: + +```bash +make init +``` + +If `.env` already contains application data credentials, edit only the invalid value. Changing `SESSION_SECRET` logs out browser sessions. Changing `API_KEY_PEPPER` invalidates existing Vectorless RAG API keys. + +## PostgreSQL rejects credentials after an environment change + +The PostgreSQL image creates roles only when it initializes an empty volume. Changing database passwords in `.env` does not update roles in an existing volume. + +Restore the previous values or rotate passwords inside PostgreSQL. Do not delete the normal volume merely to align a changed `.env`. + +For disposable development data only, inspect the exact Compose project before resetting it: + +```bash +docker compose ps +docker compose down +``` + +Adding `--volumes` permanently deletes that project’s database and service data. Never run it against an uncertain project name. + +## Ingestion waits for the provider + +The console can report unavailable credit, rate limiting, authentication failure, or a retry cooldown. Check **Settings** for sanitized provider state and confirm: + +- `DEEPSEEK_API_KEY` belongs to the intended account +- The account has available credit +- `deepseek-v4-pro` remains listed in the [official DeepSeek model and pricing reference](https://api-docs.deepseek.com/quick_start/pricing/) +- The configured retry cooldown has elapsed + +Restarting healthy services does not bypass provider cooldown or account state. + +## A PDF is unsupported or ingestion fails + +Version 0.2.0 supports born-digital PDFs. It does not perform optical character recognition (OCR). + +Open the PDF locally and confirm that you can select its text. Use one small file for diagnosis. Inspect sanitized worker logs without copying document text or provider response bodies: + +```bash +make logs SERVICE=worker +``` + +The mounted-corpus sync scans direct PDF children only. It does not recurse into directories. + +## Health checks do not pass + +Inspect service state and bounded logs: + +```bash +make ps +make logs SERVICE=api TAIL=200 +make logs SERVICE=web TAIL=200 +make health +``` + +PostgreSQL and MinIO must become healthy before the API starts. The web service then waits for API readiness. Langfuse also depends on PostgreSQL, ClickHouse, Redis, and MinIO. + +Run `make deploy-check ENV_FILE=.env.example` when Compose configuration fails before container startup. + +## Reset only disposable state + +These commands preserve named volumes: + +```bash +make stop +make down +``` + +`docker compose down -v` deletes named volumes for the selected project. Use it only for a confirmed disposable project such as the isolated integration or release-test stack. + +Never move, modify, ingest, or delete `arxiv-pdfs` while troubleshooting project infrastructure. That directory is user-owned and outside service cleanup. diff --git a/docs/vector-rag-vs-pageindex-analysis.md b/docs/vector-rag-vs-pageindex-analysis.md index dc9ff24..d42bb9d 100644 --- a/docs/vector-rag-vs-pageindex-analysis.md +++ b/docs/vector-rag-vs-pageindex-analysis.md @@ -3,7 +3,7 @@ # Compare vector RAG with the PageIndex system -> **Provisional:** This comparison predates hierarchical PageIndex v2, complete-catalog retrieval, frozen full-corpus indexes, and a sealed post-freeze evaluation dataset. Keep the vr3 measurements as historical evidence, but do not use them as a final architecture verdict. +> **Dated design record, July 2026:** This provisional comparison predates hierarchical PageIndex v2, complete-catalog retrieval, frozen full-corpus indexes, and a sealed post-freeze evaluation dataset. Keep the vr3 measurements as historical evidence, but do not use them as a final architecture verdict. This analysis compares conventional embedding and vector-search retrieval-augmented generation (RAG) with this repository’s PageIndex-based vectorless RAG implementation. It covers the implemented retrieval path, indexing lifecycle, retrieval quality, citations, latency, provider cost, scaling, failure modes, and the evidence required for a final product decision. @@ -297,11 +297,7 @@ Historical traces predate durable query-call accounting. The prior trace analysi The measured pilot PageIndex answer costs 1.7 times the vector baseline. The full-candidate projection costs 3.2 times the vector baseline. -The pre-v2 57-run cohort cannot validate those values because its query calls -predate durable model-call accounting; the migration deliberately does not -invent or backfill usage. New runs write durable call rows. Reports must keep -the historical cohort marked legacy/incomplete and may assign zero model cost -only to static help responses that made no model call. +The pre-v2 57-run cohort cannot validate those values because its query calls predate durable model-call accounting; the migration deliberately does not invent or backfill usage. New runs write durable call rows. Reports must keep the historical cohort marked legacy/incomplete and may assign zero model cost only to static help responses that made no model call. ## Scaling and operations @@ -312,12 +308,7 @@ The historical vr3 PageIndex path had two separate scaling limits: - Corpus search cannot retrieve a document ranked below the 50-document lexical boundary - Query cost grows when the selection model receives more candidate metadata -The v2 implementation removes the lexical exclusion and uses complete-catalog, -token-bounded map/reduce selection before nested tree search. Query cost still -grows with catalog size, but in explicit catalog batches rather than a -fixed-recall gate. This document remains provisional until the frozen -25-document v2 pilot is accepted and both comparison arms are frozen against a -new sealed evaluation dataset. +The v2 implementation removes the lexical exclusion and uses complete-catalog, token-bounded map/reduce selection before nested tree search. Query cost still grows with catalog size, but in explicit catalog batches rather than a fixed-recall gate. This document remains provisional until the frozen 25-document v2 pilot is accepted and both comparison arms are frozen against a new sealed evaluation dataset. Vector indexing also has a smaller retry surface. An embedding pipeline can batch chunks, retry failed batches independently, and persist successful batches. PageIndex can spend dozens of model calls before a late validation or provider error invalidates an attempt. diff --git a/infra/helm/vectorless-rag/Chart.yaml b/infra/helm/vectorless-rag/Chart.yaml index 7700b21..9a3321d 100644 --- a/infra/helm/vectorless-rag/Chart.yaml +++ b/infra/helm/vectorless-rag/Chart.yaml @@ -4,3 +4,12 @@ description: Full-stack Vectorless RAG with TanStack Start, FastAPI, worker, and type: application version: 0.2.0 appVersion: "0.2.0" +home: https://github.com/ProofOfTechOrg/vectorless-rag +sources: + - https://github.com/ProofOfTechOrg/vectorless-rag +maintainers: + - name: ProofOfTechOrg + email: hello@proofoftech.org + url: https://github.com/ProofOfTechOrg +annotations: + artifacthub.io/license: MIT diff --git a/package.json b/package.json index 09e2ac9..1e06f15 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,14 @@ { "name": "vectorless-rag", + "version": "0.2.0", + "description": "Self-hosted vectorless RAG for grounded question answering over local PDFs", "private": true, + "license": "MIT", + "homepage": "https://github.com/ProofOfTechOrg/vectorless-rag#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ProofOfTechOrg/vectorless-rag.git" + }, "packageManager": "pnpm@11.5.2", "engines": { "node": ">=22", diff --git a/scripts/docs_check.py b/scripts/docs_check.py new file mode 100755 index 0000000..ef47f65 --- /dev/null +++ b/scripts/docs_check.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import re +import sys +from collections import Counter +from pathlib import Path +from urllib.parse import unquote + +REQUIRED_FILES = { + "CHANGELOG.md", + "CODE_OF_CONDUCT.md", + "CONTRIBUTING.md", + "LICENSE", + "README.md", + "SECURITY.md", + "SUPPORT.md", + "THIRD_PARTY_NOTICES.md", + ".github/ISSUE_TEMPLATE/bug_report.yml", + ".github/ISSUE_TEMPLATE/config.yml", + ".github/ISSUE_TEMPLATE/feature_request.yml", + ".github/PULL_REQUEST_TEMPLATE.md", + "docs/README.md", + "docs/getting-started.md", + "docs/testing.md", + "docs/troubleshooting.md", +} +CONTENT_TYPE = re.compile( + r"^$" +) +CONTENT_PLAN = re.compile( + r"^$" +) +INLINE_LINK = re.compile(r"!?\[[^\]]*]\(([^)\s]+)(?:\s+[\"'][^\"']*[\"'])?\)") +REFERENCE_LINK = re.compile(r"^\s*\[[^\]]+]:\s*(\S+)", re.MULTILINE) +HTML_TAG = re.compile(r"<[^>]+>") +PUNCTUATION = re.compile(r"[^\w\s-]", re.UNICODE) + + +def public_markdown_files(root: Path) -> list[Path]: + candidates = [ + *root.glob("*.md"), + *root.glob("docs/**/*.md"), + root / "apps/api/README.md", + root / "apps/api/evaluation/review-pack.md", + ] + excluded = {"CLAUDE.md"} + return sorted( + { + path + for path in candidates + if path.is_file() and path.name not in excluded and ".venv" not in path.parts + } + ) + + +def github_anchor(heading: str) -> str: + value = HTML_TAG.sub("", heading).strip().lower() + value = PUNCTUATION.sub("", value) + return re.sub(r"\s+", "-", value) + + +def heading_anchors(text: str) -> set[str]: + counts: Counter[str] = Counter() + anchors: set[str] = set() + in_fence = False + for line in text.splitlines(): + if line.lstrip().startswith("```"): + in_fence = not in_fence + continue + if in_fence: + continue + match = re.match(r"^#{1,6}\s+(.+?)\s*#*\s*$", line) + if not match: + continue + base = github_anchor(match.group(1)) + suffix = counts[base] + counts[base] += 1 + anchors.add(base if suffix == 0 else f"{base}-{suffix}") + return anchors + + +def validate_markdown_file(root: Path, path: Path) -> list[str]: + relative = path.relative_to(root) + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + errors: list[str] = [] + + if not any(CONTENT_TYPE.fullmatch(line) for line in lines[:6]): + errors.append(f"{relative}: missing or invalid content-type declaration") + if not any(CONTENT_PLAN.fullmatch(line) for line in lines[:6]): + errors.append(f"{relative}: missing or invalid content-plan declaration") + + in_fence = False + for number, line in enumerate(lines, start=1): + stripped = line.lstrip() + if not stripped.startswith("```"): + continue + if in_fence: + in_fence = False + continue + language = stripped[3:].strip() + if not language: + errors.append(f"{relative}:{number}: opening code fence has no language tag") + in_fence = True + if in_fence: + errors.append(f"{relative}: unclosed code fence") + + links = [*INLINE_LINK.findall(text), *REFERENCE_LINK.findall(text)] + for raw_target in links: + target = raw_target.strip("<>") + if target.startswith(("http://", "https://", "mailto:", "tel:")): + continue + path_part, _, fragment = target.partition("#") + target_path = path if not path_part else (path.parent / unquote(path_part)).resolve() + if not target_path.exists(): + errors.append(f"{relative}: local link target does not exist: {target}") + continue + if fragment and target_path.is_file() and target_path.suffix.lower() == ".md": + target_text = target_path.read_text(encoding="utf-8") + if unquote(fragment).lower() not in heading_anchors(target_text): + errors.append(f"{relative}: local link anchor does not exist: {target}") + return errors + + +def validate_index(root: Path, markdown_files: list[Path]) -> list[str]: + index_path = root / "docs/README.md" + if not index_path.is_file(): + return ["docs/README.md: documentation index is missing"] + index_text = index_path.read_text(encoding="utf-8") + indexed_paths: set[Path] = set() + for raw_target in INLINE_LINK.findall(index_text): + target = raw_target.strip("<>").partition("#")[0] + if not target or target.startswith(("http://", "https://", "mailto:", "tel:")): + continue + indexed_paths.add((index_path.parent / unquote(target)).resolve()) + + errors: list[str] = [] + for path in markdown_files: + if path == index_path: + continue + if path.resolve() not in indexed_paths: + errors.append(f"docs/README.md: public document is not indexed: {path.relative_to(root)}") + return errors + + +def check(root: Path) -> list[str]: + errors: list[str] = [] + for required in sorted(REQUIRED_FILES): + if not (root / required).is_file(): + errors.append(f"{required}: required file is missing") + + markdown_files = public_markdown_files(root) + for path in markdown_files: + errors.extend(validate_markdown_file(root, path)) + errors.extend(validate_index(root, markdown_files)) + return errors + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Validate the public documentation contract.") + parser.add_argument("--root", type=Path, default=Path.cwd()) + return parser.parse_args() + + +def main() -> int: + root = parse_args().root.resolve() + errors = check(root) + if errors: + print("\n".join(errors), file=sys.stderr) + return 1 + print(f"documentation contract passed for {len(public_markdown_files(root))} pages") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/init_env.py b/scripts/init_env.py new file mode 100755 index 0000000..8e63a8f --- /dev/null +++ b/scripts/init_env.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import os +import secrets +import sys +from pathlib import Path + + +def generated_values() -> dict[str, str]: + return { + "SESSION_SECRET": secrets.token_hex(32), + "POSTGRES_PASSWORD": secrets.token_hex(32), + "RAG_OWNER_PASSWORD": secrets.token_hex(32), + "RAG_APP_PASSWORD": secrets.token_hex(32), + "RAG_READER_PASSWORD": secrets.token_hex(32), + "API_KEY_PEPPER": secrets.token_hex(32), + "MINIO_ROOT_PASSWORD": secrets.token_hex(32), + "CLICKHOUSE_PASSWORD": secrets.token_hex(32), + "REDIS_PASSWORD": secrets.token_hex(32), + "LANGFUSE_DB_PASSWORD": secrets.token_hex(32), + "LANGFUSE_NEXTAUTH_SECRET": secrets.token_hex(32), + "LANGFUSE_SALT": secrets.token_hex(32), + "LANGFUSE_ENCRYPTION_KEY": secrets.token_hex(32), + "LANGFUSE_INIT_PROJECT_PUBLIC_KEY": f"lf_pk_{secrets.token_hex(32)}", + "LANGFUSE_INIT_PROJECT_SECRET_KEY": f"lf_sk_{secrets.token_hex(32)}", + } + + +def render(template: str, values: dict[str, str]) -> str: + remaining = set(values) + lines: list[str] = [] + for line in template.splitlines(keepends=True): + key, separator, _ = line.partition("=") + if separator and key in values: + lines.append(f"{key}={values[key]}\n") + remaining.remove(key) + else: + lines.append(line) + + if remaining: + missing = ", ".join(sorted(remaining)) + raise ValueError(f"template is missing required assignments: {missing}") + return "".join(lines) + + +def create_atomically(template_path: Path, output_path: Path) -> None: + if output_path.exists(): + raise FileExistsError(f"{output_path} already exists; leaving it unchanged") + + content = render(template_path.read_text(encoding="utf-8"), generated_values()) + temporary_path = output_path.with_name( + f".{output_path.name}.tmp-{os.getpid()}-{secrets.token_hex(8)}" + ) + descriptor = os.open(temporary_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, "w", encoding="utf-8") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + os.link(temporary_path, output_path) + finally: + temporary_path.unlink(missing_ok=True) + + os.chmod(output_path, 0o600) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Create a local Vectorless RAG environment without overwriting an existing file." + ) + parser.add_argument("--template", type=Path, default=Path(".env.example")) + parser.add_argument("--output", type=Path, default=Path(".env")) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + try: + create_atomically(args.template, args.output) + except (FileExistsError, OSError, ValueError) as error: + print(error, file=sys.stderr) + return 2 + print(f"created {args.output} with mode 600; add DEEPSEEK_API_KEY before provider-backed use") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test-integration-stack.sh b/scripts/test-integration-stack.sh new file mode 100755 index 0000000..3b47418 --- /dev/null +++ b/scripts/test-integration-stack.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail + +root_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +docker_bin="${DOCKER:-docker}" +uv_bin="${UV:-uv}" +make_bin="${MAKE:-make}" +postgres_port="${TEST_POSTGRES_PORT:-55432}" +project_name="vectorless-rag-integration-$PPID-$$-$RANDOM" +compose_args=( + compose + --project-name "$project_name" + --env-file "$root_dir/.env.example" + -f "$root_dir/compose.yaml" +) + +if [[ ! "$postgres_port" =~ ^[0-9]+$ ]] || ((postgres_port < 1 || postgres_port > 65535)); then + echo "TEST_POSTGRES_PORT must be an integer from 1 through 65535" >&2 + exit 2 +fi + +cleanup() { + POSTGRES_PORT="$postgres_port" "$docker_bin" "${compose_args[@]}" \ + down --volumes --remove-orphans >/dev/null 2>&1 || true +} +trap cleanup EXIT INT TERM + +echo "starting disposable PostgreSQL project on port $postgres_port" +POSTGRES_PORT="$postgres_port" "$docker_bin" "${compose_args[@]}" up -d --wait postgres + +owner_url="postgresql+psycopg://rag_owner:rag-owner-change-me@127.0.0.1:$postgres_port/rag" +reader_url="postgresql+psycopg://rag_reader:rag-reader-change-me@127.0.0.1:$postgres_port/rag" + +( + cd "$root_dir/apps/api" + DATABASE_ADMIN_URL="$owner_url" "$uv_bin" run vectorless-rag migrate +) +( + cd "$root_dir" + TEST_OWNER_DATABASE_URL="$owner_url" TEST_DATABASE_URL="$reader_url" \ + "$make_bin" test-integration +)