V3

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,24 @@
+## Summary
+
+<!-- What does this PR do? Link any related issues: "Closes #123" -->
+
+## Type of change
+
+- [ ] Bug fix
+- [ ] New feature (reader / embedder / generator / chunker)
+- [ ] Documentation update
+- [ ] Refactor / cleanup
+- [ ] Other
+
+## Testing
+
+<!-- How did you test this? -->
+
+- [ ] Tested locally against a running Verba instance
+- [ ] Added / updated tests in `goldenverba/tests/`
+
+## Checklist
+
+- [ ] My code follows the existing patterns in the codebase
+- [ ] I have added environment variable docs to `.env.example` if applicable
+- [ ] I have updated `CHANGELOG.md` under the relevant version section

.github/dependabot.yml

@@ -5,26 +5,39 @@ updates:
     schedule:
       interval: weekly
       day: monday
-    open-pull-requests-limit: 10
+    open-pull-requests-limit: 3
     labels:
       - dependencies
       - python
+    groups:
+      python-deps:
+        patterns:
+          - "*"
 
   - package-ecosystem: npm
     directory: "/frontend"
     schedule:
       interval: weekly
       day: monday
-    open-pull-requests-limit: 10
+    open-pull-requests-limit: 3
     labels:
       - dependencies
       - javascript
+    groups:
+      npm-deps:
+        patterns:
+          - "*"
 
   - package-ecosystem: github-actions
     directory: "/"
     schedule:
       interval: weekly
       day: monday
+    open-pull-requests-limit: 1
     labels:
       - dependencies
       - github-actions
+    groups:
+      github-actions:
+        patterns:
+          - "*"

.github/workflows/ci.yml

@@ -0,0 +1,58 @@
+name: CI
+
+on:
+  push:
+    branches: ["main", "v3"]
+  pull_request:
+    branches: ["main", "v3"]
+
+jobs:
+  backend-test:
+    name: Backend (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12"]
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install ruff
+          pip install '.[dev]'
+
+      - name: Lint with ruff
+        run: ruff check goldenverba/
+
+      - name: Run tests
+        run: pytest goldenverba/tests/ -v --tb=short
+
+  frontend-lint:
+    name: Frontend (Node 20)
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "20"
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+
+      - name: Install dependencies
+        working-directory: frontend
+        run: npm ci
+
+      - name: Lint
+        working-directory: frontend
+        run: npm run lint

.github/workflows/docker-image.yml

@@ -22,10 +22,12 @@ jobs:
           password: ${{secrets.DOCKER_PASSWORD}}
 
       - name: Build and push
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v6
         with:
           context: .
           file: ./Dockerfile
           push: true
           tags: semitechnologies/verba:latest
           platforms: linux/amd64,linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.github/workflows/release.yml

@@ -0,0 +1,67 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+jobs:
+  pypi-publish:
+    name: Publish to PyPI
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+      url: https://pypi.org/project/goldenverba/
+    permissions:
+      id-token: write  # Required for trusted publishing
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install build tools
+        run: pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+
+  docker-push:
+    name: Build and push Docker image
+    runs-on: ubuntu-latest
+    needs: pypi-publish
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Extract version tag
+        id: meta
+        run: echo "tag=${GITHUB_REF_NAME}" >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push versioned + latest tags
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: ./Dockerfile
+          push: true
+          tags: |
+            semitechnologies/verba:${{ steps.meta.outputs.tag }}
+            semitechnologies/verba:latest
+          platforms: linux/amd64,linux/arm64
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

.gitignore

@@ -1,24 +1,39 @@
+# Environment
 .env
 .env*
+
+# Python
 __pycache__
-.DS_Store
-.pytest_cache
 .python-version
 *.egg-info
+.ruff_cache
+.pytest_cache
+
+# Virtual environments
 venv
 .venv
 venv*
+
+# Build artifacts
 dist
 build
-~
+
+# Local data and config
+data/
+verba_config.json
+*_secrets.json
+cache.txt
+
+# OS
+.DS_Store
+
+# Tools
 .local
 .cache
 .verba
 .vscode
-verba_config.json
+ollama
 text-generation-inference
 test.py
-cache.txt
-.ruff_cache
-*_secrets.json
-ollama
+
+todo.md

.pre-commit-config.yaml

@@ -0,0 +1,27 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v5.0.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-json
+      - id: check-merge-conflict
+      - id: check-added-large-files
+        args: ["--maxkb=1000"]
+      - id: detect-private-key
+
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.11.6
+    hooks:
+      - id: ruff
+        args: ["--fix"]
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v4.0.0-alpha.8
+    hooks:
+      - id: prettier
+        types_or: [javascript, jsx, ts, tsx, json, css, markdown]
+        files: ^frontend/
+        exclude: frontend/node_modules/

CHANGELOG.md

@@ -2,6 +2,64 @@
 
 All notable changes to this project will be documented in this file.
 
+## [3.0.0] - Unreleased
+
+## Added
+
+- **WhisperReader** — local audio/video transcription via `faster-whisper`. Replaces the AssemblyAI reader with a zero-cost, offline alternative. Supports 35+ formats (`.mp3`, `.wav`, `.m4a`, `.flac`, `.ogg`, `.mp4`, `.mov`, `.webm`, …). No API key required; model is downloaded automatically on first use. Configurable model size (`tiny` → `large-v3`) and compute device (`cpu` / `cuda` / `auto`).
+
+## Removed
+
+- **FirecrawlReader** — removed due to high maintenance surface (async job polling, versioned API endpoints, paid usage-based pricing). HTMLReader covers the primary use case (static site scraping). Users needing JavaScript rendering should run a local Playwright/Puppeteer setup.
+- **UpstageDocumentParse reader** — removed as redundant with UnstructuredAPI for complex PDF parsing. Upstage embedding and generation models are unaffected and remain available.
+- **AssemblyAI reader** — replaced by the new local WhisperReader. Removes a paid-per-minute external dependency while providing the same audio/video transcription capability.
+
+## Added
+
+- **DeepSeek generator** with reasoning model (R1) support — supports `deepseek-chat` and `deepseek-reasoner` via DeepSeek's OpenAI-compatible API. R1 thinking process shown in a collapsible section with a "Show Reasoning" toggle. Dynamic model discovery at startup. Env vars: `DEEPSEEK_API_KEY`, `DEEPSEEK_BASE_URL`, `DEEPSEEK_MODEL` (https://github.com/weaviate/Verba/pull/395)
+- **LM Studio integration** — `LMStudioEmbedder` and `LMStudioGenerator` for running fully local models via LM Studio's OpenAI-compatible API (`http://localhost:1234/v1` by default). No API key required. Env vars: `LMSTUDIO_BASE_URL`, `LMSTUDIO_API_KEY`, `LMSTUDIO_MODEL`, `LMSTUDIO_EMBEDDER_MODEL` (https://github.com/weaviate/Verba/pull/391)
+- **Test suite** — 71 tests covering `BatchManager` (including TTL eviction), `LoggerManager`, all three chunkers (`TokenChunker`, `SentenceChunker`, `MarkdownChunker`), and the `Document` class
+- **BACKEND.md** — developer guide covering architecture, component plugin system, step-by-step guides for adding new Generators/Embedders/Readers/Chunkers, WebSocket protocol, config system, and local dev setup
+
+## Fixed
+
+- Chunk deserialization from JSON: `doc_uuid` was stored as a tuple due to a stray trailing comma; `title`, `labels`, and `pca` were silently dropped on round-trip. Includes a full serialization round-trip test. (https://github.com/weaviate/Verba/pull/398)
+- CORS misconfiguration: `allow_credentials=True` with `allow_origins=["*"]` is rejected by browsers per the CORS spec. Access control is enforced by the existing custom same-origin middleware. (https://github.com/weaviate/Verba/issues/393)
+- **Mutable default argument** in `import_document()` — `LoggerManager()` was instantiated once at definition time and shared across concurrent imports, causing them to clobber each other's WebSocket reference
+- **Lock creation race** in `ClientManager.get_or_create_lock()` — two coroutines could both see an absent key and create duplicate locks; fixed with `setdefault()`
+- **Dict mutation during iteration** in `ClientManager.clean_up()` and `disconnect()` — iterating over a live dict while deleting entries raises `RuntimeError`; fixed by iterating over a snapshot
+- **Exception object sent to WebSocket** — `send_json({"message": e})` where `e` is an `Exception` object fails JSON serialization; changed to `str(e)`
+- **`msg.good()` fired after exceptions** — success log was outside the `try` block and ran unconditionally even when the streaming failed
+- **`asyncio.create_task()` immediately awaited** — pattern `task = create_task(fn()); result = await task` is equivalent to `await fn()` but with extra overhead; simplified to direct `await`
+- **O(n²) string concatenation** in streaming loop — `full_text += chunk["message"]` in a hot loop replaced with list accumulation and `"".join()` at the end
+- **`SentenceTransformer` model reloaded on every call** — `SentenceTransformer(model_name)` was called inside `vectorize()`, loading ~300 MB from disk on every embedding request; model is now cached by name in `_model_cache`
+- **`model.encode()` blocking the event loop** — synchronous CPU-bound call now wrapped with `asyncio.to_thread()`
+- **Streaming timeout `None`** on all generators — a hung upstream API would hold a WebSocket connection open forever; all generators now use `httpx.Timeout(connect=10, read=300)` or `aiohttp.ClientTimeout(connect=10, total=300)`
+- **Debug `print()` statements** in `util.py` `pca()` — four statements printing raw matrices to stdout in production removed
+- **Bare `except:` clauses** — `except:` in `GeminiGenerator.py` (import guard) and `document.py` (language detection) changed to `except ImportError` and `except Exception` respectively
+- **Deprecated `asyncio==3.4.3`** in `setup.py` — the PyPI `asyncio` package conflicts with the stdlib module present since Python 3.4; entry removed
+
+## Changed
+
+- **Payload size limits** — `GeneratePayload` and `QueryPayload` now enforce Pydantic `max_length` validators: query ≤ 50,000 chars, context ≤ 500,000 chars, conversation ≤ 100 items
+- **`BatchManager` TTL eviction** — abandoned incomplete uploads are now evicted after 5 minutes (previously leaked memory indefinitely)
+- Automated PyPI publishing via GitHub Actions on `v*.*.*` tag push using trusted publishing — no stored API token needed (replaces manual `pypi_commands.sh`)
+- Docker image now also tagged with version (e.g. `semitechnologies/verba:v3.0.0`) in addition to `:latest`
+- Upgraded Docker GitHub Actions to `build-push-action@v6` with GHA build cache for faster builds
+- `httpx` now explicitly declared in `setup.py` (was already used by multiple generators but only present as a transitive dependency)
+
+## Infrastructure
+
+- CI workflow (`ci.yml`): runs pytest on Python 3.11/3.12, ruff linting, and ESLint on every PR targeting `main` or `v3`
+- `ruff.toml`: Python linting and formatting config (replaces Black)
+- `.pre-commit-config.yaml`: ruff, ruff-format, prettier for frontend, and file hygiene hooks
+- Dependabot: automated weekly dependency updates for pip, npm, and GitHub Actions (grouped into one PR per ecosystem)
+- `SECURITY.md`: responsible disclosure policy via GitHub private vulnerability reporting
+- `CODE_OF_CONDUCT.md`: Contributor Covenant 2.1
+- `.github/PULL_REQUEST_TEMPLATE.md`: PR checklist for contributors
+
+---
+
 ## [2.1.3] More data types
 
 ## Added

CODE_OF_CONDUCT.md

@@ -0,0 +1,41 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our community a harassment-free experience for everyone, regardless of age, body size, visible or invisible disability, ethnicity, sex characteristics, gender identity and expression, level of experience, education, socio-economic status, nationality, personal appearance, race, caste, color, religion, or sexual identity and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming, diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes
+* Focusing on what is best not just for us as individuals, but for the overall community
+
+Examples of unacceptable behavior:
+
+* The use of sexualized language or imagery, and sexual attention or advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a professional setting
+
+## Enforcement Responsibilities
+
+Project maintainers are responsible for clarifying and enforcing our standards of acceptable behavior and will take appropriate and fair corrective action in response to any behavior that they deem inappropriate, threatening, offensive, or harmful.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when an individual is officially representing the community in public spaces.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the project maintainer at edward@weaviate.io. All complaints will be reviewed and investigated promptly and fairly.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org), version 2.1.