From f1c4126270fc5650374936bc4660e3a9dc5bbc67 Mon Sep 17 00:00:00 2001 From: Damir Dulic Date: Tue, 17 Mar 2026 16:42:17 +0000 Subject: [PATCH 1/3] feat: add UI-managed prompt configuration (004-ui-prompt-config) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace hardcoded server-side prompt files with a database-backed prompt configuration system fully manageable from the web UI. Key changes: - New PromptConfigService backed by SQLAlchemy model + Alembic migration - REST endpoints: GET/PUT/DELETE /api/extended/prompts, POST /api/extended/reprocess-all - PromptLoader refactored to resolve prompts from DB with hardcoded constants as fallback - ProcessorService gains reprocess_all() and per-page prompt-hash staleness tracking - New PromptsModal.js: tabbed OCR/Summary editor with custom type support, inline cost-warning confirmation before "Reprocess All Notes" - FileViewer.js: per-page stale badge + individual reprocess button; amber header icon when note has stale pages - Removed 11 hardcoded prompt .md files from supernote/server/resources/prompts/ - §VIII Frontend UI Conventions added to project constitution (v1.2.0); all button styles audited and normalised across ApiKeysPanel, MoveModal, RenameModal, FileViewer, and index.html to match the canonical SystemPanel reference palette - Full test coverage: route security, service unit, processor hash, model completeness --- .specify/memory/constitution.md | 55 ++- CLAUDE.md | 25 +- .../checklists/requirements.md | 35 ++ .../contracts/prompts-api.md | 221 +++++++++ specs/004-ui-prompt-config/data-model.md | 145 ++++++ specs/004-ui-prompt-config/plan.md | 277 ++++++++++++ specs/004-ui-prompt-config/quickstart.md | 66 +++ specs/004-ui-prompt-config/research.md | 83 ++++ specs/004-ui-prompt-config/spec.md | 216 +++++++++ specs/004-ui-prompt-config/tasks.md | 223 +++++++++ .../b2c3d4e5f6a7_add_prompt_config.py | 46 ++ supernote/models/prompt_config.py | 117 +++++ supernote/server/app.py | 10 +- supernote/server/db/models/__init__.py | 2 + supernote/server/db/models/note_processing.py | 3 + supernote/server/db/models/prompt_config.py | 44 ++ .../resources/prompts/ocr/common/context.md | 19 - .../resources/prompts/ocr/common/legend.md | 11 - .../resources/prompts/ocr/daily/prompt.md | 12 - .../resources/prompts/ocr/default/system.md | 14 - .../resources/prompts/ocr/monthly/prompt.md | 22 - .../resources/prompts/ocr/weekly/prompt.md | 16 - .../prompts/summary/common/instruction.md | 8 - .../resources/prompts/summary/daily/prompt.md | 6 - .../prompts/summary/default/prompt.md | 4 - .../prompts/summary/monthly/prompt.md | 6 - .../prompts/summary/weekly/prompt.md | 6 - supernote/server/routes/prompts.py | 423 ++++++++++++++++++ supernote/server/services/processor.py | 128 +++++- .../server/services/processor_modules/ocr.py | 25 +- .../services/processor_modules/summary.py | 17 +- supernote/server/services/prompt_config.py | 284 ++++++++++++ supernote/server/static/index.html | 23 +- supernote/server/static/js/api/client.js | 144 ++++++ .../static/js/components/ApiKeysPanel.js | 14 +- .../server/static/js/components/FileViewer.js | 111 ++++- .../server/static/js/components/MoveModal.js | 4 +- .../static/js/components/PromptsModal.js | 268 +++++++++++ .../static/js/components/RenameModal.js | 4 +- supernote/server/static/js/main.js | 4 + supernote/server/utils/prompt_loader.py | 156 +++---- .../models/test_prompt_config_completeness.py | 142 ++++++ tests/server/routes/test_prompts.py | 385 ++++++++++++++++ .../server/services/test_processor_modules.py | 49 +- .../services/test_processor_prompt_hash.py | 313 +++++++++++++ .../services/test_prompt_config_service.py | 324 ++++++++++++++ tests/server/services/test_prompt_loader.py | 93 ++-- uv.lock | 2 +- 48 files changed, 4270 insertions(+), 335 deletions(-) create mode 100644 specs/004-ui-prompt-config/checklists/requirements.md create mode 100644 specs/004-ui-prompt-config/contracts/prompts-api.md create mode 100644 specs/004-ui-prompt-config/data-model.md create mode 100644 specs/004-ui-prompt-config/plan.md create mode 100644 specs/004-ui-prompt-config/quickstart.md create mode 100644 specs/004-ui-prompt-config/research.md create mode 100644 specs/004-ui-prompt-config/spec.md create mode 100644 specs/004-ui-prompt-config/tasks.md create mode 100644 supernote/alembic/versions/b2c3d4e5f6a7_add_prompt_config.py create mode 100644 supernote/models/prompt_config.py create mode 100644 supernote/server/db/models/prompt_config.py delete mode 100644 supernote/server/resources/prompts/ocr/common/context.md delete mode 100644 supernote/server/resources/prompts/ocr/common/legend.md delete mode 100644 supernote/server/resources/prompts/ocr/daily/prompt.md delete mode 100644 supernote/server/resources/prompts/ocr/default/system.md delete mode 100644 supernote/server/resources/prompts/ocr/monthly/prompt.md delete mode 100644 supernote/server/resources/prompts/ocr/weekly/prompt.md delete mode 100644 supernote/server/resources/prompts/summary/common/instruction.md delete mode 100644 supernote/server/resources/prompts/summary/daily/prompt.md delete mode 100644 supernote/server/resources/prompts/summary/default/prompt.md delete mode 100644 supernote/server/resources/prompts/summary/monthly/prompt.md delete mode 100644 supernote/server/resources/prompts/summary/weekly/prompt.md create mode 100644 supernote/server/routes/prompts.py create mode 100644 supernote/server/services/prompt_config.py create mode 100644 supernote/server/static/js/components/PromptsModal.js create mode 100644 tests/models/test_prompt_config_completeness.py create mode 100644 tests/server/routes/test_prompts.py create mode 100644 tests/server/services/test_processor_prompt_hash.py create mode 100644 tests/server/services/test_prompt_config_service.py diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md index de3a9d77..ade7b731 100644 --- a/.specify/memory/constitution.md +++ b/.specify/memory/constitution.md @@ -1,19 +1,14 @@ @@ -155,6 +150,46 @@ security-focused tests (see `tests/server/routes/test_*_security*.py` and and AI-derived insights. A single authorization bypass or credential leak would expose highly sensitive personal data. Security must be designed in, not bolted on. +### VIII. Frontend UI Conventions + +The frontend is Vanilla JS with Vue 3 (ESM browser build, no build step). All +components live in `supernote/server/static/js/components/` and are served as +static files. UI changes MUST follow the established visual language exactly +so the interface remains consistent across features. + +**Button styles** (Tailwind CSS — MUST be used verbatim for each category): + +| Category | Required classes | +|---|---| +| Primary (save / confirm) | `px-4 py-2 bg-black border border-black rounded text-sm font-medium text-white hover:bg-gray-800 disabled:opacity-50 transition-colors` | +| Secondary / cancel | `px-4 py-2 bg-white dark:bg-gray-600 border border-gray-300 dark:border-gray-500 rounded text-sm font-medium text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-500 disabled:opacity-50 transition-colors` | +| Danger (delete / remove) | `px-3 py-1.5 text-xs font-medium text-red-600 dark:text-red-400 border border-red-200 dark:border-red-700 bg-red-50 dark:bg-red-900/20 hover:bg-red-100 dark:hover:bg-red-900/40 rounded transition-colors disabled:opacity-50` | +| Icon-only (header / toolbar) | `text-gray-400 hover:text-black dark:hover:text-white transition-colors` | +| Amber / warning | `p-2 text-amber-600 dark:text-amber-400 hover:bg-amber-50 dark:hover:bg-amber-900/30 disabled:opacity-50 rounded transition-colors border border-amber-300 dark:border-amber-600` — reserved for status-driven indicators only (e.g. stale content), never for routine actions | + +**Prohibited**: Framework accent colors (indigo, blue, green, etc.) MUST NOT +be used for interactive controls. The permitted palette for interactive elements +is black / gray / red / amber only, as defined above. + +**Disabled state**: Always `disabled:opacity-50`; add `disabled:cursor-not-allowed` +only when a full-width button is used (e.g. form submission). + +**Modal overlay pattern**: `fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center z-[N] p-4` with `@click.self="$emit('close')"`. Modal close (×) buttons MUST use the icon-only style above. + +**Focus rings**: `focus:ring-2 focus:ring-black dark:focus:ring-white` — color-specific rings (indigo, blue, etc.) are prohibited. + +**Loading spinners**: `animate-spin rounded-full h-8 w-8 border-b-2 border-black dark:border-white` (full-size) or `animate-spin w-4 h-4` with an SVG spinner inline (compact, inside a button). + +**Dark mode**: Every interactive element MUST declare a `dark:` variant for all +background, text, and border properties. An element without a dark mode class +MUST NOT be merged. + +**Rationale**: A consistent visual language makes the application feel coherent +and reduces cognitive overhead. Because there is no design system or component +library, the Tailwind class strings above serve as the single source of truth. +Deviating without updating this constitution creates drift that compounds +across features. + ## Technology Stack - **Runtime**: Python 3.13+, managed with `uv` @@ -215,4 +250,4 @@ all seven Core Principles. Complexity that violates a principle MUST be explicitly justified in the PR description with reference to the specific principle and why the simpler alternative was rejected. -**Version**: 1.1.0 | **Ratified**: 2026-03-15 | **Last Amended**: 2026-03-15 +**Version**: 1.2.0 | **Ratified**: 2026-03-15 | **Last Amended**: 2026-03-17 diff --git a/CLAUDE.md b/CLAUDE.md index 047711c2..20afc6bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,9 +1,12 @@ # supernote Development Guidelines -Auto-generated from all feature plans. Last updated: 2026-03-16 +Auto-generated from all feature plans. Last updated: 2026-03-17 ## Active Technologies - N/A (no Python source changes) + GitHub Dependabot (native GitHub feature, no external service) (002-switch-dependabot) +- N/A (no Python source changes — CI/CD configuration only) + GitHub Actions (`docker/metadata-action`, `docker/build-push-action`, `docker/login-action`) (003-github-releases) +- Python 3.13+ + aiohttp (server), SQLAlchemy asyncio + aiosqlite, mashumaro, alembic; Vanilla JS (Vue 3, no build step) for frontend (004-ui-prompt-config) +- SQLite via SQLAlchemy asyncio — new `f_prompt_config` table; new `prompt_hash` column on `f_note_page_content` (004-ui-prompt-config) - Python 3.13+ + mypy (strict), SQLAlchemy asyncio, aiohttp, mashumaro, pytest + pytest-asyncio (001-constitution-alignment) @@ -43,10 +46,28 @@ supernote serve --ephemeral # Ephemeral server with debug@example.com / password - **Testing**: `unittest.mock.patch` only (no `monkeypatch`); all test functions/fixtures must have type annotations; tests written before implementation - **Logging**: `logging.getLogger(__name__)`; NEVER log note content +## Frontend UI Conventions (constitution §VIII) + +Button Tailwind classes — use verbatim: + +| Category | Classes | +|---|---| +| Primary (save/confirm) | `px-4 py-2 bg-black border border-black rounded text-sm font-medium text-white hover:bg-gray-800 disabled:opacity-50 transition-colors` | +| Secondary/cancel | `px-4 py-2 bg-white dark:bg-gray-600 border border-gray-300 dark:border-gray-500 rounded text-sm font-medium text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-500 transition-colors` | +| Danger (delete/remove) | `px-3 py-1.5 text-xs font-medium text-red-600 dark:text-red-400 border border-red-200 dark:border-red-700 bg-red-50 dark:bg-red-900/20 hover:bg-red-100 dark:hover:bg-red-900/40 rounded transition-colors disabled:opacity-50` | +| Icon-only (header) | `text-gray-400 hover:text-black dark:hover:text-white transition-colors` | +| Amber/warning | `p-2 text-amber-600 dark:text-amber-400 hover:bg-amber-50 dark:hover:bg-amber-900/30 disabled:opacity-50 rounded transition-colors border border-amber-300 dark:border-amber-600` — status indicators only | + +- **No accent colors**: indigo, blue, green etc. are prohibited for interactive controls +- **Focus rings**: `focus:ring-2 focus:ring-black dark:focus:ring-white` +- **Spinners**: `animate-spin rounded-full h-8 w-8 border-b-2 border-black dark:border-white` +- **Dark mode**: every interactive element must have `dark:` variants + ## Recent Changes +- 004-ui-prompt-config: Added Python 3.13+ + aiohttp (server), SQLAlchemy asyncio + aiosqlite, mashumaro, alembic; Vanilla JS (Vue 3, no build step) for frontend +- 003-github-releases: Added N/A (no Python source changes — CI/CD configuration only) + GitHub Actions (`docker/metadata-action`, `docker/build-push-action`, `docker/login-action`) - 002-switch-dependabot: Added N/A (no Python source changes) + GitHub Dependabot (native GitHub feature, no external service) -- 001-constitution-alignment: Added Python 3.13+ + mypy (strict), SQLAlchemy asyncio, aiohttp, mashumaro, pytest + pytest-asyncio diff --git a/specs/004-ui-prompt-config/checklists/requirements.md b/specs/004-ui-prompt-config/checklists/requirements.md new file mode 100644 index 00000000..2c52bd41 --- /dev/null +++ b/specs/004-ui-prompt-config/checklists/requirements.md @@ -0,0 +1,35 @@ +# Specification Quality Checklist: UI Prompt Configuration + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-03-17 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- All items pass. Spec updated to include prompt hash tracking, stale detection, and manual Reprocess button (FR-014–FR-020, User Story 5, SC-006–SC-007). +- Ready for `/speckit.clarify` or `/speckit.plan`. diff --git a/specs/004-ui-prompt-config/contracts/prompts-api.md b/specs/004-ui-prompt-config/contracts/prompts-api.md new file mode 100644 index 00000000..99260e37 --- /dev/null +++ b/specs/004-ui-prompt-config/contracts/prompts-api.md @@ -0,0 +1,221 @@ +# API Contracts: Prompt Configuration + +All endpoints are under the `extended` route module (`/api/extended/`). All require a valid `x-access-token` JWT header. Users may only access their own prompt configurations; cross-user access returns 403. + +--- + +## Prompt Configuration Endpoints + +### GET `/api/extended/prompts` + +Returns all effective prompt configurations for the authenticated user. The response includes every known `(category, layer)` combination — built-in layers from the server files plus any user-defined custom layers — merged with the user's saved overrides. + +**Auth**: JWT required + +**Response 200**: +```json +{ + "success": true, + "prompts": [ + { + "category": "ocr", + "layer": "common", + "content": "...current effective text...", + "isOverride": true, + "defaultContent": "...server file text..." + }, + { + "category": "ocr", + "layer": "default", + "content": "...server file text...", + "isOverride": false, + "defaultContent": "...server file text..." + }, + { + "category": "ocr", + "layer": "daily", + "content": "...server file text...", + "isOverride": false, + "defaultContent": "...server file text..." + }, + { + "category": "summary", + "layer": "common", + "content": "...current effective text...", + "isOverride": false, + "defaultContent": "...server file text..." + } + ] +} +``` + +**Notes**: +- Built-in layers always appear (common, default, daily, weekly, monthly) for both categories +- User-defined custom layers also appear when saved +- `isOverride: true` means a `f_prompt_config` row exists for this user + category + layer +- `defaultContent` always contains the server file text for Reset support + +--- + +### PUT `/api/extended/prompts` + +Save or update a single prompt configuration for the authenticated user. Creates a new `f_prompt_config` row or updates the existing one (upsert on `(user_id, category, layer)`). + +**Auth**: JWT required + +**Request body**: +```json +{ + "category": "summary", + "layer": "monthly", + "content": "This is a Monthly Log for bullet journaling. Summarise by week..." +} +``` + +**Validation**: +- `category`: required, must be `"ocr"` or `"summary"` +- `layer`: required, 1–64 characters, alphanumeric + hyphens only +- `content`: required, must not be empty or whitespace-only + +**Response 200**: +```json +{ + "success": true +} +``` + +**Response 400** (validation failure): +```json +{ + "success": false, + "errorCode": "INVALID_INPUT", + "errorMsg": "content must not be empty" +} +``` + +--- + +### DELETE `/api/extended/prompts/{category}/{layer}` + +Remove the user's saved override for a specific `(category, layer)`, reverting it to the server default. For user-defined custom layers, this fully removes the type. For built-in layers, this just removes the override row. + +**Auth**: JWT required + +**Path parameters**: +- `category`: `"ocr"` or `"summary"` +- `layer`: layer name + +**Response 200**: +```json +{ + "success": true +} +``` + +**Response 404** (no override exists to delete): +```json +{ + "success": false, + "errorCode": "NOT_FOUND", + "errorMsg": "No override found for ocr/monthly" +} +``` + +--- + +## Staleness & Reprocess Endpoints + +### GET `/api/extended/files/{file_id}/staleness` + +Computes the current effective prompt hash for this file's note type and compares it against the stored `prompt_hash` on each page. Returns per-page staleness status. + +**Auth**: JWT required. User must own the file (403 otherwise). + +**Response 200**: +```json +{ + "success": true, + "currentPromptHash": "a3f1e9c2...", + "staleCount": 2, + "totalCount": 12, + "pages": [ + { + "pageId": "P20231027120000abc", + "pageIndex": 0, + "storedHash": "a3f1e9c2...", + "isStale": false + }, + { + "pageId": "P20231028090000xyz", + "pageIndex": 1, + "storedHash": null, + "isStale": true + } + ] +} +``` + +**Notes**: +- `storedHash: null` means the page was processed before this feature; treated as stale +- If the file has no processed pages yet, returns `staleCount: 0, totalCount: 0` + +--- + +### POST `/api/extended/files/{file_id}/reprocess` + +Queues stale pages of a note for reprocessing. Resets `SystemTaskDO` status for `OCR_EXTRACTION`, `EMBEDDING_GENERATION` (per page) and `SUMMARY_GENERATION` (global) for stale pages only, then enqueues the file. + +**Auth**: JWT required. User must own the file (403 otherwise). + +**Request body** (optional): +```json +{ + "pageIds": ["P20231028090000xyz"] +} +``` +If `pageIds` is omitted or null, all stale pages for the file are queued. + +**Validation**: +- If `pageIds` is provided, only pages with `is_stale: true` are accepted; non-stale page IDs in the list are silently skipped (consistent with spec FR-018). + +**Response 200**: +```json +{ + "success": true, + "queuedPageCount": 1 +} +``` + +**Response 409** (processing already in progress for this file): +```json +{ + "success": false, + "errorCode": "ALREADY_PROCESSING", + "errorMsg": "This file is already queued for processing" +} +``` + +--- + +### POST `/api/extended/files/{file_id}/pages/{page_id}/reprocess` + +Queues a single page for reprocessing. Resets task status for OCR and embedding for the specified page, and resets the summary task at file level. + +**Auth**: JWT required. User must own the file (403 otherwise). + +**Response 200**: +```json +{ + "success": true, + "queuedPageCount": 1 +} +``` + +**Response 400** (page is not stale): +```json +{ + "success": false, + "errorCode": "NOT_STALE", + "errorMsg": "This page does not require reprocessing" +} +``` diff --git a/specs/004-ui-prompt-config/data-model.md b/specs/004-ui-prompt-config/data-model.md new file mode 100644 index 00000000..35c66f06 --- /dev/null +++ b/specs/004-ui-prompt-config/data-model.md @@ -0,0 +1,145 @@ +# Data Model: UI Prompt Configuration + +## New Table: `f_prompt_config` + +Stores per-user prompt overrides. A row only exists when the user has explicitly saved a customisation for that `(category, layer)` combination. + +### `PromptConfigDO` + +| Field | Type | Nullable | Notes | +|-------|------|----------|-------| +| `id` | BigInteger PK | No | `next_id()` default | +| `user_id` | BigInteger | No | Indexed. FK to `f_user.id` (not enforced at DB level, consistent with project pattern) | +| `category` | String | No | `"ocr"` or `"summary"` | +| `layer` | String | No | `"common"`, `"default"`, or any user-defined type name (e.g. `"daily"`, `"project"`) | +| `content` | Text | No | Full prompt text for this layer | +| `create_time` | BigInteger | No | Epoch milliseconds | +| `update_time` | BigInteger | No | Epoch milliseconds, auto-updated on write | + +**Unique constraint**: `uq_prompt_config` on `(user_id, category, layer)` + +**Validation rules**: +- `content` MUST NOT be empty +- `category` MUST be one of `["ocr", "summary"]` +- `layer` MUST NOT be empty; max 64 characters; alphanumeric + hyphens only +- `layer` names `"common"` and `"default"` are valid layer values for built-in overrides + +--- + +## Modified Table: `f_note_page_content` + +Add one nullable column to the existing `NotePageContentDO`. + +### New column: `prompt_hash` + +| Field | Type | Nullable | Notes | +|-------|------|----------|-------| +| `prompt_hash` | String | Yes | SHA-256 hex digest of the full composed OCR prompt + `"\|"` + full composed summary prompt used during the last successful processing run. `NULL` means the page was processed before this feature was deployed (treated as stale). | + +--- + +## State Transitions: `PromptConfigDO` + +``` +[No row] ---(user saves override)---> [row exists: content = user text] +[row exists] ---(user resets to default)---> [No row] (row deleted) +[row exists] ---(user edits)---> [row exists: content = updated text] +[No row] ---(user creates custom type)---> [row exists: layer = custom name] +[row exists: custom type] ---(user deletes type)---> [No row] +``` + +--- + +## State Transitions: `NotePageContentDO.prompt_hash` + +``` +NULL (not yet processed or pre-feature) + ---(successful OCR + summary processing run)---> hash_v1 (sha256 of composed prompts at time of processing) + +hash_v1 (user changes a prompt, lazy check detects mismatch) + ---> page marked stale in UI (no DB write) + ---(user triggers reprocess, processing completes)---> hash_v2 (updated sha256) +``` + +--- + +## DTO Definitions + +### `PromptConfigDTO` (API response item) + +```python +@dataclass +class PromptConfigDTO(DataClassJSONMixin): + category: str # "ocr" | "summary" + layer: str # "common" | "default" | custom + content: str # effective text (user override if present, else server default) + is_override: bool # True if a user-saved row exists for this (category, layer) + default_content: str # server-file default text (always present for reset support) +``` + +### `UpsertPromptConfigDTO` (request body for save/update) + +```python +@dataclass +class UpsertPromptConfigDTO(DataClassJSONMixin): + category: str + layer: str + content: str +``` + +### `GetPromptsResponseVO` (GET /api/extended/prompts response) + +```python +@dataclass +class GetPromptsResponseVO(BaseResponse): + prompts: list[PromptConfigDTO] = field(default_factory=list) +``` + +### `PageStalenessDTO` (per-page staleness info) + +```python +@dataclass +class PageStalenessDTO(DataClassJSONMixin): + page_id: str + page_index: int + stored_hash: str | None # None = pre-feature, treated as stale + is_stale: bool +``` + +### `FileStalenessResponseVO` (GET /api/extended/files/{id}/staleness) + +```python +@dataclass +class FileStalenessResponseVO(BaseResponse): + current_prompt_hash: str + pages: list[PageStalenessDTO] = field(default_factory=list) + stale_count: int = 0 + total_count: int = 0 +``` + +### `ReprocessRequestDTO` (POST /api/extended/files/{id}/reprocess body, optional) + +```python +@dataclass +class ReprocessRequestDTO(DataClassJSONMixin): + page_ids: list[str] | None = None # None = all stale pages +``` + +### `ReprocessResponseVO` + +```python +@dataclass +class ReprocessResponseVO(BaseResponse): + queued_page_count: int = 0 +``` + +--- + +## Alembic Migration + +Single migration covers both changes: + +1. **Create** `f_prompt_config` with all columns and unique constraint +2. **Add column** `prompt_hash` (String, nullable=True, no server_default) to `f_note_page_content` + +Downgrade: drop `prompt_hash` column, drop `f_prompt_config` table. diff --git a/specs/004-ui-prompt-config/plan.md b/specs/004-ui-prompt-config/plan.md new file mode 100644 index 00000000..446cf0c0 --- /dev/null +++ b/specs/004-ui-prompt-config/plan.md @@ -0,0 +1,277 @@ +# Implementation Plan: UI Prompt Configuration + +**Branch**: `004-ui-prompt-config` | **Date**: 2026-03-17 | **Spec**: [spec.md](spec.md) + +## Summary + +Move AI prompt configuration from hardcoded server-side `.md` files to a per-user database-backed store, surfaced through a Prompts modal in the UI header. Record a combined prompt hash on each processed page so the viewer can detect stale processing results and offer a targeted manual Reprocess button. All prompt changes take effect on the next processing run; no automatic reprocessing occurs. + +## Technical Context + +**Language/Version**: Python 3.13+ +**Primary Dependencies**: aiohttp (server), SQLAlchemy asyncio + aiosqlite, mashumaro, alembic; Vanilla JS (Vue 3, no build step) for frontend +**Storage**: SQLite via SQLAlchemy asyncio — new `f_prompt_config` table; new `prompt_hash` column on `f_note_page_content` +**Testing**: pytest + pytest-asyncio (auto mode); `unittest.mock.patch` only; integration tests use ephemeral in-process DB +**Target Platform**: Linux server (self-hosted) +**Project Type**: Web service + static frontend +**Performance Goals**: Staleness check at display time must complete within a single DB round-trip (one SELECT on `f_note_page_content` for the file); prompt hash computation is in-memory only +**Constraints**: All new Python code must pass mypy strict; no blocking I/O in async context; user note content must never be logged +**Scale/Scope**: Single-user or small-group self-hosted; no horizontal scale considerations required + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-checked after Phase 1 design.* + +| Principle | Status | Notes | +|-----------|--------|-------| +| I. Library-First | ✅ Pass | New service in `supernote/server/services/`; new route in `supernote/server/routes/`; new model in `supernote/server/db/models/` and `supernote/models/`. No circular dependencies introduced. | +| II. Protocol Fidelity | ✅ Pass | All new endpoints are under `/api/extended/` — not part of the device firmware protocol. No existing device endpoints modified. | +| III. Async-First | ✅ Pass | All DB operations in `PromptConfigService` use `async with session_manager.session()`. Prompt hash computation is in-memory (no blocking). Reprocess queuing uses the existing async `ProcessorService` queue. | +| IV. Strict Type Safety | ✅ Pass | All DTOs are `@dataclass` + `DataClassJSONMixin` with `omit_none=True`. New DB models use `Mapped[T]` / `mapped_column`. All functions carry explicit type annotations. | +| V. Observability & Data Privacy | ✅ Pass | Log prompt config saves/deletes at INFO level (category + layer only; never log prompt text content). `prompt_hash` values may be logged at DEBUG level. No note content appears in logs. | +| VI. TDD (NON-NEGOTIABLE) | ✅ Pass | Every implementation task below lists its test file. Tests are written and confirmed failing before implementation code is written. | +| VII. Security (NON-NEGOTIABLE) | ✅ Pass | All new endpoints require JWT. `PromptConfigService` enforces `user_id` scoping on every query — users cannot read or modify other users' configs. File ownership checked before staleness/reprocess endpoints. Input validated via mashumaro at route boundary. | + +**Post-design re-check**: No violations identified after Phase 1 design. + +## Project Structure + +### Documentation (this feature) + +```text +specs/004-ui-prompt-config/ +├── plan.md ← this file +├── research.md +├── data-model.md +├── quickstart.md +├── contracts/ +│ └── prompts-api.md +└── tasks.md (created by /speckit.tasks) +``` + +### Source Code + +```text +supernote/ +├── models/ +│ └── prompt_config.py # NEW — DTOs + VOs (PromptConfigDTO, staleness VOs) +├── server/ +│ ├── db/ +│ │ └── models/ +│ │ ├── note_processing.py # MODIFY — add prompt_hash column +│ │ └── prompt_config.py # NEW — PromptConfigDO +│ ├── routes/ +│ │ └── prompts.py # NEW — prompts CRUD + staleness + reprocess routes +│ ├── services/ +│ │ ├── prompt_config.py # NEW — PromptConfigService +│ │ └── processor_modules/ +│ │ ├── ocr.py # MODIFY — accept prompt_resolver kwarg, write prompt_hash +│ │ └── summary.py # MODIFY — accept prompt_resolver kwarg +│ ├── utils/ +│ │ └── prompt_loader.py # MODIFY — add get_all_known_layers() helper +│ └── app.py # MODIFY — inject PromptConfigService, register routes +│ └── static/ +│ ├── index.html # MODIFY — add Prompts header button +│ └── js/ +│ ├── api/client.js # MODIFY — add prompt + reprocess API functions +│ ├── components/ +│ │ ├── PromptsModal.js # NEW — prompt editor modal +│ │ └── FileViewer.js # MODIFY — staleness fetch + indicators + reprocess +│ └── main.js # MODIFY — showPromptsModal state + component registration +supernote/ +└── alembic/versions/ + └── XXX_add_prompt_config.py # NEW — migration: create f_prompt_config + add prompt_hash +``` + +**Structure Decision**: Single-project layout. New service/route/model files follow the existing naming and location conventions exactly. + +--- + +## Implementation Phases + +> **TDD Rule**: For every backend task, write the test file first, confirm it fails (`pytest -x`), then implement. + +--- + +### Phase A — Database Layer + +**A1 — DB Model: `PromptConfigDO`** +- File: `supernote/server/db/models/prompt_config.py` +- Fields: `id`, `user_id`, `category`, `layer`, `content`, `create_time`, `update_time` +- Unique constraint: `uq_prompt_config (user_id, category, layer)` +- Follow exact pattern of `NotePageContentDO` in `note_processing.py` +- Add import to `supernote/server/db/models/__init__.py` + +**A2 — DB Model: `NotePageContentDO` modification** +- File: `supernote/server/db/models/note_processing.py` +- Add: `prompt_hash: Mapped[str | None] = mapped_column(String, nullable=True)` + +**A3 — Alembic Migration** +- File: `supernote/alembic/versions/XXX_add_prompt_config.py` +- `upgrade()`: create `f_prompt_config` table; `op.add_column` `prompt_hash` on `f_note_page_content` +- `downgrade()`: drop column; drop table +- Test: `./script/test` (existing model completeness tests must still pass) + +--- + +### Phase B — DTOs and Models + +**B1 — DTOs** +- File: `supernote/models/prompt_config.py` +- Define: `PromptConfigDTO`, `UpsertPromptConfigDTO`, `GetPromptsResponseVO`, `PageStalenessDTO`, `FileStalenessResponseVO`, `ReprocessRequestDTO`, `ReprocessResponseVO` +- All `@dataclass` + `DataClassJSONMixin`, `omit_none=True`, camelCase aliases via `field_options` +- Test first: `tests/models/test_prompt_config_completeness.py` — round-trip serialisation for all fields + +--- + +### Phase C — `PromptConfigService` + +**C1 — Service skeleton + tests** +- Test file: `tests/server/services/test_prompt_config_service.py` +- Write failing tests for: `list_configs`, `upsert_config`, `delete_config`, `get_effective_prompt`, `compute_combined_prompt_hash` +- Tests use ephemeral in-process DB (no mocking of DB) + +**C2 — Service implementation** +- File: `supernote/server/services/prompt_config.py` +- `__init__(self, session_manager, prompt_loader)` — inject both +- `list_configs(user_id: int) -> list[PromptConfigDO]` +- `upsert_config(user_id: int, category: str, layer: str, content: str) -> PromptConfigDO` + - Validate category in `["ocr", "summary"]`; validate content not empty; validate layer pattern + - UPSERT on unique constraint +- `delete_config(user_id: int, category: str, layer: str) -> None` + - Raise `NotFoundError` if no row exists +- `get_effective_prompt(user_id: int, prompt_id: PromptId, note_type: str | None) -> str` + - Check DB for user's override for common layer → type-specific layer + - Fall back to `prompt_loader.get_prompt()` for any missing layer + - Compose using same `Common + (Custom or Default)` logic as `PromptLoader.get_prompt()` +- `compute_combined_prompt_hash(user_id: int, note_type: str | None) -> str` + - `ocr_prompt = get_effective_prompt(user_id, OCR_TRANSCRIPTION, note_type)` + - `summary_prompt = get_effective_prompt(user_id, SUMMARY_GENERATION, note_type)` + - `return sha256_string(ocr_prompt + "|" + summary_prompt)` +- `get_all_configs_with_defaults(user_id: int) -> list[PromptConfigDTO]` + - Returns merged view: all known layers from prompt_loader, overlaid with user's DB rows + - Calls `prompt_loader.get_all_known_layers()` (new helper — see Phase D) + +**C3 — `PromptLoader` helper** +- File: `supernote/server/utils/prompt_loader.py` +- Add: `get_all_known_layers() -> dict[str, dict[str, str]]` — returns `{prompt_id: {layer: default_text}}` for all layers loaded from files +- This powers the "show all layers with defaults pre-populated" response + +--- + +### Phase D — Processor Integration + +**D1 — Tests for prompt-aware processing** +- Test file: `tests/server/services/test_processor_prompt_hash.py` +- Write failing tests: + - OCR module stores `prompt_hash` on `NotePageContentDO` after processing + - Summary module uses resolved prompt text (not file-based loader) when `prompt_resolver` provided + - `ProcessorService.process_file()` passes correct hash for the file's user + note type + - `prompt_hash` is `None` for pages processed without a resolver (backward compat) + +**D2 — OCR module modification** +- File: `supernote/server/services/processor_modules/ocr.py` +- `process()` accepts `prompt_resolver: Callable[[PromptId, str | None], str] | None = None` and `prompt_hash: str | None = None` via `**kwargs` +- Replace `PROMPT_LOADER.get_prompt(...)` with `prompt_resolver(...)` if provided, else fall back to `PROMPT_LOADER.get_prompt(...)` +- After writing `text_content`, also write `content.prompt_hash = prompt_hash` if provided + +**D3 — Summary module modification** +- File: `supernote/server/services/processor_modules/summary.py` +- Same `prompt_resolver` kwarg pattern as OCR module +- Replace `PROMPT_LOADER.get_prompt(...)` call with resolver if provided + +**D4 — `ProcessorService` modification** +- File: `supernote/server/services/processor.py` +- Inject `PromptConfigService` (add to `__init__`) +- In `process_file(file_id)`: + - Look up `file_do` to get `user_id` and `file_name` + - Derive `note_type = Path(file_do.file_name).stem.lower()` + - Create `prompt_resolver = lambda prompt_id, custom_type: prompt_config_service.get_effective_prompt(user_id, prompt_id, custom_type or note_type)` + - Compute `prompt_hash = await prompt_config_service.compute_combined_prompt_hash(user_id, note_type)` + - Pass `prompt_resolver=prompt_resolver, prompt_hash=prompt_hash` to all module `.run()` calls via kwargs +- In `app.py`: instantiate `PromptConfigService` and pass to `ProcessorService` + +**D5 — Reprocess service method** +- Add `reprocess_pages(file_id: int, page_ids: list[str]) -> int` to `ProcessorService` + - For each `page_id` in list: reset `SystemTaskDO` for `OCR_EXTRACTION` and `EMBEDDING_GENERATION` to PENDING + - Reset `SystemTaskDO` for `SUMMARY_GENERATION` (global key) to PENDING + - Enqueue `file_id` via existing `self.queue.put_nowait(file_id)` + - Return count of pages queued +- Test: `tests/server/services/test_processor_prompt_hash.py` (extend existing test file) + +--- + +### Phase E — Routes + +**E1 — Tests for prompt routes** +- Test file: `tests/server/routes/test_prompts.py` +- Write failing tests for all 5 endpoints (see contracts) +- Include security tests: unauthenticated → 401; other user's file → 403 +- Test input validation: empty content → 400; invalid category → 400 + +**E2 — Route implementation** +- File: `supernote/server/routes/prompts.py` +- `GET /api/extended/prompts` → `prompt_config_service.get_all_configs_with_defaults(user_id)` +- `PUT /api/extended/prompts` → validate via mashumaro `UpsertPromptConfigDTO`, call `upsert_config` +- `DELETE /api/extended/prompts/{category}/{layer}` → call `delete_config`; return 404 if not found +- `GET /api/extended/files/{file_id}/staleness` → verify ownership; load pages; compute current hash; return per-page diff +- `POST /api/extended/files/{file_id}/reprocess` → verify ownership; compute stale page IDs; call `processor_service.reprocess_pages` +- `POST /api/extended/files/{file_id}/pages/{page_id}/reprocess` → verify ownership; check page is stale; call `reprocess_pages([page_id])` +- Register `prompts.routes` in `app.py` + +--- + +### Phase F — Frontend + +> Frontend tasks do not require pre-written tests but should be manually verified against the quickstart end-to-end scenario. + +**F1 — API client functions** +- File: `supernote/server/static/js/api/client.js` +- Add: `fetchPrompts()`, `savePrompt(category, layer, content)`, `deletePrompt(category, layer)`, `fetchStaleness(fileId)`, `reprocessFile(fileId, pageIds)`, `reprocessPage(fileId, pageId)` + +**F2 — `PromptsModal.js`** +- File: `supernote/server/static/js/components/PromptsModal.js` +- Options API component, string template (matches existing style) +- Two tabs: OCR / Summary (or two collapsible sections) +- Each section shows: Common, Default, and any custom type layers +- Each layer shows: textarea (editable), Save button, Reset to Default button (only when `isOverride: true`) +- Add custom type: text input for name + textarea; Save creates the new layer +- Delete custom type: trash icon on user-created layers +- Uses `useToast()` for save/error feedback +- Emits `close` event; parent controls visibility with `v-if` + +**F3 — `FileViewer.js` modification** +- On mount, call `fetchStaleness(fileId)` and store result +- Show stale banner in viewer header if `staleCount > 0`: `"X pages processed with outdated prompts"` + "Reprocess All" button +- In page header (`
`), add stale badge next to page number when `page.isStale` +- Add per-page "Reprocess" button in stale page headers +- On reprocess click: disable button, call API, poll processing status, re-fetch staleness on completion +- While reprocessing: show spinner inline (consistent with existing processing overlay pattern) + +**F4 — Header button + wiring** +- File: `supernote/server/static/index.html` +- Add Prompts button icon to header (between API Keys and Sign Out), visible only when `isLoggedIn` +- SVG icon: document/pencil style consistent with existing header icons +- File: `supernote/server/static/js/main.js` +- Add `showPromptsModal = ref(false)` +- Register `PromptsModal` component +- Add `` to template + +--- + +## Complexity Tracking + +No constitution violations. All changes fit within existing architectural patterns. + +--- + +## Testing Strategy Summary + +| Test File | Coverage | +|-----------|----------| +| `tests/models/test_prompt_config_completeness.py` | DTO round-trip serialisation | +| `tests/server/services/test_prompt_config_service.py` | Service CRUD, prompt resolution, hash computation, fallback to defaults | +| `tests/server/services/test_processor_prompt_hash.py` | Hash written to DB after OCR, resolver passed through pipeline, reprocess task reset | +| `tests/server/routes/test_prompts.py` | All 5 endpoints: happy path + 401/403/400/404 error cases | + +Existing tests must all continue to pass — OCR and summary modules fall back to `PROMPT_LOADER` when no resolver is injected, preserving pre-feature behaviour. diff --git a/specs/004-ui-prompt-config/quickstart.md b/specs/004-ui-prompt-config/quickstart.md new file mode 100644 index 00000000..0b8df0a8 --- /dev/null +++ b/specs/004-ui-prompt-config/quickstart.md @@ -0,0 +1,66 @@ +# Quickstart: UI Prompt Configuration + +## Development Setup + +No new dependencies. Uses existing stack: Python 3.13+, aiohttp, SQLAlchemy asyncio, alembic, mashumaro, pytest + pytest-asyncio. + +```bash +./script/bootstrap # if not already set up +./script/server # ephemeral dev server at http://localhost:8080 + # credentials: debug@example.com / password +``` + +## Running Tests + +```bash +./script/test +# or target just this feature's tests: +.venv/bin/pytest tests/server/routes/test_prompts.py \ + tests/server/services/test_prompt_config_service.py \ + tests/server/services/test_processor_prompt_hash.py \ + -v +``` + +## Applying the Migration + +The alembic migration runs automatically on server start via `run_migrations()`. For manual application: + +```bash +.venv/bin/alembic -c supernote/alembic.ini upgrade head +``` + +## Key Files + +| File | Purpose | +|------|---------| +| `supernote/server/db/models/prompt_config.py` | `PromptConfigDO` SQLAlchemy model | +| `supernote/server/db/models/note_processing.py` | `NotePageContentDO` — add `prompt_hash` column | +| `supernote/alembic/versions/XXX_add_prompt_config.py` | Migration: create table + add column | +| `supernote/models/prompt_config.py` | DTOs: `PromptConfigDTO`, `UpsertPromptConfigDTO`, staleness VOs | +| `supernote/server/services/prompt_config.py` | `PromptConfigService` | +| `supernote/server/routes/prompts.py` | Route handlers for prompts + staleness + reprocess | +| `supernote/server/services/processor.py` | Inject `PromptConfigService`, pass resolver + hash to modules | +| `supernote/server/services/processor_modules/ocr.py` | Accept `prompt_resolver`, write `prompt_hash` | +| `supernote/server/services/processor_modules/summary.py` | Accept `prompt_resolver` | +| `supernote/server/static/js/components/PromptsModal.js` | New modal component | +| `supernote/server/static/js/components/FileViewer.js` | Add staleness fetch + stale indicators | +| `supernote/server/static/js/api/client.js` | Add prompt and reprocess API functions | +| `supernote/server/static/index.html` | Add Prompts header button | +| `supernote/server/static/js/main.js` | Add `showPromptsModal` state + component registration | + +## Feature Flag / Rollout + +No feature flag needed. The feature is additive: +- Users with no saved prompt configs see identical behaviour to pre-feature (server defaults used) +- Pre-feature notes show stale indicators immediately but are not automatically reprocessed + +## Verifying End-to-End + +1. Start the ephemeral server +2. Log in as `debug@example.com` +3. Upload a `monthly.note` file and wait for processing to complete +4. Click the Prompts button in the header +5. Edit the summary prompt for the `monthly` layer and save +6. Open the note in the viewer — a stale indicator should appear +7. Click Reprocess on a stale page — verify it processes and the indicator clears +8. Verify the AI output in the Insights panel reflects the new prompt text diff --git a/specs/004-ui-prompt-config/research.md b/specs/004-ui-prompt-config/research.md new file mode 100644 index 00000000..694352da --- /dev/null +++ b/specs/004-ui-prompt-config/research.md @@ -0,0 +1,83 @@ +# Research: UI Prompt Configuration + +## Decision 1: Prompt Config Storage + +**Decision**: New `f_prompt_config` DB table with `(user_id, category, layer)` unique constraint. + +**Rationale**: Fits the existing SQLAlchemy asyncio + alembic migration pattern. Per-user rows mean no lock contention; upsert semantics keep the API simple. Server-side prompt files remain untouched as fallback. + +**Alternatives considered**: Key-value store via existing `KeyValueDO` (rejected — untyped, no query by category/layer); JSON blob on `UserDO` (rejected — no fine-grained access or per-row indexing). + +--- + +## Decision 2: Prompt Hash Scope + +**Decision**: Single SHA-256 hash per page computed over the full composed OCR prompt concatenated with the full composed summary prompt for the note's type. Stored in a new `prompt_hash` column on `NotePageContentDO`. + +**Rationale**: Confirmed by clarification Q1. A single hash avoids tracking two separate hashes per page and simplifies the staleness comparison to a single equality check. SHA-256 (already used in `supernote/server/utils/hashing.py`) is appropriate; MD5 is reserved for protocol-compatibility uses. + +**Hash input**: `sha256(ocr_prompt_text + "|" + summary_prompt_text)` where each prompt is the fully composed string (common layer + type-specific layer) as it would be sent to the AI. + +**Alternatives considered**: Separate OCR hash + summary hash per page (rejected per clarification); hashing only the user-override portion (rejected — does not detect changes to server-side defaults). + +--- + +## Decision 3: Staleness Detection Timing + +**Decision**: Lazy — computed at request time. The staleness API endpoint computes the current effective prompt hash and compares against stored `prompt_hash` per page. No DB write on prompt save. + +**Rationale**: Confirmed by clarification Q2. Avoids a potentially expensive scan-all-notes operation on every prompt save. The hash computation is a fast in-memory string operation followed by a single DB read (all pages for a file). + +**Alternatives considered**: Eager scan on save (rejected — expensive for users with many notes); background async job (rejected — adds complexity and a new task type for marginal benefit given fast lazy computation). + +--- + +## Decision 4: Prompt Resolution Architecture + +**Decision**: Introduce `PromptConfigService` which wraps `PromptLoader`. `ProcessorService` is injected with `PromptConfigService` and creates a per-file `prompt_resolver` callable before dispatching. Existing module call sites replace `PROMPT_LOADER.get_prompt(...)` with `prompt_resolver(prompt_id, custom_type)`. + +**Rationale**: Minimal invasive change to existing processor modules. The resolver pattern is a clean dependency inversion — modules remain testable in isolation by injecting a mock resolver. `PROMPT_LOADER` singleton is preserved as the fallback inside `PromptConfigService`. + +**Alternatives considered**: Pass `user_id` directly into every module method (rejected — bloats signatures, harder to test); replace `PROMPT_LOADER` singleton with a request-scoped service (rejected — processor modules run outside the HTTP request context). + +--- + +## Decision 5: Reprocess Mechanism + +**Decision**: Reset `SystemTaskDO` status to PENDING for `OCR_EXTRACTION` and `EMBEDDING_GENERATION` for targeted pages, plus `SUMMARY_GENERATION` at the global level, then enqueue the `file_id` via the existing `ProcessorService` queue. `run_if_needed()` checks for COMPLETED status, so resetting to PENDING causes modules to re-run. + +**Rationale**: Reuses the entire existing pipeline with zero changes to the processing flow. Idempotency is preserved. The existing `recover_stalled_tasks` polling loop provides resilience if a reprocess is interrupted. + +**Alternatives considered**: A separate reprocess-only code path (rejected — duplicates pipeline logic); marking the page as deleted and re-creating it (rejected — loses existing content during the window before reprocessing completes). + +--- + +## Decision 6: Stale Indicator Location + +**Decision**: Note viewer only. A new `GET /api/extended/files/{file_id}/staleness` endpoint is called on FileViewer mount. Stale indicators appear in page headers within the viewer. A note-level Reprocess button appears in the viewer header if `stale_count > 0`. + +**Rationale**: Confirmed by clarification Q3. Keeps the file list clean; users who don't care about prompt staleness are not distracted. The viewer is the natural place to see per-page detail. + +--- + +## Decision 7: Pre-feature Notes (No Stored Hash) + +**Decision**: `prompt_hash IS NULL` is treated identically to a hash mismatch — the page is stale. Reprocess buttons appear immediately for all such pages on first view after deployment. + +**Rationale**: Confirmed by clarification Q5. Consistent with the spec assumption. Users retain full control — nothing is reprocessed automatically. + +--- + +## Decision 8: Note-level Reprocess Scope + +**Decision**: Note-level Reprocess queues only pages where `stored_hash != current_hash OR stored_hash IS NULL`. Up-to-date pages are skipped. + +**Rationale**: Confirmed by clarification Q4. Avoids wasting AI tokens on pages that don't need it. + +--- + +## Decision 9: Built-in Layer Names + +**Decision**: Built-in layers (`common`, `default`, `daily`, `weekly`, `monthly`) are never written to `f_prompt_config` unless the user explicitly overrides them. The UI pre-populates these fields from `GET /api/extended/prompts/defaults` (server file content) for display only. + +**Rationale**: Zero behavioural change for users who never open the modal. Existing `PromptLoader` file-based resolution continues to serve all users until they actively save an override. diff --git a/specs/004-ui-prompt-config/spec.md b/specs/004-ui-prompt-config/spec.md new file mode 100644 index 00000000..518e5942 --- /dev/null +++ b/specs/004-ui-prompt-config/spec.md @@ -0,0 +1,216 @@ +# Feature Specification: UI Prompt Configuration + +**Feature Branch**: `004-ui-prompt-config` +**Created**: 2026-03-17 +**Status**: Implemented + +## Overview + +Previously, the AI prompts used to transcribe and summarise handwritten notes were stored as static `.md` files on the server. The note type (e.g., daily, weekly, monthly) was inferred from the filename stem, which determined which hardcoded prompt file was used. Users had no way to view, edit, or customise these prompts without direct server access. + +This feature moves prompt management into the UI, giving users full control over the instructions sent to the AI for both OCR transcription and summary generation — globally, per note type, or for custom note types they define themselves. + +The server-side prompt `.md` files have been removed entirely. Canonical defaults are now hardcoded Python constants in the server. These defaults are used when a user has no saved override for a given layer. + +The system tracks which prompt version was used to process each note and page via a combined hash. When a user updates a prompt, affected notes and pages are flagged as stale. An amber icon button appears in the note viewer header, and individual page-level reprocess buttons allow targeted reprocessing. A "Reprocess All Notes" button in the Prompts modal allows bulk reprocessing with a cost-warning confirmation step. + +## Clarifications + +### Session 2026-03-17 + +- Q: Should OCR prompt hash and summary prompt hash be tracked independently per page, or as a single combined hash? → A: Single combined hash per page — any prompt change (OCR or summary) marks the whole page stale. +- Q: Is staleness computed eagerly on prompt save, lazily at display time, or via a background job? → A: Lazy — computed at display time by comparing current effective prompt hash against the stored hash; no DB write occurs on prompt save. +- Q: Where in the UI should the stale indicator and Reprocess button appear? → A: Note viewer only — an amber icon button in the note viewer header (note level) and a Reprocess button on individual stale pages; not surfaced on file list cards. +- Q: When Reprocess is clicked at the note level, does it reprocess all pages or only stale ones? → A: Only stale pages — pages whose stored hash already matches the current effective prompt are skipped. +- Q: Should notes with no stored hash (processed before this feature) show Reprocess buttons immediately after deployment? → A: Yes — notes and pages with no stored hash are treated as stale immediately; Reprocess buttons appear in the viewer for all such items. + +### Session 2026-03-17 (implementation refinements) + +- Q: Should the UI offer a "Reset to Default" action or a "Remove" action for custom overrides? → A: Remove — clicking Remove deletes the user's saved override entirely; the field then shows the hardcoded default. No separate reset action exists. +- Q: Should server-side prompt files remain as fallback defaults? → A: No — the `.md` files were removed. Hardcoded Python string constants serve as the canonical defaults. +- Q: Which layers are protected (non-removable)? → A: Exactly three: `ocr/default`, `summary/default`, and `summary/common`. All other user-created layers (custom note types like "daily", "project", etc.) can be removed. +- Q: Does OCR have a "common" layer? → A: No — only Summary has a common layer. OCR has only a `default` layer plus any user-defined custom type layers. +- Q: Should there be a bulk reprocessing option in the Prompts modal? → A: Yes — a "Reprocess All Notes" button sits in the tab bar at the far right, always active, and shows an inline cost-warning confirmation before firing. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - View and Edit Prompts via Modal (Priority: P1) + +A logged-in user opens the Prompts modal from the header, sees their current prompt configuration (showing hardcoded defaults where no customisation exists), edits the summary prompt for "monthly" notes, saves it, and the next time their monthly note is processed the AI uses their custom text. + +**Why this priority**: This is the core value delivery — it unlocks customisation and makes the system transparent to the user for the first time. + +**Independent Test**: Can be fully tested by opening the modal, editing a prompt, saving, and triggering note reprocessing to verify the custom text reaches the AI. + +**Acceptance Scenarios**: + +1. **Given** a logged-in user with no custom prompts saved, **When** they open the Prompts modal, **Then** they see the hardcoded default text pre-populated in all prompt fields, clearly labelled as defaults (not customised). +2. **Given** a user who has edited the summary prompt for "monthly", **When** a monthly note is processed, **Then** the AI receives the user's saved text instead of the hardcoded default. +3. **Given** a user who clicks Save in the modal, **When** the save completes, **Then** a confirmation toast is shown and the changes are immediately persisted. +4. **Given** a user who closes the modal without saving, **When** they reopen it, **Then** unsaved changes are discarded and previously saved values are shown. + +--- + +### User Story 2 - Manage Custom Note Types (Priority: P2) + +A user wants to create a prompt configuration for a custom note type called "project" so that any note named `project.note` is transcribed and summarised using project-specific instructions. + +**Why this priority**: Removes the hardcoded daily/weekly/monthly limitation and makes the system extensible without server changes. + +**Independent Test**: Can be fully tested by creating a "project" type in the modal, adding prompt text, uploading a `project.note` file, and verifying the custom prompt is used during processing. + +**Acceptance Scenarios**: + +1. **Given** a user in the Prompts modal, **When** they enter a new type name (e.g., "project") and save prompt text for it, **Then** the new type appears in the modal alongside built-in types. +2. **Given** a custom note type exists, **When** a note whose filename stem matches that type is processed, **Then** the server uses the user-defined prompt for that type. +3. **Given** a custom note type, **When** the user removes it, **Then** the type disappears from the modal and future notes of that name fall back to the default prompt. +4. **Given** a user attempts to create a type with a name that already exists, **When** they save, **Then** an error is shown and no duplicate is created. + +--- + +### User Story 3 - Remove Custom Override (Priority: P3) + +A user has modified a prompt and wants to discard their customisation, reverting to the hardcoded default text. + +**Why this priority**: Provides a safety net so users are never stuck with a broken or unwanted prompt. + +**Independent Test**: Can be fully tested by editing a prompt, saving, then using the Remove action and verifying the hardcoded default text is shown and used on the next processing run. + +**Acceptance Scenarios**: + +1. **Given** a user has a saved custom override for a non-protected layer, **When** they click "Remove", **Then** the override is deleted from the database and the field reverts to displaying the hardcoded default text. +2. **Given** a user has not customised a prompt layer, **When** they view the modal, **Then** no "Remove" action is available for that layer. +3. **Given** a protected layer (`ocr/default`, `summary/default`, `summary/common`), **When** the user views it, **Then** no "Remove" action is shown regardless of whether they have a saved override. + +--- + +### User Story 4 - Summary Common Prompt Editing (Priority: P3) + +A user edits the Summary "common" prompt layer — the instructions always prepended to every summary request regardless of note type — to reflect their personal journaling style. + +**Why this priority**: The common layer is shared across all note types for summary generation; making it editable extends personalisation to the baseline behaviour. + +**Independent Test**: Can be tested by editing the summary common prompt and verifying its text appears in all subsequent summary requests regardless of note filename. + +**Acceptance Scenarios**: + +1. **Given** a user edits the summary common prompt, **When** any note is next processed, **Then** the AI receives the user's common text prepended to the type-specific summary prompt. +2. **Given** a user saves a summary common prompt, **When** they reopen the modal, **Then** their saved text is shown, not the hardcoded default. +3. **Given** the OCR tab in the Prompts modal, **When** the user views it, **Then** no "common" layer is shown — OCR has only a `default` layer (plus custom types). + +--- + +### User Story 5 - Reprocess Stale Notes and Pages (Priority: P2) + +A user updates their summary prompt for "monthly" notes. When they open a monthly note in the viewer, they see an amber reprocess icon button in the header indicating stale pages. Individual page-level Reprocess buttons appear on each stale page. The user can click either to queue targeted reprocessing. A "Reprocess All Notes" button in the Prompts modal allows bulk reprocessing of all notes. + +**Why this priority**: Without this, users who update prompts have no way to apply the new prompt to existing content without a full re-upload. This closes the feedback loop between prompt editing and AI output. + +**Independent Test**: Can be fully tested by processing a note, updating its prompt, observing the stale indicator and reprocess buttons, clicking Reprocess, and verifying the AI output reflects the new prompt. + +**Acceptance Scenarios**: + +1. **Given** a note has been processed, **When** the user opens the note viewer and the current prompt hash differs from the stored hash, **Then** an amber icon button appears in the note viewer header indicating the number of stale pages. +2. **Given** a stale note, **When** the user clicks the amber header icon button, **Then** only the stale pages are queued for reprocessing; pages whose stored hash already matches the current prompt are skipped. +3. **Given** a note where specific pages are stale, **When** the user views the note, **Then** a Reprocess button is shown at the page level for each affected page. +4. **Given** a note that was processed with the current prompt version, **When** the user views it, **Then** no amber indicator or Reprocess button is shown. +5. **Given** a user clicks Reprocess, **When** processing is already in progress for that item, **Then** duplicate queue entries are not created and the button is disabled until processing completes. +6. **Given** a user clicks "Reprocess All Notes" in the Prompts modal, **When** the inline cost warning is shown, **Then** they must confirm before any reprocessing is queued; cancelling aborts without side effects. +7. **Given** a user confirms "Reprocess All Notes", **Then** all their active `.note` files are queued for reprocessing and a toast confirms how many files were queued. + +--- + +### Edge Cases + +- What happens when a user has saved a prompt for a type but the note filename changes? The new stem is used for matching; the old type config remains but is simply unused. +- What happens when the AI service is not configured? Prompt editing is still available; changes take effect once the AI service is configured. +- What if a user saves an empty prompt? The system rejects the save with a validation message; an empty prompt would cause AI processing to fail. +- What if two browser tabs edit the same prompt simultaneously? Last write wins; the modal shows the value at time of opening. +- What happens to notes processed before a prompt change? No automatic reprocessing; notes are flagged as stale and Reprocess buttons are shown in the viewer. +- What if a custom type name conflicts with a built-in type name? The system prevents this with a validation error. +- What if a prompt override is removed and the hardcoded default matches the hash stored on the page? The page is no longer considered stale and the Reprocess button is hidden. +- What if a note's type changes (file renamed) after processing? The stale check uses the note's current filename stem against the current effective prompt; a renamed note may become stale or un-stale accordingly. +- What happens if reprocessing fails? The stale indicator remains and an error toast is shown; the user can retry. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: System MUST display a "Prompts" button in the top navigation header, visible only to logged-in users, styled consistently with existing header icons. +- **FR-002**: System MUST open a Prompts modal when the header Prompts button is clicked. +- **FR-003**: The Prompts modal MUST display an "OCR" tab and a "Summary" tab. The OCR tab shows a `default` field plus any user-defined custom type layers. The Summary tab shows a `common` field, a `default` field, and any user-defined custom type layers. +- **FR-004**: Each prompt field MUST be editable as free text and MUST display the current effective value — the user's saved override if one exists, otherwise the hardcoded default text. +- **FR-005**: System MUST visually distinguish fields that reflect a user-saved override (labelled "customised") from fields showing the hardcoded default. +- **FR-006**: System MUST persist saved prompt values per user so that each user's configuration is independent of other users. +- **FR-007**: System MUST allow users to add new custom note types by supplying a type name and prompt text for OCR, Summary, or both. +- **FR-008**: The three core layers (`ocr/default`, `summary/default`, `summary/common`) MUST be protected — no Remove button is shown for them regardless of customisation state. All user-created custom type layers CAN be removed. +- **FR-009**: System MUST allow users to remove their saved override for any non-protected layer via a "Remove" button; doing so deletes the DB row and reverts display to the hardcoded default. +- **FR-010**: System MUST reject saving an empty or whitespace-only prompt field with a clear validation message. +- **FR-011**: When processing a note, the server MUST resolve prompts by checking the processing user's saved configuration first, falling back to hardcoded defaults only when no user override exists for a given layer or type. +- **FR-012**: Note type matching during processing MUST use the note's filename stem, matched case-insensitively against the user's configured type names. +- **FR-013**: The server MUST NOT require a file-system change or restart to reflect a user's updated prompt configuration. +- **FR-014**: When processing a note or page, the server MUST record a single combined hash covering the full composed OCR prompt and the full composed summary prompt alongside the processing result. +- **FR-015**: Staleness MUST be determined lazily at display time by computing the current effective prompt hash and comparing it against the stored hash on each page; no additional DB writes are required when a prompt is saved. +- **FR-016**: The UI MUST display an amber icon button in the note viewer header when any pages are stale, indicating the count of stale pages; stale indicators MUST NOT appear on file list cards. +- **FR-017**: The UI MUST display a Reprocess button on individual stale pages inside the note viewer. +- **FR-018**: Clicking the amber header icon button MUST queue only the pages whose stored hash differs from the current effective prompt hash; pages that are not stale MUST be skipped. +- **FR-019**: While reprocessing is in progress, the Reprocess button MUST be disabled and a processing indicator shown; duplicate queue entries MUST NOT be created. +- **FR-020**: Once reprocessing completes successfully, the stale indicator and Reprocess button MUST be removed from that note or page. +- **FR-021**: The Prompts modal MUST include a "Reprocess All Notes" button in the tab bar, always visible and enabled, which queues all the user's active `.note` files for reprocessing. +- **FR-022**: Clicking "Reprocess All Notes" MUST show an inline cost-warning confirmation ("This will reprocess all notes and may incur substantial AI costs") before queuing any work; the user must explicitly confirm to proceed. + +### Key Entities + +- **PromptConfig**: A user-owned prompt override. Attributes: owning user, prompt category (`ocr` or `summary`), layer (`default`, `common`, or a named custom type string such as `daily`), prompt text content, created and updated timestamps. +- **NoteType**: A named classification for notes determined by filename stem. May be a built-in type (daily, weekly, monthly) or a user-defined custom string. Represented implicitly by the layer value on a PromptConfig — not a separate stored entity. +- **PromptHash**: A single fingerprint of the full composed prompt covering both OCR and summary layers used during a processing run. Stored once per page. Any change to either prompt for a note's type marks the whole page stale. +- **ProtectedLayer**: One of `ocr/default`, `summary/default`, or `summary/common`. These layers always exist (backed by hardcoded defaults), are editable (users can save overrides), but cannot be removed via the UI or API. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A user can open the Prompts modal, edit a prompt, and save their changes within 60 seconds without guidance. +- **SC-002**: A user-saved prompt is used by the server on the very next processing run for a matching note, with zero server restarts or file changes required. +- **SC-003**: A user with no saved prompts receives identical AI output to the pre-feature behaviour (hardcoded defaults produce the same results as the previous server-side prompt files). +- **SC-004**: The Prompts modal is reachable within 2 clicks from any page in the application. +- **SC-005**: All existing built-in note types (daily, weekly, monthly, default, common) remain fully functional after the migration, whether or not the user has saved custom overrides. +- **SC-006**: After a user saves a prompt change, stale indicators appear on affected notes on next open of the note viewer. +- **SC-007**: A user can trigger reprocessing of a single stale page without reprocessing the entire note. +- **SC-008**: Clicking "Reprocess All Notes" without confirming the cost warning does not queue any reprocessing work. + +## Scope + +### In Scope + +- Prompts modal UI with OCR and Summary tabs, view, edit, save, and remove-custom-override actions +- Per-user persistence of prompt configurations +- Server-side prompt resolution that checks user config before falling back to hardcoded defaults +- Support for both OCR and Summary prompt categories +- Summary has `common` + `default` + custom type layers; OCR has `default` + custom type layers only +- Hardcoded default constants replacing the previous server-side `.md` prompt files +- Three protected layers (`ocr/default`, `summary/default`, `summary/common`) that are editable but not removable +- Prompt hash recorded alongside every processing result +- Stale detection: comparison of current effective prompt hash against stored processing hash +- Amber icon button in note viewer header showing stale page count +- Individual page-level Reprocess buttons in note viewer +- "Reprocess All Notes" button in the Prompts modal tab bar with inline cost-warning confirmation + +### Out of Scope + +- Automatic reprocessing of previously processed notes when a prompt changes (stale detection is automatic; reprocessing requires explicit user action) +- Sharing or exporting prompt configurations between users +- Version history of prompt edits +- Live preview or test functionality (trying a prompt against a specific note in the modal) +- Admin-level management of global defaults across all users + +## Assumptions + +- The existing filename-stem matching convention (e.g., `daily.note` → type `daily`) is retained; this feature makes the registered types dynamic rather than changing the matching mechanism itself. +- For Summary, the common layer prompt is always prepended to the type-specific prompt. OCR has no common layer. +- The previous server-side prompt `.md` files have been removed. Hardcoded Python string constants (`DEFAULT_OCR_PROMPT`, `DEFAULT_SUMMARY_COMMON_PROMPT`, `DEFAULT_SUMMARY_PROMPT`) serve as canonical defaults. +- Built-in type names (daily, weekly, monthly) are not pre-populated in the DB; they are only written when a user explicitly saves a customisation for them and can be removed like any other custom type. +- All existing users start with zero saved prompt configs, meaning no behavioural change occurs until a user actively saves something. +- The modal follows the same visual conventions as existing modals in the application. +- Notes and pages processed before this feature is deployed have no stored prompt hash; they are treated as stale immediately upon deployment, and Reprocess buttons are shown in the note viewer for all such items. No automatic reprocessing occurs. +- The prompt hash covers the full composed prompt text (common layer + type-specific layer) as sent to the AI, not the individual stored PromptConfig entries. diff --git a/specs/004-ui-prompt-config/tasks.md b/specs/004-ui-prompt-config/tasks.md new file mode 100644 index 00000000..5af20764 --- /dev/null +++ b/specs/004-ui-prompt-config/tasks.md @@ -0,0 +1,223 @@ +# Tasks: UI Prompt Configuration + +**Input**: Design documents from `/specs/004-ui-prompt-config/` +**Prerequisites**: plan.md ✅ spec.md ✅ research.md ✅ data-model.md ✅ contracts/prompts-api.md ✅ quickstart.md ✅ + +**TDD**: Per Constitution §VI, tests are written FIRST and confirmed failing before implementation. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no shared dependencies) +- **[Story]**: Maps to user story from spec.md +- Exact file paths included in all descriptions + +--- + +## Phase 1: Setup + +**Purpose**: No new project or dependency setup needed — existing stack used throughout. + +- [X] T001 Verify existing test suite passes before starting: run `./script/test` and confirm green + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: DB models, DTOs, and `PromptConfigService` that all user stories depend on. + +**⚠️ CRITICAL**: No user story work can begin until this phase is complete. + +- [X] T002 Add `PromptConfigDO` SQLAlchemy model in `supernote/server/db/models/prompt_config.py` (fields: id, user_id, category, layer, content, create_time, update_time; unique constraint on user_id+category+layer per data-model.md) +- [X] T003 Add `prompt_hash: Mapped[str | None]` column to `NotePageContentDO` in `supernote/server/db/models/note_processing.py` +- [X] T004 Register `PromptConfigDO` import in `supernote/server/db/models/__init__.py` +- [X] T005 Write alembic migration in `supernote/alembic/versions/` — create `f_prompt_config` table and add `prompt_hash` column to `f_note_page_content`; include downgrade +- [X] T006 [P] Write failing completeness test in `tests/models/test_prompt_config_completeness.py` — round-trip serialisation for `PromptConfigDTO`, `UpsertPromptConfigDTO`, `GetPromptsResponseVO`, `PageStalenessDTO`, `FileStalenessResponseVO`, `ReprocessRequestDTO`, `ReprocessResponseVO` +- [X] T007 Define DTOs in `supernote/models/prompt_config.py`: `PromptConfigDTO`, `UpsertPromptConfigDTO`, `GetPromptsResponseVO`, `PageStalenessDTO`, `FileStalenessResponseVO`, `ReprocessRequestDTO`, `ReprocessResponseVO` — all `@dataclass` + `DataClassJSONMixin`, camelCase aliases, `omit_none=True` — confirm T006 now passes +- [X] T008 Write failing service tests in `tests/server/services/test_prompt_config_service.py`: `list_configs`, `upsert_config` (create + update), `delete_config` (success + NotFoundError), `get_effective_prompt` (DB override path + fallback path), `compute_combined_prompt_hash`, `get_all_configs_with_defaults`; use ephemeral in-process DB, no DB mocks +- [X] T009 Add `get_all_known_layers() -> dict[str, dict[str, str]]` helper to `supernote/server/utils/prompt_loader.py` — returns all layers loaded from files keyed by prompt_id +- [X] T010 Implement `PromptConfigService` in `supernote/server/services/prompt_config.py` with methods: `list_configs`, `upsert_config` (validates category/layer/content, upserts row), `delete_config` (raises `NotFoundError` if missing), `get_effective_prompt` (DB-first then fallback to `PromptLoader`), `compute_combined_prompt_hash` (sha256 of ocr_prompt + `"|"` + summary_prompt), `get_all_configs_with_defaults` (merged view) — confirm T008 passes +- [X] T011 Instantiate `PromptConfigService` in `supernote/server/app.py` and add to `app["prompt_config_service"]`; inject into `ProcessorService` constructor + +**Checkpoint**: `./script/test` passes. DB models, migration, DTOs, and service are complete and tested. + +--- + +## Phase 3: User Story 1 — View and Edit Prompts via Modal (Priority: P1) 🎯 MVP + +**Goal**: Logged-in user can open a Prompts modal from the header, see current prompts (defaults shown when no override exists), edit any prompt, and save it. The AI uses the saved text on the next processing run. + +**Independent Test**: Open the modal, edit the summary `monthly` layer, save, upload/reprocess a `monthly.note`, verify AI output reflects the new prompt text. + +### Tests for User Story 1 ⚠️ Write FIRST — confirm FAILING + +- [X] T012 Write failing route tests in `tests/server/routes/test_prompts.py` for: `GET /api/extended/prompts` (200 with merged defaults), `PUT /api/extended/prompts` (200 success, 400 empty content, 400 invalid category, 401 unauthenticated, verify user isolation — another user's config is not returned) + +### Implementation for User Story 1 + +- [X] T013 [US1] Implement `GET /api/extended/prompts` handler in `supernote/server/routes/prompts.py` — calls `prompt_config_service.get_all_configs_with_defaults(user_id)`, returns `GetPromptsResponseVO` +- [X] T014 [US1] Implement `PUT /api/extended/prompts` handler in `supernote/server/routes/prompts.py` — validates `UpsertPromptConfigDTO` via mashumaro, calls `upsert_config`, returns 400 on validation failure +- [X] T015 [US1] Register `prompts.routes` in `supernote/server/app.py` — confirm T012 tests now pass +- [X] T016 [P] [US1] Add API client functions `fetchPrompts()` and `savePrompt(category, layer, content)` to `supernote/server/static/js/api/client.js` +- [X] T017 [US1] Create `PromptsModal.js` in `supernote/server/static/js/components/PromptsModal.js` — Options API component; two sections (OCR / Summary); each shows Common, Default, daily, weekly, monthly layers as textareas; Save button per layer calls `savePrompt`; `isOverride` styling distinguishes saved overrides from defaults; uses `useToast()` for feedback; emits `close` +- [X] T018 [US1] Add Prompts header button to `supernote/server/static/index.html` (between API Keys and Sign Out, visible when `isLoggedIn`; document/pencil SVG icon consistent with existing icons) +- [X] T019 [US1] Wire `PromptsModal` in `supernote/server/static/js/main.js`: add `showPromptsModal = ref(false)`, register component, add `` to template + +**Checkpoint**: Prompts modal opens from header, shows defaults, saves overrides. `./script/test` still passes. + +--- + +## Phase 4: User Story 2 — Manage Custom Note Types (Priority: P2) + +**Goal**: User can add a new custom note type (e.g., `project`) with its own OCR/summary prompt, and delete it when no longer needed. + +**Independent Test**: Create a `project` type in the modal, upload a `project.note`, verify the custom prompt is used during processing. + +### Tests for User Story 2 ⚠️ Write FIRST — confirm FAILING + +- [X] T020 Extend `tests/server/routes/test_prompts.py` with failing tests for: `DELETE /api/extended/prompts/{category}/{layer}` (200 success, 404 when no override exists, 401 unauthenticated); `PUT` with custom layer name (creates new type); cross-user 403 guard on DELETE + +### Implementation for User Story 2 + +- [X] T021 [US2] Implement `DELETE /api/extended/prompts/{category}/{layer}` handler in `supernote/server/routes/prompts.py` — calls `delete_config`; returns 404 if no row exists — confirm T020 passes +- [X] T022 [US2] Add `deletePrompt(category, layer)` to `supernote/server/static/js/api/client.js` +- [X] T023 [US2] Extend `PromptsModal.js` in `supernote/server/static/js/components/PromptsModal.js`: add "Add custom type" UI (text input for name + textarea for content, Save button); add trash/delete icon on user-created custom layers (calls `deletePrompt`; removes from local state on success); prevent deletion of built-in layers (common, default, daily, weekly, monthly) + +**Checkpoint**: Custom prompt types can be created, used, and deleted via the modal. `./script/test` passes. + +--- + +## Phase 5: User Story 5 — Reprocess Stale Notes and Pages (Priority: P2) + +**Goal**: After updating a prompt, the user can open a note in the viewer, see which pages used an older prompt version, and click Reprocess to queue targeted reprocessing. + +**Independent Test**: Process a note, update its prompt, open the note viewer — stale indicators appear on pages. Click Reprocess; after processing completes, stale indicators disappear and AI output reflects the new prompt. + +### Tests for User Story 5 ⚠️ Write FIRST — confirm FAILING + +- [X] T024 Write failing tests in `tests/server/services/test_processor_prompt_hash.py`: after OCR runs with a `prompt_resolver`, `NotePageContentDO.prompt_hash` is set; `prompt_hash` is `None` when no resolver injected (backward compat); `ProcessorService.reprocess_pages` resets correct `SystemTaskDO` entries to PENDING and enqueues file; `compute_combined_prompt_hash` output matches expected sha256 +- [X] T025 [P] Extend `tests/server/routes/test_prompts.py` with failing tests for: `GET /api/extended/files/{id}/staleness` (200 with per-page stale flags, NULL hash treated as stale, 401/403 guards); `POST /api/extended/files/{id}/reprocess` (200 with queued count, 409 when already processing, 401/403 guards); `POST /api/extended/files/{id}/pages/{page_id}/reprocess` (200, 400 when not stale, 401/403 guards) + +### Implementation for User Story 5 + +- [X] T026 [US5] Modify `supernote/server/services/processor_modules/ocr.py`: accept `prompt_resolver: Callable | None = None` and `prompt_hash: str | None = None` via `**kwargs`; replace `PROMPT_LOADER.get_prompt(...)` with resolver when provided; after writing `text_content`, write `content.prompt_hash = prompt_hash` if provided +- [X] T027 [P] [US5] Modify `supernote/server/services/processor_modules/summary.py`: accept `prompt_resolver: Callable | None = None` via `**kwargs`; replace `PROMPT_LOADER.get_prompt(...)` with resolver when provided +- [X] T028 [US5] Modify `supernote/server/services/processor.py`: in `process_file()`, look up file's `user_id` and derive `note_type` from filename stem; create `prompt_resolver` closure via `prompt_config_service.get_effective_prompt`; compute `prompt_hash` via `prompt_config_service.compute_combined_prompt_hash`; pass both to all module `.run()` calls as kwargs — confirm T024 passes +- [X] T029 [US5] Add `reprocess_pages(file_id: int, page_ids: list[str]) -> int` to `supernote/server/services/processor.py`: reset `SystemTaskDO` status to PENDING for `OCR_EXTRACTION` and `EMBEDDING_GENERATION` per page and `SUMMARY_GENERATION` globally; enqueue `file_id`; return page count queued +- [X] T030 [US5] Implement `GET /api/extended/files/{file_id}/staleness` in `supernote/server/routes/prompts.py`: verify file ownership (403 if not owner); load all `NotePageContentDO` for file; compute current `prompt_hash`; return `FileStalenessResponseVO` with per-page `is_stale` flags — confirm T025 staleness tests pass +- [X] T031 [US5] Implement `POST /api/extended/files/{file_id}/reprocess` and `POST /api/extended/files/{file_id}/pages/{page_id}/reprocess` in `supernote/server/routes/prompts.py`: verify ownership; compute stale page IDs; reject non-stale page-level requests with 400; call `processor_service.reprocess_pages`; return `ReprocessResponseVO` — confirm all T025 tests pass +- [X] T032 [P] [US5] Add `fetchStaleness(fileId)`, `reprocessFile(fileId, pageIds)`, `reprocessPage(fileId, pageId)` to `supernote/server/static/js/api/client.js` +- [X] T033 [US5] Modify `supernote/server/static/js/components/FileViewer.js`: on mount call `fetchStaleness(fileId)` and store result; show stale banner in viewer header (`"X pages processed with outdated prompts"` + "Reprocess All" button) when `staleCount > 0`; add stale badge in page header div (`
`) for stale pages; add per-page "Reprocess" button; on reprocess click disable button and show inline spinner; re-fetch staleness after processing status returns complete + +**Checkpoint**: Stale indicators appear in viewer after prompt change. Reprocess queues targeted pages. `./script/test` passes. + +--- + +## Phase 6: User Stories 3 & 4 — Reset to Default + Common Prompt Editing (Priority: P3) + +**Goal**: Users can reset any prompt override back to the server default, and the common (always-applied) layer is visible and editable alongside type-specific layers. + +**Independent Test (US3)**: Edit a prompt, save, click "Reset to Default" — field reverts to server text, future processing uses server default. + +**Independent Test (US4)**: Edit the common OCR prompt, save, process any note — the custom common text appears in the AI call. + +### Tests for User Stories 3 & 4 ⚠️ Write FIRST — confirm FAILING + +- [X] T034 Extend `tests/server/routes/test_prompts.py`: verify `DELETE /api/extended/prompts/{category}/{layer}` on a built-in layer (e.g., `summary/monthly`) removes the override row and subsequent `GET /prompts` returns server default for that layer; verify common layer override is returned by `GET /prompts` and used in `compute_combined_prompt_hash` + +### Implementation for User Stories 3 & 4 + +- [X] T035 [US3] [US4] Extend `PromptsModal.js` in `supernote/server/static/js/components/PromptsModal.js`: show "Reset to Default" button only on fields where `isOverride === true`; clicking calls `deletePrompt(category, layer)` then reloads prompts and repopulates field with returned default; ensure the `common` layer row is always shown first in each category section — confirm T034 passes + +**Checkpoint**: Reset to Default works for all layers. Common layer is editable and affects hash computation. `./script/test` passes. + +--- + +## Phase 7: Polish & Cross-Cutting Concerns + +- [X] T036 [P] Review all new log statements in `supernote/server/services/prompt_config.py` and `supernote/server/routes/prompts.py` — confirm prompt text content is never logged; only category, layer, and user_id appear in INFO-level logs +- [X] T037 Run `./script/lint` and resolve any ruff or mypy strict violations across all new and modified files +- [X] T038 Run end-to-end verification from `quickstart.md`: start ephemeral server, upload a `monthly.note`, edit the monthly summary prompt, open the note viewer, confirm stale indicator, reprocess, confirm indicator clears and AI output reflects new prompt + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Phase 1** (Setup): No dependencies — start immediately +- **Phase 2** (Foundational): Depends on Phase 1 — **BLOCKS all user story phases** +- **Phase 3** (US1 — View/Edit): Depends on Phase 2 +- **Phase 4** (US2 — Custom Types): Depends on Phase 3 (extends the modal and DELETE endpoint) +- **Phase 5** (US5 — Reprocess): Depends on Phase 2; can run in parallel with Phases 3–4 +- **Phase 6** (US3+US4 — Reset/Common): Depends on Phase 3 (reuses DELETE endpoint already implemented) and Phase 2 +- **Phase 7** (Polish): Depends on all phases complete + +### User Story Dependencies + +- **US1 (P1)**: After Phase 2 — no dependencies on other stories +- **US2 (P2)**: After US1 — extends the modal with custom type management +- **US5 (P2)**: After Phase 2 — processor changes independent of modal UI; can proceed in parallel with US1 +- **US3 (P3)**: After US1 and US2 — Reset button reuses the DELETE endpoint from US2 +- **US4 (P3)**: After US1 — common layer is already returned by GET /prompts; only UI display change needed + +### Parallel Opportunities + +Within Phase 2: T002, T003, T006 can run in parallel (different files) +Within Phase 3: T016 (client.js) can run in parallel with T013–T015 (routes) +Within Phase 5: T024 and T025 (test files) can run in parallel; T026 and T027 (OCR + summary modules) can run in parallel; T032 (client.js) can run in parallel with T026–T031 + +--- + +## Parallel Example: Phase 5 (User Story 5) + +``` +# Write tests in parallel: +Task T024: tests/server/services/test_processor_prompt_hash.py +Task T025: tests/server/routes/test_prompts.py (staleness + reprocess endpoints) + +# Once tests are failing, implement in parallel: +Task T026: processor_modules/ocr.py (prompt_resolver + prompt_hash write) +Task T027: processor_modules/summary.py (prompt_resolver) +Task T032: api/client.js (fetchStaleness, reprocessFile, reprocessPage) + +# Then sequentially (dependencies): +Task T028: processor.py (pass resolver + hash to modules) +Task T029: processor.py (reprocess_pages method) +Task T030: routes/prompts.py (staleness endpoint) +Task T031: routes/prompts.py (reprocess endpoints) +Task T033: FileViewer.js (stale indicators + reprocess buttons) +``` + +--- + +## Implementation Strategy + +### MVP First (User Story 1 Only — ~11 tasks) + +1. Complete Phase 1: Setup (T001) +2. Complete Phase 2: Foundational (T002–T011) +3. Complete Phase 3: User Story 1 (T012–T019) +4. **STOP and VALIDATE**: Modal opens, prompts editable, saves override, AI uses it +5. Deploy/demo if ready + +### Incremental Delivery + +1. Phase 2 → Foundation ready (DB + service + DTOs) +2. Phase 3 → US1: Modal with view/edit/save **[MVP]** +3. Phase 4 → US2: Custom types (extends modal with add/delete) +4. Phase 5 → US5: Stale detection + reprocess (processor + viewer changes) +5. Phase 6 → US3+US4: Reset to default + common layer editing (UI polish) +6. Phase 7 → Polish, lint, e2e + +### Total Task Count + +| Phase | Tasks | Stories | +|-------|-------|---------| +| Phase 1: Setup | 1 | — | +| Phase 2: Foundational | 10 | — | +| Phase 3: US1 (P1) | 8 | US1 | +| Phase 4: US2 (P2) | 4 | US2 | +| Phase 5: US5 (P2) | 10 | US5 | +| Phase 6: US3+US4 (P3) | 2 | US3, US4 | +| Phase 7: Polish | 3 | — | +| **Total** | **38** | | diff --git a/supernote/alembic/versions/b2c3d4e5f6a7_add_prompt_config.py b/supernote/alembic/versions/b2c3d4e5f6a7_add_prompt_config.py new file mode 100644 index 00000000..9d28e408 --- /dev/null +++ b/supernote/alembic/versions/b2c3d4e5f6a7_add_prompt_config.py @@ -0,0 +1,46 @@ +"""add prompt_config table and prompt_hash column + +Revision ID: b2c3d4e5f6a7 +Revises: a1b2c3d4e5f6 +Create Date: 2026-03-17 00:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "b2c3d4e5f6a7" +down_revision: Union[str, Sequence[str], None] = "a1b2c3d4e5f6" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + """Upgrade schema.""" + op.create_table( + "f_prompt_config", + sa.Column("id", sa.BigInteger(), nullable=False), + sa.Column("user_id", sa.BigInteger(), nullable=False), + sa.Column("category", sa.String(), nullable=False), + sa.Column("layer", sa.String(), nullable=False), + sa.Column("content", sa.Text(), nullable=False), + sa.Column("create_time", sa.BigInteger(), nullable=False), + sa.Column("update_time", sa.BigInteger(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("user_id", "category", "layer", name="uq_prompt_config"), + ) + op.create_index("ix_f_prompt_config_user_id", "f_prompt_config", ["user_id"]) + op.add_column( + "f_note_page_content", + sa.Column("prompt_hash", sa.String(), nullable=True), + ) + + +def downgrade() -> None: + """Downgrade schema.""" + op.drop_column("f_note_page_content", "prompt_hash") + op.drop_index("ix_f_prompt_config_user_id", table_name="f_prompt_config") + op.drop_table("f_prompt_config") diff --git a/supernote/models/prompt_config.py b/supernote/models/prompt_config.py new file mode 100644 index 00000000..d27213fd --- /dev/null +++ b/supernote/models/prompt_config.py @@ -0,0 +1,117 @@ +"""Prompt configuration API data models.""" + +from dataclasses import dataclass, field + +from mashumaro import field_options +from mashumaro.config import BaseConfig +from mashumaro.mixins.json import DataClassJSONMixin + +from .base import BaseResponse + + +@dataclass +class PromptConfigDTO(DataClassJSONMixin): + """A single prompt layer configuration item.""" + + category: str + """Prompt category: 'ocr' or 'summary'.""" + + layer: str + """Prompt layer: 'common', 'default', or a custom type name.""" + + content: str + """Effective prompt text (user override if present, else server default).""" + + is_override: bool = field(metadata=field_options(alias="isOverride"), default=False) + """True if a user-saved row exists for this (category, layer).""" + + default_content: str = field( + metadata=field_options(alias="defaultContent"), default="" + ) + """Server-file default text (always present for Reset support).""" + + class Config(BaseConfig): + serialize_by_alias = True + omit_none = True + + +@dataclass +class UpsertPromptConfigDTO(DataClassJSONMixin): + """Request body for save/update of a single prompt layer.""" + + category: str + layer: str + content: str + + class Config(BaseConfig): + serialize_by_alias = True + omit_none = True + + +@dataclass +class GetPromptsResponseVO(BaseResponse): + """Response for GET /api/extended/prompts.""" + + prompts: list[PromptConfigDTO] = field(default_factory=list) + + class Config(BaseConfig): + serialize_by_alias = True + omit_none = True + + +@dataclass +class PageStalenessDTO(DataClassJSONMixin): + """Per-page staleness status.""" + + page_id: str = field(metadata=field_options(alias="pageId")) + page_index: int = field(metadata=field_options(alias="pageIndex")) + stored_hash: str | None = field( + metadata=field_options(alias="storedHash"), default=None + ) + is_stale: bool = field(metadata=field_options(alias="isStale"), default=False) + + class Config(BaseConfig): + serialize_by_alias = True + omit_none = True + + +@dataclass +class FileStalenessResponseVO(BaseResponse): + """Response for GET /api/extended/files/{id}/staleness.""" + + current_prompt_hash: str = field( + metadata=field_options(alias="currentPromptHash"), default="" + ) + pages: list[PageStalenessDTO] = field(default_factory=list) + stale_count: int = field(metadata=field_options(alias="staleCount"), default=0) + total_count: int = field(metadata=field_options(alias="totalCount"), default=0) + + class Config(BaseConfig): + serialize_by_alias = True + omit_none = True + + +@dataclass +class ReprocessRequestDTO(DataClassJSONMixin): + """Optional request body for POST /api/extended/files/{id}/reprocess.""" + + page_ids: list[str] | None = field( + metadata=field_options(alias="pageIds"), default=None + ) + + class Config(BaseConfig): + serialize_by_alias = True + omit_none = True + + +@dataclass +class ReprocessResponseVO(BaseResponse): + """Response for reprocess endpoints.""" + + queued_page_count: int = field( + metadata=field_options(alias="queuedPageCount"), default=0 + ) + + class Config(BaseConfig): + serialize_by_alias = True + omit_none = True diff --git a/supernote/server/app.py b/supernote/server/app.py index 675fe14f..7398a112 100644 --- a/supernote/server/app.py +++ b/supernote/server/app.py @@ -33,6 +33,7 @@ file_web, mcp, oss, + prompts, schedule, summary, system, @@ -51,6 +52,7 @@ from .services.processor_modules.page_hashing import PageHashingModule from .services.processor_modules.png_conversion import PngConversionModule from .services.processor_modules.summary import SummaryModule +from .services.prompt_config import PromptConfigService from .services.schedule import ScheduleService from .services.search import SearchService from .services.summary import SummaryService @@ -364,8 +366,13 @@ def create_app(config: ServerConfig) -> web.Application: app["sync_locks"] = {} # user -> (equipment_no, expiry_time) app["rate_limiter"] = RateLimiter(coordination_service) + from .utils.prompt_loader import PROMPT_LOADER + + prompt_config_service = PromptConfigService(session_manager, PROMPT_LOADER) + app["prompt_config_service"] = prompt_config_service + processor_service = ProcessorService( - event_bus, session_manager, file_service, summary_service + event_bus, session_manager, file_service, summary_service, prompt_config_service ) app["processor_service"] = processor_service @@ -405,6 +412,7 @@ def create_app(config: ServerConfig) -> web.Application: app.add_routes(schedule.routes) app.add_routes(summary.routes) app.add_routes(extended.routes) + app.add_routes(prompts.routes) app.add_routes(mcp.routes) # Serve static frontend files diff --git a/supernote/server/db/models/__init__.py b/supernote/server/db/models/__init__.py index 9eb88ce4..6ef95fc8 100644 --- a/supernote/server/db/models/__init__.py +++ b/supernote/server/db/models/__init__.py @@ -6,6 +6,7 @@ kv, login_record, note_processing, + prompt_config, schedule, summary, user, @@ -17,6 +18,7 @@ "kv", "login_record", "note_processing", + "prompt_config", "schedule", "summary", "user", diff --git a/supernote/server/db/models/note_processing.py b/supernote/server/db/models/note_processing.py index a4339357..519d41ae 100644 --- a/supernote/server/db/models/note_processing.py +++ b/supernote/server/db/models/note_processing.py @@ -34,6 +34,9 @@ class NotePageContentDO(Base): embedding: Mapped[str | None] = mapped_column(Text, nullable=True) """JSON string representation of the vector embedding.""" + prompt_hash: Mapped[str | None] = mapped_column(String, nullable=True) + """SHA-256 hash of the prompts used during last processing. NULL means pre-feature.""" + create_time: Mapped[int] = mapped_column( BigInteger, default=lambda: int(time.time() * 1000) ) diff --git a/supernote/server/db/models/prompt_config.py b/supernote/server/db/models/prompt_config.py new file mode 100644 index 00000000..f829cff9 --- /dev/null +++ b/supernote/server/db/models/prompt_config.py @@ -0,0 +1,44 @@ +import time + +from sqlalchemy import BigInteger, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from supernote.server.db.base import Base +from supernote.server.utils.unique_id import next_id + + +class PromptConfigDO(Base): + """Per-user prompt configuration overrides.""" + + __tablename__ = "f_prompt_config" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True, default=next_id) + """Internal database ID.""" + + user_id: Mapped[int] = mapped_column(BigInteger, index=True, nullable=False) + """The numeric ID of the user who owns this config.""" + + category: Mapped[str] = mapped_column(String, nullable=False) + """Prompt category: 'ocr' or 'summary'.""" + + layer: Mapped[str] = mapped_column(String, nullable=False) + """Prompt layer: 'common', 'default', or a user-defined type name.""" + + content: Mapped[str] = mapped_column(Text, nullable=False) + """Full prompt text for this layer.""" + + create_time: Mapped[int] = mapped_column( + BigInteger, default=lambda: int(time.time() * 1000) + ) + """System creation timestamp.""" + + update_time: Mapped[int] = mapped_column( + BigInteger, + default=lambda: int(time.time() * 1000), + onupdate=lambda: int(time.time() * 1000), + ) + """System update timestamp.""" + + __table_args__ = ( + UniqueConstraint("user_id", "category", "layer", name="uq_prompt_config"), + ) diff --git a/supernote/server/resources/prompts/ocr/common/context.md b/supernote/server/resources/prompts/ocr/common/context.md deleted file mode 100644 index 25dbf743..00000000 --- a/supernote/server/resources/prompts/ocr/common/context.md +++ /dev/null @@ -1,19 +0,0 @@ -The Bullet Journal method is a system that combines elements of mindfulness, -productivity, and self-discovery. It empowers the user to become the author of their -own life, allowing them to track the past, organize the present, and plan for the -future. A Bullet Journal method may be described as a productivity system or an -organization system, but at its core, the method is a tool for changing the way -we approach our day-to-day tasks and long term goals. The Bullet Journal method -is centered on one key idea: intentionality. Why do we do what we do? What makes -these goals meaningful to us? What tasks are the most relevant to us at any -given point in time? - -Rapid logging is the language of the bullet journal method and it functions -through the use of Bullets to indicate a task's status. A task starts with a simple -dot "•" to represent a task. If a task is completed, mark it with an "X". If it's -migrated to a future date, use a right arrow (>) to indicate that. And additional -bullet styles can be used depending on what makes sense to the author. - -Tasks within the Bullet Journal method can then fall within any of the logs used -depending on where they fall in the author's timeline. Typically, journals contain -a Daily Log, Weekly Log, and Monthly Log. diff --git a/supernote/server/resources/prompts/ocr/common/legend.md b/supernote/server/resources/prompts/ocr/common/legend.md deleted file mode 100644 index f1704315..00000000 --- a/supernote/server/resources/prompts/ocr/common/legend.md +++ /dev/null @@ -1,11 +0,0 @@ -Here is a legend for the different types of tasks in a rapid log: - o event - • task -* • critical task - X completed task - - note - < migrated to the future log - > migrated the task forward to another daily, weekly, or monthly log - -Note that * can be added to any task to indicate it is a critical task (e.g. a -critical task can be completed or migrated) diff --git a/supernote/server/resources/prompts/ocr/daily/prompt.md b/supernote/server/resources/prompts/ocr/daily/prompt.md deleted file mode 100644 index 510f1335..00000000 --- a/supernote/server/resources/prompts/ocr/daily/prompt.md +++ /dev/null @@ -1,12 +0,0 @@ -Daily Log ---------- -The daily log is used to record tasks, events, and notes on a day-to-day -basis. A daily log is simply the current date as the header and then a list of -log entries. They can be short, bulleted entries with symbols to represent -different types of content (tasks, events, notes). - -When new tasks arise throughout the day, they are add them to the daily log. If -a task is completed, it is marked with an "X". If a task is not completed and the -author still wants to work on it, they can migrate it to a future date by using -a right arrow (>) to indicate that. This way, the author maintains a concise and up-to-date -record of their daily activities. diff --git a/supernote/server/resources/prompts/ocr/default/system.md b/supernote/server/resources/prompts/ocr/default/system.md deleted file mode 100644 index 9d6aa98b..00000000 --- a/supernote/server/resources/prompts/ocr/default/system.md +++ /dev/null @@ -1,14 +0,0 @@ -You are analyzing PNG images of handwritten text from an -e-ink notebook SuperNote. The notes are written in English and are in a -bullet journal format. You can see that the text is not perfect and will need -some cleaning up - -Rapid logging is the language of the bullet journal method and it functions -through the use of Bullets to indicate a task's status. A task starts with a simple -dot "•" to represent a task. If a task is completed, mark it with an "X". If it's -migrated to a future date, use a right arrow (>) to indicate that. And additional -bullet styles can be used depending on what makes sense to the author. - -Tasks within the Bullet Journal method can then fall within any of the logs used -depending on where they fall in the author's timeline. Typically, journals contain -a Daily Log, Weekly Log, and Monthly Log. diff --git a/supernote/server/resources/prompts/ocr/monthly/prompt.md b/supernote/server/resources/prompts/ocr/monthly/prompt.md deleted file mode 100644 index a59eab25..00000000 --- a/supernote/server/resources/prompts/ocr/monthly/prompt.md +++ /dev/null @@ -1,22 +0,0 @@ -Monthly Log ------------ -The monthly log offers a broader perspective, enabling the author to plan and -track their activities, goals, and events throughout the month. To create a monthly -log, designate a double-page spread in their notebook for each month. - -A monthly spread has the name of the month at the top of the list. The left-hand -page has a list of dates (1-30/31) down the side. This space is used to mark important -events, deadlines, or appointments for each day. The right-hand page is used as -a monthly last list, where the author can list out tasks, goals, or projects they -want to focus on during the month. - -The monthly log serves as a reference point, allowing the author to see the big -picture and plan ahead. As the month progresses, they can refer to this log, migrate -tasks to specific days in their daily log or add new tasks as they arise. - -There is a special "Monthly Review" page. This is used for reflection on -the previous monthy as well as the month ahead, usually in the form of prompt -questions. Typically the monthly reflection is a time to check in on progress -on quarterly or yearly goals, and to set new goals for the upcoming month. (e.g. -What was the most memorable part of last month? What were the three biggest -lessons learned? What are my goals for the upcoming month?) diff --git a/supernote/server/resources/prompts/ocr/weekly/prompt.md b/supernote/server/resources/prompts/ocr/weekly/prompt.md deleted file mode 100644 index 08364359..00000000 --- a/supernote/server/resources/prompts/ocr/weekly/prompt.md +++ /dev/null @@ -1,16 +0,0 @@ -Weekly Log ----------- -The weekly log provides an overview of the week, allowing the author to plan and -organize tasks, events, and commitments in a broader context. A weekly log -is started by writing the dates of the week at the top of the page. Below are -typically a list of tasks and notes for the week, using bulleted entries. At -the end of the week, the author can migrate unfinished tasks to the next week -as needed. - -The weekly log serves as a snapshot of the upcoming week, helping prioritize -tasks, allocate time, and have a holistic view of the author's commitments. - -A weekly log may also be accompanied by a weekly review, where the author -reflects on the previous week's accomplishments, challenges, and goals for the upcoming -week (e.g. What did I accomplish last week? What were my biggest challenges? -What are my goals for next week?) diff --git a/supernote/server/resources/prompts/summary/common/instruction.md b/supernote/server/resources/prompts/summary/common/instruction.md deleted file mode 100644 index e4c25bef..00000000 --- a/supernote/server/resources/prompts/summary/common/instruction.md +++ /dev/null @@ -1,8 +0,0 @@ -You are an expert assistant helping to digitize and summarize a handwritten Bullet Journal. -You must extract a list of `SummarySegment` objects. -Each segment should represent a logical unit of time or topic (e.g. a single day, a week, a project). -Extract any specific dates mentioned in the segment in ISO 8601 format (YYYY-MM-DD). -Cite the page numbers (e.g. 1, 2) that contributed to each segment based on the `--- Page X ---` markers. - -The input text is an OCR transcript of handwritten notes. It may contain errors or noise. -Do your best to infer the correct meaningful content. diff --git a/supernote/server/resources/prompts/summary/daily/prompt.md b/supernote/server/resources/prompts/summary/daily/prompt.md deleted file mode 100644 index ad86e792..00000000 --- a/supernote/server/resources/prompts/summary/daily/prompt.md +++ /dev/null @@ -1,6 +0,0 @@ -This is a Daily Log. - -1. **Summary**: Create a concise paragraph summarizing today's key events, completed tasks, and any significant notes. - * Focus on what was *done* (marked with X) and what was *noted* (marked with -). - * Ignore migrated tasks (marked with > or <) unless they seem critically important context. -2. **Dates**: Extract the date of this daily log. If the text only says "Monday" or "12th", try to infer the full date from context if possible, otherwise return the partial date. diff --git a/supernote/server/resources/prompts/summary/default/prompt.md b/supernote/server/resources/prompts/summary/default/prompt.md deleted file mode 100644 index 9f0bc7e6..00000000 --- a/supernote/server/resources/prompts/summary/default/prompt.md +++ /dev/null @@ -1,4 +0,0 @@ -Analyze the following handwritten note transcript. - -1. **Summary**: Write a concise summary of the content, capturing the main ideas, tasks, and events. -2. **Dates**: Extract any specific dates mentioned in the text. diff --git a/supernote/server/resources/prompts/summary/monthly/prompt.md b/supernote/server/resources/prompts/summary/monthly/prompt.md deleted file mode 100644 index a3d5fde8..00000000 --- a/supernote/server/resources/prompts/summary/monthly/prompt.md +++ /dev/null @@ -1,6 +0,0 @@ -This is a Monthly Log or Monthly Review. - -1. **Summary**: Summarize the month's primary focus, major events, and strategic goals. - * Highlight the "Monthly Review" reflection if present. - * List the most significant events from the calendar section. -2. **Dates**: Extract the month and year (e.g. "October 2023"). diff --git a/supernote/server/resources/prompts/summary/weekly/prompt.md b/supernote/server/resources/prompts/summary/weekly/prompt.md deleted file mode 100644 index 9bc3c57d..00000000 --- a/supernote/server/resources/prompts/summary/weekly/prompt.md +++ /dev/null @@ -1,6 +0,0 @@ -This is a Weekly Log or Weekly Review. - -1. **Summary**: Summarize the week's major accomplishments, challenges, and goals. - * If there is a "Weekly Review" section (reflection questions), prioritize that content. - * Summarize the high-level tasks accomplished during the week. -2. **Dates**: Extract the date range of the week (e.g. "2023-10-23 to 2023-10-29"). diff --git a/supernote/server/routes/prompts.py b/supernote/server/routes/prompts.py new file mode 100644 index 00000000..e0670347 --- /dev/null +++ b/supernote/server/routes/prompts.py @@ -0,0 +1,423 @@ +"""Routes for prompt configuration, staleness checking, and reprocessing.""" + +import logging +from pathlib import Path +from typing import cast + +from aiohttp import web +from sqlalchemy import select + +from supernote.models.base import BaseResponse, create_error_response +from supernote.models.prompt_config import ( + FileStalenessResponseVO, + GetPromptsResponseVO, + PageStalenessDTO, + ReprocessRequestDTO, + ReprocessResponseVO, + UpsertPromptConfigDTO, +) +from supernote.server.db.models.file import UserFileDO +from supernote.server.db.models.note_processing import NotePageContentDO +from supernote.server.exceptions import SupernoteError +from supernote.server.services.processor import ProcessorService +from supernote.server.services.prompt_config import NotFoundError, PromptConfigService +from supernote.server.services.user import UserService + +logger = logging.getLogger(__name__) + +routes = web.RouteTableDef() + + +async def _get_user_id(request: web.Request) -> int | None: + """Resolve the integer user_id from the authenticated request.""" + user_email: str = request["user"] + user_service: UserService = request.app["user_service"] + return await user_service.get_user_id(user_email) + + +async def _verify_file_ownership( + request: web.Request, file_id: int, user_id: int +) -> UserFileDO | None: + """Return the UserFileDO if it belongs to user_id, else None.""" + session_manager = request.app["session_manager"] + async with session_manager.session() as session: + file_do = cast(UserFileDO | None, await session.get(UserFileDO, file_id)) + if file_do is None or file_do.user_id != user_id: + return None + return file_do + + +# --------------------------------------------------------------------------- +# GET /api/extended/prompts +# --------------------------------------------------------------------------- + + +@routes.get("/api/extended/prompts") +async def handle_get_prompts(request: web.Request) -> web.Response: + user_id = await _get_user_id(request) + if not user_id: + return web.json_response( + create_error_response("User not found").to_dict(), status=404 + ) + + prompt_config_service: PromptConfigService = request.app["prompt_config_service"] + try: + configs = await prompt_config_service.get_all_configs_with_defaults(user_id) + return web.json_response( + GetPromptsResponseVO(success=True, prompts=configs).to_dict() + ) + except Exception as err: + logger.exception("Error fetching prompts") + return SupernoteError.uncaught(err).to_response() + + +# --------------------------------------------------------------------------- +# PUT /api/extended/prompts +# --------------------------------------------------------------------------- + + +@routes.put("/api/extended/prompts") +async def handle_put_prompt(request: web.Request) -> web.Response: + user_id = await _get_user_id(request) + if not user_id: + return web.json_response( + create_error_response("User not found").to_dict(), status=404 + ) + + try: + data = await request.json() + dto = UpsertPromptConfigDTO.from_dict(data) + except Exception as e: + return web.json_response( + create_error_response(f"Invalid request: {e}", "INVALID_INPUT").to_dict(), + status=400, + ) + + prompt_config_service: PromptConfigService = request.app["prompt_config_service"] + try: + await prompt_config_service.upsert_config( + user_id=user_id, + category=dto.category, + layer=dto.layer, + content=dto.content, + ) + return web.json_response(BaseResponse(success=True).to_dict()) + except ValueError as e: + return web.json_response( + create_error_response(str(e), "INVALID_INPUT").to_dict(), status=400 + ) + except Exception as err: + logger.exception("Error saving prompt config") + return SupernoteError.uncaught(err).to_response() + + +# --------------------------------------------------------------------------- +# DELETE /api/extended/prompts/{category}/{layer} +# --------------------------------------------------------------------------- + + +@routes.delete("/api/extended/prompts/{category}/{layer}") +async def handle_delete_prompt(request: web.Request) -> web.Response: + user_id = await _get_user_id(request) + if not user_id: + return web.json_response( + create_error_response("User not found").to_dict(), status=404 + ) + + category = request.match_info["category"] + layer = request.match_info["layer"] + + prompt_config_service: PromptConfigService = request.app["prompt_config_service"] + try: + await prompt_config_service.delete_config( + user_id=user_id, category=category, layer=layer + ) + return web.json_response(BaseResponse(success=True).to_dict()) + except ValueError as e: + return web.json_response( + create_error_response(str(e), "PROTECTED_LAYER").to_dict(), + status=400, + ) + except NotFoundError: + return web.json_response( + create_error_response( + f"No override found for {category}/{layer}", "NOT_FOUND" + ).to_dict(), + status=404, + ) + except Exception as err: + logger.exception("Error deleting prompt config") + return SupernoteError.uncaught(err).to_response() + + +# --------------------------------------------------------------------------- +# GET /api/extended/files/{file_id}/staleness +# --------------------------------------------------------------------------- + + +@routes.get("/api/extended/files/{file_id}/staleness") +async def handle_get_staleness(request: web.Request) -> web.Response: + user_id = await _get_user_id(request) + if not user_id: + return web.json_response( + create_error_response("User not found").to_dict(), status=404 + ) + + try: + file_id = int(request.match_info["file_id"]) + except ValueError: + return web.json_response( + create_error_response("Invalid file_id", "INVALID_INPUT").to_dict(), + status=400, + ) + + file_do = await _verify_file_ownership(request, file_id, user_id) + if file_do is None: + return web.json_response( + create_error_response( + "File not found or access denied", "NOT_FOUND" + ).to_dict(), + status=403, + ) + + prompt_config_service: PromptConfigService = request.app["prompt_config_service"] + session_manager = request.app["session_manager"] + + try: + note_type = Path(file_do.file_name).stem.lower() if file_do.file_name else None + current_hash = await prompt_config_service.compute_combined_prompt_hash( + user_id=user_id, note_type=note_type + ) + + async with session_manager.session() as session: + stmt = ( + select(NotePageContentDO) + .where(NotePageContentDO.file_id == file_id) + .order_by(NotePageContentDO.page_index) + ) + result = await session.execute(stmt) + pages = list(result.scalars().all()) + + page_dtos = [] + stale_count = 0 + for p in pages: + is_stale = p.prompt_hash != current_hash + if is_stale: + stale_count += 1 + page_dtos.append( + PageStalenessDTO( + page_id=p.page_id or "", + page_index=p.page_index, + stored_hash=p.prompt_hash, + is_stale=is_stale, + ) + ) + + return web.json_response( + FileStalenessResponseVO( + success=True, + current_prompt_hash=current_hash, + pages=page_dtos, + stale_count=stale_count, + total_count=len(page_dtos), + ).to_dict() + ) + except Exception as err: + logger.exception("Error computing staleness") + return SupernoteError.uncaught(err).to_response() + + +# --------------------------------------------------------------------------- +# POST /api/extended/files/{file_id}/reprocess +# --------------------------------------------------------------------------- + + +@routes.post("/api/extended/files/{file_id}/reprocess") +async def handle_reprocess_file(request: web.Request) -> web.Response: + user_id = await _get_user_id(request) + if not user_id: + return web.json_response( + create_error_response("User not found").to_dict(), status=404 + ) + + try: + file_id = int(request.match_info["file_id"]) + except ValueError: + return web.json_response( + create_error_response("Invalid file_id", "INVALID_INPUT").to_dict(), + status=400, + ) + + file_do = await _verify_file_ownership(request, file_id, user_id) + if file_do is None: + return web.json_response( + create_error_response( + "File not found or access denied", "NOT_FOUND" + ).to_dict(), + status=403, + ) + + # Parse optional request body + requested_page_ids: list[str] | None = None + try: + body = await request.read() + if body: + dto = ReprocessRequestDTO.from_json(body.decode()) + requested_page_ids = dto.page_ids + except Exception: + pass # Body is optional; ignore parse errors + + prompt_config_service: PromptConfigService = request.app["prompt_config_service"] + processor_service: ProcessorService = request.app["processor_service"] + session_manager = request.app["session_manager"] + + try: + note_type = Path(file_do.file_name).stem.lower() if file_do.file_name else None + current_hash = await prompt_config_service.compute_combined_prompt_hash( + user_id=user_id, note_type=note_type + ) + + # Determine which pages to reprocess + async with session_manager.session() as session: + stmt = ( + select(NotePageContentDO) + .where(NotePageContentDO.file_id == file_id) + .order_by(NotePageContentDO.page_index) + ) + result = await session.execute(stmt) + pages = list(result.scalars().all()) + + stale_page_ids = [ + p.page_id for p in pages if p.page_id and p.prompt_hash != current_hash + ] + + if requested_page_ids is not None: + # Filter to only stale pages from the requested list + stale_set = set(stale_page_ids) + page_ids_to_reprocess = [ + pid for pid in requested_page_ids if pid in stale_set + ] + else: + page_ids_to_reprocess = stale_page_ids + + if not page_ids_to_reprocess: + return web.json_response( + ReprocessResponseVO(success=True, queued_page_count=0).to_dict() + ) + + # Check if already processing + if file_id in processor_service.processing_files: + return web.json_response( + create_error_response( + "This file is already queued for processing", "ALREADY_PROCESSING" + ).to_dict(), + status=409, + ) + + count = await processor_service.reprocess_pages( + file_id=file_id, page_ids=page_ids_to_reprocess + ) + return web.json_response( + ReprocessResponseVO(success=True, queued_page_count=count).to_dict() + ) + except Exception as err: + logger.exception("Error triggering reprocess") + return SupernoteError.uncaught(err).to_response() + + +# --------------------------------------------------------------------------- +# POST /api/extended/files/{file_id}/pages/{page_id}/reprocess +# --------------------------------------------------------------------------- + + +@routes.post("/api/extended/files/{file_id}/pages/{page_id}/reprocess") +async def handle_reprocess_page(request: web.Request) -> web.Response: + user_id = await _get_user_id(request) + if not user_id: + return web.json_response( + create_error_response("User not found").to_dict(), status=404 + ) + + try: + file_id = int(request.match_info["file_id"]) + except ValueError: + return web.json_response( + create_error_response("Invalid file_id", "INVALID_INPUT").to_dict(), + status=400, + ) + + page_id = request.match_info["page_id"] + + file_do = await _verify_file_ownership(request, file_id, user_id) + if file_do is None: + return web.json_response( + create_error_response( + "File not found or access denied", "NOT_FOUND" + ).to_dict(), + status=403, + ) + + prompt_config_service: PromptConfigService = request.app["prompt_config_service"] + processor_service: ProcessorService = request.app["processor_service"] + session_manager = request.app["session_manager"] + + try: + note_type = Path(file_do.file_name).stem.lower() if file_do.file_name else None + current_hash = await prompt_config_service.compute_combined_prompt_hash( + user_id=user_id, note_type=note_type + ) + + async with session_manager.session() as session: + stmt = select(NotePageContentDO).where( + NotePageContentDO.file_id == file_id, + NotePageContentDO.page_id == page_id, + ) + result = await session.execute(stmt) + page = result.scalar_one_or_none() + + if page is None: + return web.json_response( + create_error_response("Page not found", "NOT_FOUND").to_dict(), + status=404, + ) + + if page.prompt_hash == current_hash: + return web.json_response( + create_error_response( + "This page does not require reprocessing", "NOT_STALE" + ).to_dict(), + status=400, + ) + + count = await processor_service.reprocess_pages( + file_id=file_id, page_ids=[page_id] + ) + return web.json_response( + ReprocessResponseVO(success=True, queued_page_count=count).to_dict() + ) + except Exception as err: + logger.exception("Error triggering page reprocess") + return SupernoteError.uncaught(err).to_response() + + +# --------------------------------------------------------------------------- +# POST /api/extended/reprocess-all +# --------------------------------------------------------------------------- + + +@routes.post("/api/extended/reprocess-all") +async def handle_reprocess_all(request: web.Request) -> web.Response: + user_id = await _get_user_id(request) + if not user_id: + return web.json_response( + create_error_response("Unauthorized", "UNAUTHORIZED").to_dict(), status=401 + ) + try: + processor_service: ProcessorService = request.app["processor_service"] + count = await processor_service.reprocess_all(user_id=user_id) + return web.json_response( + ReprocessResponseVO(success=True, queued_page_count=count).to_dict() + ) + except Exception as err: + logger.exception("Error triggering reprocess all") + return SupernoteError.uncaught(err).to_response() diff --git a/supernote/server/services/processor.py b/supernote/server/services/processor.py index 1f1b8260..58e3f015 100644 --- a/supernote/server/services/processor.py +++ b/supernote/server/services/processor.py @@ -1,8 +1,12 @@ +from __future__ import annotations + import asyncio import logging import time +from pathlib import Path +from typing import TYPE_CHECKING -from sqlalchemy import delete, select +from sqlalchemy import delete, select, update from supernote.models.base import ProcessingStatus @@ -16,6 +20,9 @@ from ..services.summary import SummaryService from ..utils.paths import get_page_png_path +if TYPE_CHECKING: + from ..services.prompt_config import PromptConfigService + logger = logging.getLogger(__name__) @@ -35,12 +42,14 @@ def __init__( session_manager: DatabaseSessionManager, file_service: FileService, summary_service: SummaryService, + prompt_config_service: PromptConfigService | None = None, concurrency: int = 2, ) -> None: self.event_bus = event_bus self.session_manager = session_manager self.file_service = file_service self.summary_service = summary_service + self.prompt_config_service = prompt_config_service self.concurrency = concurrency self.queue: asyncio.Queue[int] = asyncio.Queue() # Queue of file_ids @@ -270,6 +279,26 @@ async def process_file(self, file_id: int) -> None: logger.error("Modules not fully registered. Skipping processing.") return + # Resolve prompt context for this file + prompt_resolver = None + prompt_hash: str | None = None + if self.prompt_config_service is not None: + async with self.session_manager.session() as session: + file_do = await session.get(UserFileDO, file_id) + if file_do is not None: + user_id: int = file_do.user_id + note_type: str | None = ( + Path(file_do.file_name).stem.lower() if file_do.file_name else None + ) + prompt_resolver = self.prompt_config_service.make_prompt_resolver( + user_id, note_type + ) + prompt_hash = ( + await self.prompt_config_service.compute_combined_prompt_hash( + user_id, note_type + ) + ) + # Pipeline Stage: Global Pre-processing (Hashing) for module in self.global_pre_modules: if not await module.run(file_id, self.session_manager): @@ -292,7 +321,13 @@ async def process_file(self, file_id: int) -> None: else: # Pipeline Stage: Per-Page Processing (Parallel across pages) tasks = [ - self._process_page(file_id, page_index, page_id) + self._process_page( + file_id, + page_index, + page_id, + prompt_resolver=prompt_resolver, + prompt_hash=prompt_hash, + ) for page_index, page_id in pages if page_id # Strict Check: Everything must have a page_id ] @@ -300,9 +335,20 @@ async def process_file(self, file_id: int) -> None: # Pipeline Stage: Global Post-processing (Summary) for module in self.global_post_modules: - await module.run(file_id, self.session_manager) + await module.run( + file_id, + self.session_manager, + prompt_resolver=prompt_resolver, + ) - async def _process_page(self, file_id: int, page_index: int, page_id: str) -> None: + async def _process_page( + self, + file_id: int, + page_index: int, + page_id: str, + prompt_resolver: object = None, + prompt_hash: str | None = None, + ) -> None: """Process all modules for a single page sequentially.""" for module in self.page_modules: # We enforce page_id as the task key @@ -310,7 +356,9 @@ async def _process_page(self, file_id: int, page_index: int, page_id: str) -> No file_id, self.session_manager, page_index=page_index, - page_id=page_id, # Pass page_id to modules + page_id=page_id, + prompt_resolver=prompt_resolver, + prompt_hash=prompt_hash, ) if not success: logger.warning( @@ -318,6 +366,76 @@ async def _process_page(self, file_id: int, page_index: int, page_id: str) -> No ) break + async def reprocess_pages(self, file_id: int, page_ids: list[str]) -> int: + """Reset task status for specified pages and re-enqueue the file. + + Resets OCR_EXTRACTION and EMBEDDING_GENERATION for each page_id, and + SUMMARY_GENERATION (global key) once. Returns the count of pages queued. + """ + if not page_ids: + return 0 + + async with self.session_manager.session() as session: + for page_id in page_ids: + for task_type in ("OCR_EXTRACTION", "EMBEDDING_GENERATION"): + await session.execute( + update(SystemTaskDO) + .where( + SystemTaskDO.file_id == file_id, + SystemTaskDO.task_type == task_type, + SystemTaskDO.key == page_id, + ) + .values(status=ProcessingStatus.PENDING) + ) + # Reset global summary task + await session.execute( + update(SystemTaskDO) + .where( + SystemTaskDO.file_id == file_id, + SystemTaskDO.task_type == "SUMMARY_GENERATION", + SystemTaskDO.key == "global", + ) + .values(status=ProcessingStatus.PENDING) + ) + await session.commit() + + self.processing_files.add(file_id) + await self.queue.put(file_id) + logger.info("Reprocess queued: file_id=%d pages=%d", file_id, len(page_ids)) + return len(page_ids) + + async def reprocess_all(self, user_id: int) -> int: + """Reset all processing tasks for every active note file owned by user_id. + + Resets all SystemTaskDO entries for each file to PENDING and re-enqueues + the file. Returns the count of files queued. + """ + async with self.session_manager.session() as session: + result = await session.execute( + select(UserFileDO).where( + UserFileDO.user_id == user_id, + UserFileDO.file_name.like("%.note"), + UserFileDO.is_active == "Y", + UserFileDO.is_folder == "N", + ) + ) + files = list(result.scalars().all()) + + for file_do in files: + await session.execute( + update(SystemTaskDO) + .where(SystemTaskDO.file_id == file_do.id) + .values(status=ProcessingStatus.PENDING) + ) + await session.commit() + + for file_do in files: + self.processing_files.add(file_do.id) + await self.queue.put(file_do.id) + + logger.info("Reprocess all queued: user_id=%d files=%d", user_id, len(files)) + return len(files) + async def list_system_tasks(self, limit: int = 100) -> list[SystemTaskDO]: """List recent system tasks.""" async with self.session_manager.session() as session: diff --git a/supernote/server/services/processor_modules/ocr.py b/supernote/server/services/processor_modules/ocr.py index 41998ff1..adee63de 100644 --- a/supernote/server/services/processor_modules/ocr.py +++ b/supernote/server/services/processor_modules/ocr.py @@ -1,4 +1,5 @@ import logging +from collections.abc import Awaitable, Callable from dataclasses import dataclass from pathlib import Path @@ -15,6 +16,8 @@ from supernote.server.utils.paths import get_page_png_path from supernote.server.utils.prompt_loader import PROMPT_LOADER, PromptId +PromptResolver = Callable[[PromptId, str | None], Awaitable[str]] + logger = logging.getLogger(__name__) @@ -32,10 +35,18 @@ def file_name_basis(self) -> str | None: return None -def _build_ocr_prompt(page_metadata: PageMetadata) -> str: - prompt = PROMPT_LOADER.get_prompt( - PromptId.OCR_TRANSCRIPTION, custom_type=page_metadata.file_name_basis - ) +async def _build_ocr_prompt( + page_metadata: PageMetadata, + prompt_resolver: PromptResolver | None = None, +) -> str: + if prompt_resolver is not None: + prompt = await prompt_resolver( + PromptId.OCR_TRANSCRIPTION, page_metadata.file_name_basis + ) + else: + prompt = PROMPT_LOADER.get_prompt( + PromptId.OCR_TRANSCRIPTION, custom_type=page_metadata.file_name_basis + ) metadata_block = format_page_metadata( page_index=page_metadata.page_index, page_id=page_metadata.page_id, @@ -130,13 +141,17 @@ async def process( page_id=page_id, notebook_create_time=notebook_create_time, ) - prompt = _build_ocr_prompt(page_metadata) + prompt_resolver: PromptResolver | None = kwargs.get("prompt_resolver") # type: ignore[assignment] + prompt_hash: str | None = kwargs.get("prompt_hash") # type: ignore[assignment] + prompt = await _build_ocr_prompt(page_metadata, prompt_resolver=prompt_resolver) text_content = await self.ai_service.ocr_image(png_data, prompt) async with session_manager.session() as session: content = await get_page_content_by_id(session, file_id, page_id) if content: content.text_content = text_content + if prompt_hash is not None: + content.prompt_hash = prompt_hash else: logger.warning( f"NotePageContentDO missing for {file_id} page {page_id} during OCR" diff --git a/supernote/server/services/processor_modules/summary.py b/supernote/server/services/processor_modules/summary.py index 124c1c8f..db31e07c 100644 --- a/supernote/server/services/processor_modules/summary.py +++ b/supernote/server/services/processor_modules/summary.py @@ -186,10 +186,21 @@ async def process( # Determine prompt based on filename/type custom_type = Path(file_do.file_name).stem.lower() - # Load Prompt using specialized logic: Common + (Custom OR Default) - prompt_template = PROMPT_LOADER.get_prompt( - PromptId.SUMMARY_GENERATION, custom_type=custom_type + # Use prompt_resolver if provided, otherwise fall back to PROMPT_LOADER + from collections.abc import Awaitable, Callable + + prompt_resolver: Callable[[object, str | None], Awaitable[str]] | None = ( + kwargs.get("prompt_resolver") # type: ignore[assignment] ) + if prompt_resolver is not None: + prompt_template = await prompt_resolver( + PromptId.SUMMARY_GENERATION, custom_type + ) + else: + # Load Prompt using specialized logic: Common + (Custom OR Default) + prompt_template = PROMPT_LOADER.get_prompt( + PromptId.SUMMARY_GENERATION, custom_type=custom_type + ) prompt = f"{prompt_template}\n\nTRANSCRIPT:\n{full_text}" try: diff --git a/supernote/server/services/prompt_config.py b/supernote/server/services/prompt_config.py new file mode 100644 index 00000000..327eea9c --- /dev/null +++ b/supernote/server/services/prompt_config.py @@ -0,0 +1,284 @@ +"""PromptConfigService — per-user prompt configuration management.""" + +import hashlib +import logging +import re +from typing import Callable + +from sqlalchemy import delete, select +from sqlalchemy.dialects.sqlite import insert as sqlite_insert + +from supernote.models.prompt_config import PromptConfigDTO +from supernote.server.db.models.prompt_config import PromptConfigDO +from supernote.server.db.session import DatabaseSessionManager +from supernote.server.utils.prompt_loader import ( + CATEGORY_MAP, + COMMON, + DEFAULT, + PROTECTED_LAYERS, + PromptId, + PromptLoader, +) + +logger = logging.getLogger(__name__) + +VALID_CATEGORIES = {"ocr", "summary"} +_LAYER_RE = re.compile(r"^[a-zA-Z0-9-]{1,64}$") + +# Map prompt_id value → category string (reverse of CATEGORY_MAP) +_PROMPT_ID_TO_CATEGORY: dict[str, str] = {v: k for k, v in CATEGORY_MAP.items()} +# Map category string → PromptId enum +_CATEGORY_TO_PROMPT_ID: dict[str, PromptId] = { + k: PromptId(v) for k, v in CATEGORY_MAP.items() +} + + +class NotFoundError(Exception): + """Raised when a requested resource does not exist.""" + + +class PromptConfigService: + """Service for managing per-user prompt configuration overrides.""" + + def __init__( + self, + session_manager: DatabaseSessionManager, + prompt_loader: PromptLoader, + ) -> None: + self._session_manager = session_manager + self._prompt_loader = prompt_loader + + # ------------------------------------------------------------------ + # CRUD + # ------------------------------------------------------------------ + + async def list_configs(self, user_id: int) -> list[PromptConfigDO]: + """Return all saved prompt overrides for a user.""" + async with self._session_manager.session() as session: + result = await session.execute( + select(PromptConfigDO).where(PromptConfigDO.user_id == user_id) + ) + return list(result.scalars().all()) + + async def upsert_config( + self, + user_id: int, + category: str, + layer: str, + content: str, + ) -> PromptConfigDO: + """Create or update a prompt override for (user_id, category, layer). + + Raises ValueError for invalid input. + """ + if category not in VALID_CATEGORIES: + raise ValueError( + f"Invalid category '{category}'. Must be one of {sorted(VALID_CATEGORIES)}" + ) + if not content or not content.strip(): + raise ValueError("content must not be empty or whitespace-only") + if not _LAYER_RE.match(layer): + raise ValueError( + "layer must be 1-64 characters, alphanumeric and hyphens only" + ) + + async with self._session_manager.session() as session: + stmt = ( + sqlite_insert(PromptConfigDO) + .values( + user_id=user_id, + category=category, + layer=layer, + content=content, + ) + .on_conflict_do_update( + index_elements=["user_id", "category", "layer"], + set_={"content": content}, + ) + ) + await session.execute(stmt) + await session.commit() + + result = await session.execute( + select(PromptConfigDO).where( + PromptConfigDO.user_id == user_id, + PromptConfigDO.category == category, + PromptConfigDO.layer == layer, + ) + ) + row = result.scalar_one() + logger.info( + "Upserted prompt config: user_id=%d category=%s layer=%s", + user_id, + category, + layer, + ) + return row + + async def delete_config(self, user_id: int, category: str, layer: str) -> None: + """Delete a user prompt override. Raises NotFoundError if no row exists. + + Raises ValueError for protected layers (ocr/default, summary/default, + summary/common) — those layers are always present and cannot be removed. + """ + if (category, layer) in PROTECTED_LAYERS: + raise ValueError( + f"{category}/{layer} is a protected layer and cannot be removed" + ) + async with self._session_manager.session() as session: + result = await session.execute( + select(PromptConfigDO).where( + PromptConfigDO.user_id == user_id, + PromptConfigDO.category == category, + PromptConfigDO.layer == layer, + ) + ) + row = result.scalar_one_or_none() + if row is None: + raise NotFoundError(f"No override found for {category}/{layer}") + await session.execute( + delete(PromptConfigDO).where( + PromptConfigDO.user_id == user_id, + PromptConfigDO.category == category, + PromptConfigDO.layer == layer, + ) + ) + await session.commit() + logger.info( + "Deleted prompt config: user_id=%d category=%s layer=%s", + user_id, + category, + layer, + ) + + # ------------------------------------------------------------------ + # Prompt resolution + # ------------------------------------------------------------------ + + async def get_effective_prompt( + self, + user_id: int, + prompt_id: PromptId, + note_type: str | None, + ) -> str: + """Return the effective composed prompt for the given user and note type. + + Composition: + common = user override for 'common' OR loader default common (if any) + specific = user override for note_type OR user override for 'default' + OR loader default for note_type OR loader default + result = common + specific (each part omitted when empty) + """ + category = _PROMPT_ID_TO_CATEGORY[prompt_id.value] + user_configs = await self.list_configs(user_id) + user_map: dict[str, str] = { + c.layer: c.content for c in user_configs if c.category == category + } + + all_layers = self._prompt_loader.get_all_known_layers() + file_map: dict[str, str] = all_layers.get(prompt_id.value, {}) + + parts: list[str] = [] + + # Common portion (present for summary, absent for OCR by default) + common_text = user_map.get(COMMON) or file_map.get(COMMON, "") + if common_text: + parts.append(common_text) + + # Specific portion: type-specific override > default override > file type > file default + specific_text: str | None = None + if note_type: + specific_text = user_map.get(note_type) or file_map.get(note_type) + if specific_text is None: + specific_text = user_map.get(DEFAULT) or file_map.get(DEFAULT) + + if specific_text: + parts.append(specific_text) + elif not parts: + raise ValueError( + f"No prompt content for {prompt_id.value} (note_type={note_type})" + ) + + return "\n\n".join(parts) + + async def compute_combined_prompt_hash( + self, user_id: int, note_type: str | None + ) -> str: + """Return SHA-256 hex of (ocr_prompt + '|' + summary_prompt).""" + ocr_prompt = await self.get_effective_prompt( + user_id, PromptId.OCR_TRANSCRIPTION, note_type + ) + summary_prompt = await self.get_effective_prompt( + user_id, PromptId.SUMMARY_GENERATION, note_type + ) + combined = ocr_prompt + "|" + summary_prompt + return hashlib.sha256(combined.encode("utf-8")).hexdigest() + + # ------------------------------------------------------------------ + # Merged view for the API + # ------------------------------------------------------------------ + + async def get_all_configs_with_defaults( + self, user_id: int + ) -> list[PromptConfigDTO]: + """Return merged view: all known layers overlaid with user's DB rows. + + Each entry includes default_content (server file text) for Reset support. + """ + user_configs = await self.list_configs(user_id) + user_map: dict[tuple[str, str], str] = { + (c.category, c.layer): c.content for c in user_configs + } + + all_layers = self._prompt_loader.get_all_known_layers() + + result: list[PromptConfigDTO] = [] + for prompt_id_value, layer_map in all_layers.items(): + category = _PROMPT_ID_TO_CATEGORY.get(prompt_id_value) + if category is None: + continue + for layer, default_text in layer_map.items(): + key = (category, layer) + is_override = key in user_map + content = user_map[key] if is_override else default_text + result.append( + PromptConfigDTO( + category=category, + layer=layer, + content=content, + is_override=is_override, + default_content=default_text, + ) + ) + + # Also include user-saved custom layers not in file loader + for (category, layer), content in user_map.items(): + known_layers = all_layers.get(CATEGORY_MAP.get(category, ""), {}) + if layer not in known_layers: + result.append( + PromptConfigDTO( + category=category, + layer=layer, + content=content, + is_override=True, + default_content=content, + ) + ) + + return result + + def make_prompt_resolver( + self, user_id: int, note_type: str | None + ) -> Callable[[PromptId, str | None], "PromptConfigService"]: + """Return a synchronous-style resolver suitable for async wrapping. + + Use this to create a resolver lambda for passing to processor modules. + The returned callable is a coroutine factory — callers must await it. + """ + + async def resolver(prompt_id: PromptId, custom_type: str | None = None) -> str: + return await self.get_effective_prompt( + user_id, prompt_id, custom_type or note_type + ) + + return resolver # type: ignore[return-value] diff --git a/supernote/server/static/index.html b/supernote/server/static/index.html index 4285e8b6..e1b8885f 100644 --- a/supernote/server/static/index.html +++ b/supernote/server/static/index.html @@ -73,6 +73,14 @@

Supernote + + class="px-4 py-2 bg-white dark:bg-gray-600 border border-gray-300 dark:border-gray-500 rounded text-sm font-medium text-gray-700 dark:text-gray-200 hover:bg-gray-50 dark:hover:bg-gray-500 transition-colors">Cancel

@@ -205,6 +213,9 @@

Create New Folder< + + +

diff --git a/supernote/server/static/js/api/client.js b/supernote/server/static/js/api/client.js index 655c1a61..6f8e5eb1 100644 --- a/supernote/server/static/js/api/client.js +++ b/supernote/server/static/js/api/client.js @@ -569,3 +569,147 @@ export async function fetchProcessingStatus(fileIds) { statusMap: data.statusMap }; } + +/** + * Fetch all prompt configurations for the current user. + * @returns {Promise<{success: boolean, prompts: Array}>} + */ +export async function fetchPrompts() { + const currentToken = getToken(); + if (!currentToken) throw new Error("Unauthorized"); + + const response = await fetch('/api/extended/prompts', { + headers: { 'x-access-token': currentToken } + }); + + if (response.status === 401) { logout(); throw new Error("Unauthorized"); } + if (!response.ok) throw new Error(`Failed to fetch prompts: ${response.status}`); + return await response.json(); +} + +/** + * Save or update a prompt layer override. + * @param {string} category - 'ocr' or 'summary' + * @param {string} layer - layer name + * @param {string} content - prompt text + * @returns {Promise<{success: boolean}>} + */ +export async function savePrompt(category, layer, content) { + const currentToken = getToken(); + if (!currentToken) throw new Error("Unauthorized"); + + const response = await fetch('/api/extended/prompts', { + method: 'PUT', + headers: { + 'Content-Type': 'application/json', + 'x-access-token': currentToken + }, + body: JSON.stringify({ category, layer, content }) + }); + + if (response.status === 401) { logout(); throw new Error("Unauthorized"); } + if (!response.ok) throw new Error(`Failed to save prompt: ${response.status}`); + return await response.json(); +} + +/** + * Delete a prompt override, reverting to server default. + * @param {string} category + * @param {string} layer + * @returns {Promise<{success: boolean}>} + */ +export async function deletePrompt(category, layer) { + const currentToken = getToken(); + if (!currentToken) throw new Error("Unauthorized"); + + const response = await fetch(`/api/extended/prompts/${category}/${layer}`, { + method: 'DELETE', + headers: { 'x-access-token': currentToken } + }); + + if (response.status === 401) { logout(); throw new Error("Unauthorized"); } + if (!response.ok) throw new Error(`Failed to delete prompt: ${response.status}`); + return await response.json(); +} + +/** + * Fetch staleness data for a file. + * @param {number} fileId + * @returns {Promise<{success: boolean, currentPromptHash: string, staleCount: number, totalCount: number, pages: Array}>} + */ +export async function fetchStaleness(fileId) { + const currentToken = getToken(); + if (!currentToken) throw new Error("Unauthorized"); + + const response = await fetch(`/api/extended/files/${fileId}/staleness`, { + headers: { 'x-access-token': currentToken } + }); + + if (response.status === 401) { logout(); throw new Error("Unauthorized"); } + if (!response.ok) throw new Error(`Failed to fetch staleness: ${response.status}`); + return await response.json(); +} + +/** + * Reprocess stale pages for a file. + * @param {number} fileId + * @param {Array|null} pageIds - specific page IDs, or null for all stale + * @returns {Promise<{success: boolean, queuedPageCount: number}>} + */ +export async function reprocessFile(fileId, pageIds = null) { + const currentToken = getToken(); + if (!currentToken) throw new Error("Unauthorized"); + + const body = pageIds !== null ? JSON.stringify({ pageIds }) : '{}'; + + const response = await fetch(`/api/extended/files/${fileId}/reprocess`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-access-token': currentToken + }, + body + }); + + if (response.status === 401) { logout(); throw new Error("Unauthorized"); } + if (!response.ok) throw new Error(`Failed to reprocess file: ${response.status}`); + return await response.json(); +} + +/** + * Reprocess a single page. + * @param {number} fileId + * @param {string} pageId + * @returns {Promise<{success: boolean, queuedPageCount: number}>} + */ +export async function reprocessPage(fileId, pageId) { + const currentToken = getToken(); + if (!currentToken) throw new Error("Unauthorized"); + + const response = await fetch(`/api/extended/files/${fileId}/pages/${pageId}/reprocess`, { + method: 'POST', + headers: { 'x-access-token': currentToken } + }); + + if (response.status === 401) { logout(); throw new Error("Unauthorized"); } + if (!response.ok) throw new Error(`Failed to reprocess page: ${response.status}`); + return await response.json(); +} + +/** + * Reprocess all note files for the current user. + * @returns {Promise<{success: boolean, queuedPageCount: number}>} + */ +export async function reprocessAllNotes() { + const currentToken = getToken(); + if (!currentToken) throw new Error("Unauthorized"); + + const response = await fetch('/api/extended/reprocess-all', { + method: 'POST', + headers: { 'x-access-token': currentToken } + }); + + if (response.status === 401) { logout(); throw new Error("Unauthorized"); } + if (!response.ok) throw new Error(`Failed to reprocess all notes: ${response.status}`); + return await response.json(); +} diff --git a/supernote/server/static/js/components/ApiKeysPanel.js b/supernote/server/static/js/components/ApiKeysPanel.js index 41b123ea..bceb6421 100644 --- a/supernote/server/static/js/components/ApiKeysPanel.js +++ b/supernote/server/static/js/components/ApiKeysPanel.js @@ -14,7 +14,7 @@ export default {

MCP Credentials

Manage API keys and session tokens for MCP client access.

- @@ -42,14 +42,14 @@ export default { v-model="newKeyName" type="text" placeholder="Key name (e.g. Claude Desktop)" - class="flex-1 px-3 py-2 border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-600 text-black dark:text-white placeholder-gray-400 dark:placeholder-gray-400 rounded text-sm focus:outline-none focus:ring-2 focus:ring-indigo-500" + class="flex-1 px-3 py-2 border border-gray-300 dark:border-gray-600 bg-white dark:bg-gray-600 text-black dark:text-white placeholder-gray-400 dark:placeholder-gray-400 rounded text-sm focus:outline-none focus:ring-2 focus:ring-black dark:focus:ring-white" @keyup.enter="handleCreate" :disabled="creating" /> @@ -61,7 +61,7 @@ export default {

Existing Keys

-
+
No API keys yet. @@ -92,7 +92,7 @@ export default {

Connected Sessions

OAuth clients connected via the browser login flow (e.g. claude.ai). Disconnecting revokes the refresh token — the client will need to re-authenticate.

-
+
No active sessions. @@ -121,7 +121,7 @@ export default {
-
diff --git a/supernote/server/static/js/components/FileViewer.js b/supernote/server/static/js/components/FileViewer.js index 922acd09..64088a28 100644 --- a/supernote/server/static/js/components/FileViewer.js +++ b/supernote/server/static/js/components/FileViewer.js @@ -1,5 +1,5 @@ -import { ref, watch, onMounted } from 'vue'; -import { convertNoteToPng } from '../api/client.js'; +import { ref, computed, watch, onMounted } from 'https://unpkg.com/vue@3/dist/vue.esm-browser.js'; +import { convertNoteToPng, fetchStaleness, reprocessFile, reprocessPage } from '../api/client.js'; import SummaryPanel from './SummaryPanel.js'; export default { @@ -19,12 +19,36 @@ export default { const error = ref(null); const showDetails = ref(false); + // Staleness state + const stalenessData = ref(null); // full response from /staleness + const reprocessingAll = ref(false); + const reprocessingPages = ref({}); // pageId -> boolean + + const staleCount = computed(() => stalenessData.value?.staleCount ?? 0); + + // Map pageIndex (0-based) -> PageStalenessDTO + const stalenessByIndex = computed(() => { + if (!stalenessData.value) return {}; + const map = {}; + for (const p of stalenessData.value.pages) { + map[p.pageIndex] = p; + } + return map; + }); + + function isPageStale(page) { + const info = stalenessByIndex.value[page.pageNo - 1]; + return info ? info.isStale : false; + } + + function pageId(page) { + const info = stalenessByIndex.value[page.pageNo - 1]; + return info ? info.pageId : null; + } + const loadPages = async () => { if (!props.file) return; - // Only convert .note files. For others, just show placeholder for now. - // In a real app, we'd handle PDF/PNG native viewing here too. - // But our goal is .note conversion. if (!props.file.name.endsWith('.note')) { error.value = "Preview not available for this file type."; return; @@ -33,6 +57,7 @@ export default { isLoading.value = true; error.value = null; pages.value = []; + stalenessData.value = null; try { const result = await convertNoteToPng(props.file.id); @@ -47,8 +72,48 @@ export default { } finally { isLoading.value = false; } + + // Load staleness in background + loadStaleness(); }; + async function loadStaleness() { + try { + const data = await fetchStaleness(props.file.id); + stalenessData.value = data; + } catch (e) { + // Staleness is non-critical; fail silently + console.warn('Staleness fetch failed:', e.message); + } + } + + async function handleReprocessAll() { + reprocessingAll.value = true; + try { + await reprocessFile(props.file.id, null); + // Poll: reload staleness after a delay + setTimeout(loadStaleness, 3000); + } catch (e) { + console.error('Reprocess all failed:', e.message); + } finally { + reprocessingAll.value = false; + } + } + + async function handleReprocessPage(page) { + const pid = pageId(page); + if (!pid) return; + reprocessingPages.value = { ...reprocessingPages.value, [pid]: true }; + try { + await reprocessPage(props.file.id, pid); + setTimeout(loadStaleness, 3000); + } catch (e) { + console.error('Reprocess page failed:', e.message); + } finally { + reprocessingPages.value = { ...reprocessingPages.value, [pid]: false }; + } + } + onMounted(loadPages); watch(() => props.file, loadPages); @@ -56,7 +121,15 @@ export default { pages, isLoading, error, - showDetails + showDetails, + stalenessData, + staleCount, + reprocessingAll, + reprocessingPages, + isPageStale, + pageId, + handleReprocessAll, + handleReprocessPage, }; }, template: ` @@ -73,14 +146,22 @@ export default {
+
@@ -110,7 +191,19 @@ export default {
- Page {{ page.pageNo }} +
+ Page {{ page.pageNo }} + stale +
+
Note Page
diff --git a/supernote/server/static/js/components/MoveModal.js b/supernote/server/static/js/components/MoveModal.js index c2419f7a..9066fac6 100644 --- a/supernote/server/static/js/components/MoveModal.js +++ b/supernote/server/static/js/components/MoveModal.js @@ -30,9 +30,9 @@ export default {
- +
diff --git a/supernote/server/static/js/components/PromptsModal.js b/supernote/server/static/js/components/PromptsModal.js new file mode 100644 index 00000000..e991ad72 --- /dev/null +++ b/supernote/server/static/js/components/PromptsModal.js @@ -0,0 +1,268 @@ +import { ref, onMounted, computed } from 'https://unpkg.com/vue@3/dist/vue.esm-browser.js'; +import { fetchPrompts, savePrompt, deletePrompt, reprocessAllNotes } from '../api/client.js'; +import { useToast } from '../composables/useToast.js'; + +const { addToast } = useToast(); + +export default { + name: 'PromptsModal', + emits: ['close'], + template: ` +
+
+ +
+
+

Prompt Configuration

+

Customise OCR and summary prompts per note type.

+
+ +
+ + +
+ +
+ + + + +
+
+ + +
+
+
+
+ +
+ +
+
+
+ {{ prompt.layer }} + customised +
+ +
+
+ +
+ +
+
+
+ + +
+

Add Custom Note Type

+
+ +
+ + +
+
+
+
+
+ `, + setup(props, { emit }) { + const loading = ref(true); + const prompts = ref([]); + const activeTab = ref('ocr'); + const editContents = ref({}); + const saving = ref({}); + const newLayerName = ref(''); + const newLayerContent = ref(''); + const addingCustom = ref(false); + const newContentRef = ref(null); + const reprocessingAllNotes = ref(false); + const confirmingReprocessAll = ref(false); + + const tabLabels = { ocr: 'OCR', summary: 'Summary' }; + + const tabPrompts = computed(() => + prompts.value.filter(p => p.category === activeTab.value) + ); + + async function load() { + loading.value = true; + try { + const data = await fetchPrompts(); + prompts.value = data.prompts || []; + // Populate edit buffers + const contents = {}; + for (const p of prompts.value) { + contents[p.category + '/' + p.layer] = p.content; + } + editContents.value = contents; + } catch (e) { + addToast('Failed to load prompts: ' + e.message, 'error'); + } finally { + loading.value = false; + } + } + + async function handleSave(prompt) { + const key = prompt.category + '/' + prompt.layer; + const content = editContents.value[key]; + if (!content || !content.trim()) return; + saving.value = { ...saving.value, [key]: true }; + try { + await savePrompt(prompt.category, prompt.layer, content.trim()); + addToast('Prompt saved.', 'success'); + await load(); + } catch (e) { + addToast('Save failed: ' + e.message, 'error'); + } finally { + saving.value = { ...saving.value, [key]: false }; + } + } + + const PROTECTED = new Set(['ocr/default', 'summary/default', 'summary/common']); + function isProtected(prompt) { + return PROTECTED.has(prompt.category + '/' + prompt.layer); + } + + async function handleRemove(prompt) { + const key = prompt.category + '/' + prompt.layer; + saving.value = { ...saving.value, [key]: true }; + try { + await deletePrompt(prompt.category, prompt.layer); + addToast('Prompt removed.', 'success'); + await load(); + } catch (e) { + addToast('Remove failed: ' + e.message, 'error'); + } finally { + saving.value = { ...saving.value, [key]: false }; + } + } + + async function handleAddCustom() { + const layer = newLayerName.value.trim(); + const content = newLayerContent.value.trim(); + if (!layer || !content) return; + addingCustom.value = true; + try { + await savePrompt(activeTab.value, layer, content); + addToast('Custom type added.', 'success'); + newLayerName.value = ''; + newLayerContent.value = ''; + await load(); + } catch (e) { + addToast('Failed to add custom type: ' + e.message, 'error'); + } finally { + addingCustom.value = false; + } + } + + function focusNewContent() { + if (newContentRef.value) newContentRef.value.focus(); + } + + async function confirmReprocessAllNotes() { + reprocessingAllNotes.value = true; + try { + const result = await reprocessAllNotes(); + addToast(`Queued ${result.queuedPageCount ?? 0} file(s) for reprocessing.`, 'success'); + confirmingReprocessAll.value = false; + } catch (e) { + addToast('Reprocess all failed: ' + e.message, 'error'); + } finally { + reprocessingAllNotes.value = false; + } + } + + onMounted(load); + + return { + loading, + prompts, + activeTab, + tabLabels, + tabPrompts, + editContents, + saving, + newLayerName, + newLayerContent, + newContentRef, + addingCustom, + reprocessingAllNotes, + confirmingReprocessAll, + isProtected, + handleSave, + handleRemove, + handleAddCustom, + focusNewContent, + confirmReprocessAllNotes, + }; + } +}; diff --git a/supernote/server/static/js/components/RenameModal.js b/supernote/server/static/js/components/RenameModal.js index 2f90b8a7..1719b037 100644 --- a/supernote/server/static/js/components/RenameModal.js +++ b/supernote/server/static/js/components/RenameModal.js @@ -9,9 +9,9 @@ export default { class="w-full px-4 py-3 bg-white dark:bg-gray-700 dark:text-white dark:placeholder-gray-400 border border-gray-300 dark:border-gray-600 rounded focus:ring-2 focus:ring-black dark:focus:ring-white focus:border-black dark:focus:border-white outline-none transition-all mb-6" @keyup.enter="handleRename" ref="nameInput">
- +
diff --git a/supernote/server/static/js/main.js b/supernote/server/static/js/main.js index b3d3fdaf..0a7784de 100644 --- a/supernote/server/static/js/main.js +++ b/supernote/server/static/js/main.js @@ -7,6 +7,7 @@ import LoginCard from './components/LoginCard.js'; import FileViewer from './components/FileViewer.js'; import SystemPanel from './components/SystemPanel.js'; import ApiKeysPanel from './components/ApiKeysPanel.js'; +import PromptsModal from './components/PromptsModal.js'; import MoveModal from './components/MoveModal.js'; import RenameModal from './components/RenameModal.js'; import ToastContainer from './components/ToastContainer.js'; @@ -18,6 +19,7 @@ createApp({ FileViewer, SystemPanel, ApiKeysPanel, + PromptsModal, MoveModal, RenameModal, ToastContainer @@ -44,6 +46,7 @@ createApp({ const loginError = ref(null); const showSystemPanel = ref(false); const showApiKeysPanel = ref(false); + const showPromptsModal = ref(false); // UI State const showNewFolderModal = ref(false); @@ -321,6 +324,7 @@ createApp({ selectedFile, showSystemPanel, showApiKeysPanel, + showPromptsModal, // New States showNewFolderModal, diff --git a/supernote/server/utils/prompt_loader.py b/supernote/server/utils/prompt_loader.py index 813178fb..1a045e2f 100644 --- a/supernote/server/utils/prompt_loader.py +++ b/supernote/server/utils/prompt_loader.py @@ -1,12 +1,37 @@ import enum -import importlib.resources import logging -from importlib.resources.abc import Traversable -from pathlib import Path logger = logging.getLogger(__name__) -RESOURCES_DIR = importlib.resources.files("supernote.server") / "resources" / "prompts" +DEFAULT_OCR_PROMPT = """You are analyzing PNG images of handwritten text from an \ +e-ink notebook SuperNote. The notes are written in English and are in a \ +bullet journal format. You can see that the text is not perfect and will need \ +some cleaning up + +Rapid logging is the language of the bullet journal method and it functions \ +through the use of Bullets to indicate a task's status. A task starts with a simple \ +dot "•" to represent a task. If a task is completed, mark it with an "X". If it's \ +migrated to a future date, use a right arrow (>) to indicate that. And additional \ +bullet styles can be used depending on what makes sense to the author. + +Tasks within the Bullet Journal method can then fall within any of the logs used \ +depending on where they fall in the author's timeline. Typically, journals contain \ +a Daily Log, Weekly Log, and Monthly Log.""" + +DEFAULT_SUMMARY_COMMON_PROMPT = """You are an expert assistant helping to digitize \ +and summarize a handwritten Bullet Journal. +You must extract a list of `SummarySegment` objects. +Each segment should represent a logical unit of time or topic \ +(e.g. a single day, a week, a project). +Extract any specific dates mentioned in the segment in ISO 8601 format (YYYY-MM-DD). +Cite the page numbers (e.g. 1, 2) that contributed to each segment \ +based on the `--- Page X ---` markers. + +The input text is an OCR transcript of handwritten notes. It may contain errors or noise. +Do your best to infer the correct meaningful content.""" + +DEFAULT_SUMMARY_PROMPT = """Summarise the content of these notes, extracting \ +key topics, any tasks or action items, and important events or decisions.""" class PromptId(str, enum.Enum): @@ -21,109 +46,46 @@ class PromptId(str, enum.Enum): COMMON = "common" DEFAULT = "default" +# Layers that are always present and cannot be removed via the API +PROTECTED_LAYERS: frozenset[tuple[str, str]] = frozenset( + {("ocr", DEFAULT), ("summary", DEFAULT), ("summary", COMMON)} +) + class PromptLoader: - """Service to load and manage externalized prompts.""" - - def __init__(self, resources_dir: Path | Traversable | None = None) -> None: - self.resources_dir = resources_dir or RESOURCES_DIR - # Map: prompt_id -> (type -> prompt_text) - # type can be "common", "default", or specific custom types like "monthly" - self.prompts: dict[str, dict[str, str]] = {} - self._load_prompts() - - def _load_prompts(self) -> None: - """Load all prompts from the resources directory into memory.""" - if not self.resources_dir.is_dir(): - logger.warning(f"Prompts directory not found at {self.resources_dir}") - return - - # Initialize dicts for known prompt IDs - for pid in PromptId: - self.prompts[pid.value] = {} - - try: - # Map categories to PromptId - for category, prompt_id in CATEGORY_MAP.items(): - category_dir = self.resources_dir / category - if not category_dir.is_dir(): - continue - - if prompt_id not in self.prompts: - self.prompts[prompt_id] = {} - - # 1. Load Common Prompts (Always On) - common_dir = category_dir / COMMON - if common_dir.is_dir(): - common_text = self._read_prompts_from_dir(common_dir) - if common_text: - self.prompts[prompt_id][COMMON] = common_text - - # 2. Load Default Prompts - default_dir = category_dir / DEFAULT - if default_dir.is_dir(): - default_text = self._read_prompts_from_dir(default_dir) - if default_text: - self.prompts[prompt_id][DEFAULT] = default_text - - # 3. Load Custom Prompts (sub-directories) - for item in category_dir.iterdir(): - if item.is_dir() and item.name not in [COMMON, DEFAULT]: - custom_type = item.name.lower() - custom_text = self._read_prompts_from_dir(item) - if custom_text: - self.prompts[prompt_id][custom_type] = custom_text - - logger.info(f"Loaded prompts from {self.resources_dir}") - - except Exception as e: - logger.error(f"Failed to load prompts from {self.resources_dir}: {e}") - - def _read_prompts_from_dir(self, directory: Path | Traversable) -> str: - """Read and concatenate all .md files in a directory.""" - prompts = [] - # Use iterdir instead of glob for Traversable compatibility - files = sorted( - [f for f in directory.iterdir() if f.name.endswith(".md")], - key=lambda x: x.name, - ) - for file_path in files: - if file_path.is_file(): - prompts.append(file_path.read_text(encoding="utf-8").strip()) - return "\n\n".join(prompts) + """Provides default prompts used when no user override is configured.""" + + def __init__(self) -> None: + self.prompts: dict[str, dict[str, str]] = { + PromptId.OCR_TRANSCRIPTION.value: { + DEFAULT: DEFAULT_OCR_PROMPT, + }, + PromptId.SUMMARY_GENERATION.value: { + COMMON: DEFAULT_SUMMARY_COMMON_PROMPT, + DEFAULT: DEFAULT_SUMMARY_PROMPT, + }, + } + + def get_all_known_layers(self) -> dict[str, dict[str, str]]: + """Return all default layers keyed by prompt_id value then layer name.""" + return {pid: dict(layers) for pid, layers in self.prompts.items()} def get_prompt(self, prompt_id: PromptId, custom_type: str | None = None) -> str: - """Retrieve a prompt by its ID, optionally overridden by a custom type. + """Return the composed default prompt for the given prompt_id. - Logic: Common + (Custom if exists else Default) + Composes common (if present) + default. custom_type is accepted for API + compatibility but ignored — custom prompts are managed via + PromptConfigService (DB overrides), not the loader. """ - if prompt_id not in self.prompts: + key = prompt_id.value if isinstance(prompt_id, PromptId) else str(prompt_id) + type_map = self.prompts.get(key) + if not type_map: raise ValueError(f"Prompt ID '{prompt_id}' not found.") - - type_map = self.prompts[prompt_id] - parts = [] - - # 1. Add Common if COMMON in type_map: parts.append(type_map[COMMON]) - - # 2. Add Specific (Custom or Default) - specific_prompt = None - if custom_type and custom_type in type_map: - logger.info(f"Using custom prompt '{custom_type}' for {prompt_id}") - specific_prompt = type_map[custom_type] - elif DEFAULT in type_map: - specific_prompt = type_map[DEFAULT] - - if specific_prompt: - parts.append(specific_prompt) - elif not parts: - # If no common and no specific/default, that's an issue - raise ValueError( - f"No prompt content found for '{prompt_id}' (Custom: {custom_type})" - ) - + if DEFAULT in type_map: + parts.append(type_map[DEFAULT]) return "\n\n".join(parts) diff --git a/tests/models/test_prompt_config_completeness.py b/tests/models/test_prompt_config_completeness.py new file mode 100644 index 00000000..b7fabe11 --- /dev/null +++ b/tests/models/test_prompt_config_completeness.py @@ -0,0 +1,142 @@ +"""Tests for PromptConfig data models — round-trip serialisation.""" + +from supernote.models.prompt_config import ( + FileStalenessResponseVO, + GetPromptsResponseVO, + PageStalenessDTO, + PromptConfigDTO, + ReprocessRequestDTO, + ReprocessResponseVO, + UpsertPromptConfigDTO, +) + + +def test_prompt_config_dto_override() -> None: + dto = PromptConfigDTO( + category="ocr", + layer="common", + content="Do OCR.", + is_override=True, + default_content="Default OCR.", + ) + d = dto.to_dict() + assert d["category"] == "ocr" + assert d["layer"] == "common" + assert d["content"] == "Do OCR." + assert d["isOverride"] is True + assert d["defaultContent"] == "Default OCR." + + +def test_prompt_config_dto_no_override() -> None: + dto = PromptConfigDTO( + category="summary", + layer="default", + content="Summarise.", + is_override=False, + default_content="Summarise.", + ) + d = dto.to_dict() + assert d["isOverride"] is False + assert d["defaultContent"] == "Summarise." + + +def test_upsert_prompt_config_dto() -> None: + dto = UpsertPromptConfigDTO( + category="summary", + layer="monthly", + content="Monthly summary prompt.", + ) + d = dto.to_dict() + assert d["category"] == "summary" + assert d["layer"] == "monthly" + assert d["content"] == "Monthly summary prompt." + + +def test_get_prompts_response_vo() -> None: + prompts = [ + PromptConfigDTO( + category="ocr", + layer="default", + content="OCR default.", + is_override=False, + default_content="OCR default.", + ) + ] + vo = GetPromptsResponseVO(success=True, prompts=prompts) + d = vo.to_dict() + assert d["success"] is True + assert len(d["prompts"]) == 1 + assert d["prompts"][0]["category"] == "ocr" + + +def test_get_prompts_response_vo_empty() -> None: + vo = GetPromptsResponseVO(success=True) + d = vo.to_dict() + assert d["prompts"] == [] + + +def test_page_staleness_dto_stale() -> None: + dto = PageStalenessDTO( + page_id="P20231027120000abc", + page_index=0, + stored_hash=None, + is_stale=True, + ) + d = dto.to_dict() + assert d["pageId"] == "P20231027120000abc" + assert d["pageIndex"] == 0 + assert d["isStale"] is True + assert "storedHash" not in d # omit_none + + +def test_page_staleness_dto_not_stale() -> None: + dto = PageStalenessDTO( + page_id="P20231028090000xyz", + page_index=1, + stored_hash="a3f1e9c2", + is_stale=False, + ) + d = dto.to_dict() + assert d["storedHash"] == "a3f1e9c2" + assert d["isStale"] is False + + +def test_file_staleness_response_vo() -> None: + pages = [ + PageStalenessDTO(page_id="P1", page_index=0, stored_hash=None, is_stale=True), + PageStalenessDTO( + page_id="P2", page_index=1, stored_hash="abc123", is_stale=False + ), + ] + vo = FileStalenessResponseVO( + success=True, + current_prompt_hash="def456", + pages=pages, + stale_count=1, + total_count=2, + ) + d = vo.to_dict() + assert d["success"] is True + assert d["currentPromptHash"] == "def456" + assert d["staleCount"] == 1 + assert d["totalCount"] == 2 + assert len(d["pages"]) == 2 + + +def test_reprocess_request_dto_with_page_ids() -> None: + dto = ReprocessRequestDTO(page_ids=["P1", "P2"]) + d = dto.to_dict() + assert d["pageIds"] == ["P1", "P2"] + + +def test_reprocess_request_dto_none() -> None: + dto = ReprocessRequestDTO() + d = dto.to_dict() + assert "pageIds" not in d # omit_none + + +def test_reprocess_response_vo() -> None: + vo = ReprocessResponseVO(success=True, queued_page_count=3) + d = vo.to_dict() + assert d["success"] is True + assert d["queuedPageCount"] == 3 diff --git a/tests/server/routes/test_prompts.py b/tests/server/routes/test_prompts.py new file mode 100644 index 00000000..3605afb6 --- /dev/null +++ b/tests/server/routes/test_prompts.py @@ -0,0 +1,385 @@ +"""Tests for prompt configuration routes.""" + +from aiohttp.test_utils import TestClient +from sqlalchemy import select + +from supernote.server.db.models.file import UserFileDO +from supernote.server.db.models.note_processing import NotePageContentDO +from supernote.server.db.models.user import UserDO +from supernote.server.db.session import DatabaseSessionManager + +TEST_USER_EMAIL = "test@example.com" + + +# --------------------------------------------------------------------------- +# Helper +# --------------------------------------------------------------------------- + + +async def _get_user_id(session_manager: DatabaseSessionManager) -> int: + """Get the numeric user_id for the test user from the DB.""" + async with session_manager.session() as session: + result = await session.execute( + select(UserDO).where(UserDO.email == TEST_USER_EMAIL) + ) + user = result.scalar_one() + return user.id + + +# --------------------------------------------------------------------------- +# Authentication guard +# --------------------------------------------------------------------------- + + +async def test_get_prompts_unauthenticated(client: TestClient) -> None: + """GET /api/extended/prompts returns 401 without auth.""" + resp = await client.get("/api/extended/prompts") + assert resp.status == 401 + + +async def test_put_prompt_unauthenticated(client: TestClient) -> None: + """PUT /api/extended/prompts returns 401 without auth.""" + resp = await client.put( + "/api/extended/prompts", + json={"category": "ocr", "layer": "default", "content": "test"}, + ) + assert resp.status == 401 + + +async def test_delete_prompt_unauthenticated(client: TestClient) -> None: + """DELETE /api/extended/prompts/{category}/{layer} returns 401 without auth.""" + resp = await client.delete("/api/extended/prompts/ocr/default") + assert resp.status == 401 + + +# --------------------------------------------------------------------------- +# GET /api/extended/prompts +# --------------------------------------------------------------------------- + + +async def test_get_prompts_returns_all_layers( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """GET /api/extended/prompts returns all known layers.""" + resp = await client.get("/api/extended/prompts", headers=auth_headers) + assert resp.status == 200 + data = await resp.json() + assert data["success"] is True + assert isinstance(data["prompts"], list) + assert len(data["prompts"]) > 0 + + categories = {p["category"] for p in data["prompts"]} + assert "ocr" in categories + assert "summary" in categories + + +async def test_get_prompts_no_overrides_by_default( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """GET /api/extended/prompts shows isOverride=False for new users.""" + resp = await client.get("/api/extended/prompts", headers=auth_headers) + data = await resp.json() + for prompt in data["prompts"]: + assert prompt["isOverride"] is False + + +# --------------------------------------------------------------------------- +# PUT /api/extended/prompts +# --------------------------------------------------------------------------- + + +async def test_put_prompt_creates_override( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """PUT /api/extended/prompts saves an override.""" + resp = await client.put( + "/api/extended/prompts", + headers=auth_headers, + json={ + "category": "summary", + "layer": "monthly", + "content": "Monthly summary text.", + }, + ) + assert resp.status == 200 + data = await resp.json() + assert data["success"] is True + + # Now GET should show isOverride=True for summary/monthly + resp2 = await client.get("/api/extended/prompts", headers=auth_headers) + prompts = (await resp2.json())["prompts"] + monthly_summary = next( + (p for p in prompts if p["category"] == "summary" and p["layer"] == "monthly"), + None, + ) + assert monthly_summary is not None + assert monthly_summary["isOverride"] is True + assert monthly_summary["content"] == "Monthly summary text." + + +async def test_put_prompt_invalid_category( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """PUT /api/extended/prompts with invalid category returns 400.""" + resp = await client.put( + "/api/extended/prompts", + headers=auth_headers, + json={"category": "invalid", "layer": "default", "content": "text"}, + ) + assert resp.status == 400 + data = await resp.json() + assert data["success"] is False + + +async def test_put_prompt_empty_content( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """PUT /api/extended/prompts with empty content returns 400.""" + resp = await client.put( + "/api/extended/prompts", + headers=auth_headers, + json={"category": "ocr", "layer": "default", "content": " "}, + ) + assert resp.status == 400 + data = await resp.json() + assert data["success"] is False + + +# --------------------------------------------------------------------------- +# DELETE /api/extended/prompts/{category}/{layer} +# --------------------------------------------------------------------------- + + +async def test_delete_prompt_existing_override( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """DELETE removes an existing override.""" + # First save one + await client.put( + "/api/extended/prompts", + headers=auth_headers, + json={"category": "ocr", "layer": "common", "content": "Override text"}, + ) + + resp = await client.delete("/api/extended/prompts/ocr/common", headers=auth_headers) + assert resp.status == 200 + data = await resp.json() + assert data["success"] is True + + +async def test_delete_prompt_not_found( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """DELETE returns 404 when no override exists.""" + resp = await client.delete( + "/api/extended/prompts/ocr/nonexistent-layer", headers=auth_headers + ) + assert resp.status == 404 + data = await resp.json() + assert data["success"] is False + + +# --------------------------------------------------------------------------- +# GET /api/extended/files/{file_id}/staleness +# --------------------------------------------------------------------------- + + +async def test_staleness_requires_auth(client: TestClient) -> None: + """Staleness endpoint returns 401 without auth.""" + resp = await client.get("/api/extended/files/1/staleness") + assert resp.status == 401 + + +async def test_staleness_other_user_file_returns_403( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, +) -> None: + """Staleness endpoint returns 403 for files owned by another user.""" + # Seed a file owned by user_id=9999 (not the test user) + async with session_manager.session() as session: + file_do = UserFileDO( + id=5001, + user_id=9999, + file_name="other.note", + is_folder="N", + ) + session.add(file_do) + await session.commit() + + resp = await client.get("/api/extended/files/5001/staleness", headers=auth_headers) + assert resp.status == 403 + + +async def test_staleness_returns_per_page_status( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """Staleness endpoint returns per-page staleness data.""" + user_id = await _get_user_id(session_manager) + + async with session_manager.session() as session: + file_do = UserFileDO( + id=6001, + user_id=user_id, + file_name="monthly.note", + is_folder="N", + ) + session.add(file_do) + page = NotePageContentDO( + file_id=6001, + page_index=0, + page_id="P20231027120000abc", + prompt_hash=None, # pre-feature, treated as stale + ) + session.add(page) + await session.commit() + + resp = await client.get("/api/extended/files/6001/staleness", headers=auth_headers) + assert resp.status == 200 + data = await resp.json() + assert data["success"] is True + assert data["totalCount"] == 1 + assert data["staleCount"] == 1 + assert data["pages"][0]["isStale"] is True + assert data["pages"][0]["pageId"] == "P20231027120000abc" + assert "currentPromptHash" in data + + +# --------------------------------------------------------------------------- +# POST /api/extended/files/{file_id}/reprocess +# --------------------------------------------------------------------------- + + +async def test_reprocess_requires_auth(client: TestClient) -> None: + """Reprocess endpoint returns 401 without auth.""" + resp = await client.post("/api/extended/files/1/reprocess") + assert resp.status == 401 + + +async def test_reprocess_other_user_file_returns_403( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, +) -> None: + """Reprocess endpoint returns 403 for files owned by another user.""" + async with session_manager.session() as session: + file_do = UserFileDO( + id=5002, + user_id=9999, + file_name="other.note", + is_folder="N", + ) + session.add(file_do) + await session.commit() + + resp = await client.post("/api/extended/files/5002/reprocess", headers=auth_headers) + assert resp.status == 403 + + +async def test_reprocess_file_no_stale_pages( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """Reprocess with no stale pages returns queued_page_count=0.""" + user_id = await _get_user_id(session_manager) + + # Compute what the current hash would be and stamp the page with it + # (by first calling staleness to get the hash, then updating the row) + async with session_manager.session() as session: + file_do = UserFileDO( + id=7001, + user_id=user_id, + file_name="monthly.note", + is_folder="N", + ) + session.add(file_do) + await session.commit() + + # Get current hash + stale_resp = await client.get( + "/api/extended/files/7001/staleness", headers=auth_headers + ) + current_hash = (await stale_resp.json())["currentPromptHash"] + + # Seed a page with matching hash + async with session_manager.session() as session: + page = NotePageContentDO( + file_id=7001, + page_index=0, + page_id="P001", + prompt_hash=current_hash, + ) + session.add(page) + await session.commit() + + resp = await client.post("/api/extended/files/7001/reprocess", headers=auth_headers) + assert resp.status == 200 + data = await resp.json() + assert data["success"] is True + assert data["queuedPageCount"] == 0 + + +# --------------------------------------------------------------------------- +# POST /api/extended/files/{file_id}/pages/{page_id}/reprocess +# --------------------------------------------------------------------------- + + +async def test_reprocess_page_requires_auth(client: TestClient) -> None: + """Page reprocess endpoint returns 401 without auth.""" + resp = await client.post("/api/extended/files/1/pages/P001/reprocess") + assert resp.status == 401 + + +async def test_reprocess_page_not_stale_returns_400( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """Page reprocess returns 400 when the page is not stale.""" + user_id = await _get_user_id(session_manager) + + async with session_manager.session() as session: + file_do = UserFileDO( + id=8001, + user_id=user_id, + file_name="monthly.note", + is_folder="N", + ) + session.add(file_do) + await session.commit() + + # Get current hash for a fresh file + stale_resp = await client.get( + "/api/extended/files/8001/staleness", headers=auth_headers + ) + current_hash = (await stale_resp.json())["currentPromptHash"] + + async with session_manager.session() as session: + page = NotePageContentDO( + file_id=8001, + page_index=0, + page_id="P8001", + prompt_hash=current_hash, # up-to-date + ) + session.add(page) + await session.commit() + + resp = await client.post( + "/api/extended/files/8001/pages/P8001/reprocess", headers=auth_headers + ) + assert resp.status == 400 + data = await resp.json() + assert data["success"] is False diff --git a/tests/server/services/test_processor_modules.py b/tests/server/services/test_processor_modules.py index d2e9f987..f87617af 100644 --- a/tests/server/services/test_processor_modules.py +++ b/tests/server/services/test_processor_modules.py @@ -69,13 +69,41 @@ async def test_explicit_orchestration_flow( hashing.run.assert_called_once_with(file_id, sm_mock) # Per-Page Pipeline (Parallel across pages) - png.run.assert_any_call(file_id, sm_mock, page_index=0, page_id="p0") - png.run.assert_any_call(file_id, sm_mock, page_index=1, page_id="p1") - ocr.run.assert_any_call(file_id, sm_mock, page_index=0, page_id="p0") - embedding.run.assert_any_call(file_id, sm_mock, page_index=0, page_id="p0") + png.run.assert_any_call( + file_id, + sm_mock, + page_index=0, + page_id="p0", + prompt_resolver=None, + prompt_hash=None, + ) + png.run.assert_any_call( + file_id, + sm_mock, + page_index=1, + page_id="p1", + prompt_resolver=None, + prompt_hash=None, + ) + ocr.run.assert_any_call( + file_id, + sm_mock, + page_index=0, + page_id="p0", + prompt_resolver=None, + prompt_hash=None, + ) + embedding.run.assert_any_call( + file_id, + sm_mock, + page_index=0, + page_id="p0", + prompt_resolver=None, + prompt_hash=None, + ) # Summary (Global) runs last - summary.run.assert_called_once_with(file_id, sm_mock) + summary.run.assert_called_once_with(file_id, sm_mock, prompt_resolver=None) async def test_dependant_skipping( @@ -121,11 +149,18 @@ async def test_dependant_skipping( await processor_service.process_file(file_id) # Verify - png.run.assert_called_once_with(file_id, sm_mock, page_index=0, page_id="p0") + png.run.assert_called_once_with( + file_id, + sm_mock, + page_index=0, + page_id="p0", + prompt_resolver=None, + prompt_hash=None, + ) # OCR and Embedding should NOT be checked because PNG returned False ocr.run.assert_not_called() embedding.run.assert_not_called() # Summary (Global) should still run - summary.run.assert_called_once_with(file_id, sm_mock) + summary.run.assert_called_once_with(file_id, sm_mock, prompt_resolver=None) diff --git a/tests/server/services/test_processor_prompt_hash.py b/tests/server/services/test_processor_prompt_hash.py new file mode 100644 index 00000000..03ed0fce --- /dev/null +++ b/tests/server/services/test_processor_prompt_hash.py @@ -0,0 +1,313 @@ +"""Tests for prompt-aware processing and reprocess functionality.""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest +from sqlalchemy import select + +from supernote.models.base import ProcessingStatus +from supernote.server.db.models.file import UserFileDO +from supernote.server.db.models.note_processing import NotePageContentDO, SystemTaskDO +from supernote.server.db.session import DatabaseSessionManager +from supernote.server.services.processor import ProcessorService +from supernote.server.services.processor_modules.ocr import OcrModule +from supernote.server.services.processor_modules.summary import SummaryModule +from supernote.server.services.prompt_config import PromptConfigService +from supernote.server.utils.prompt_loader import PromptId, PromptLoader + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def prompt_loader() -> PromptLoader: + return PromptLoader() + + +@pytest.fixture +def prompt_config_service( + session_manager: DatabaseSessionManager, prompt_loader: PromptLoader +) -> PromptConfigService: + return PromptConfigService(session_manager, prompt_loader) + + +# --------------------------------------------------------------------------- +# OCR Module: prompt_hash written to DB +# --------------------------------------------------------------------------- + + +async def _seed_file_and_page( + session_manager: DatabaseSessionManager, + file_id: int, + page_id: str, + file_name: str = "monthly.note", +) -> None: + """Seed minimal UserFileDO and NotePageContentDO rows.""" + async with session_manager.session() as session: + file_do = UserFileDO( + id=file_id, + user_id=1, + file_name=file_name, + is_folder="N", + ) + content_do = NotePageContentDO( + file_id=file_id, + page_index=0, + page_id=page_id, + ) + session.add(file_do) + session.add(content_do) + await session.commit() + + +async def test_ocr_module_writes_prompt_hash( + session_manager: DatabaseSessionManager, +) -> None: + """OCR module writes prompt_hash to NotePageContentDO when provided.""" + file_id = 1 + page_id = "P001" + await _seed_file_and_page(session_manager, file_id, page_id) + + ai_service = MagicMock() + ai_service.is_configured = True + ai_service.ocr_image = AsyncMock(return_value="Extracted text") + + file_service = MagicMock() + blob_storage = MagicMock() + blob_storage.exists = AsyncMock(return_value=True) + blob_storage.get = MagicMock(return_value=aiter_bytes(b"png_data")) + file_service.blob_storage = blob_storage + + ocr_module = OcrModule(file_service=file_service, ai_service=ai_service) + + async def mock_prompt_resolver( + prompt_id: PromptId, custom_type: str | None = None + ) -> str: + return "Custom OCR prompt" + + test_hash = "abc123deadbeef" + + await ocr_module.process( + file_id=file_id, + session_manager=session_manager, + page_index=0, + page_id=page_id, + prompt_resolver=mock_prompt_resolver, + prompt_hash=test_hash, + ) + + # Verify hash was written to DB + async with session_manager.session() as session: + stmt = select(NotePageContentDO).where( + NotePageContentDO.file_id == file_id, + NotePageContentDO.page_id == page_id, + ) + result = await session.execute(stmt) + row = result.scalar_one() + assert row.prompt_hash == test_hash + + +async def test_ocr_module_no_hash_when_not_provided( + session_manager: DatabaseSessionManager, +) -> None: + """OCR module leaves prompt_hash as None when not provided.""" + file_id = 2 + page_id = "P002" + await _seed_file_and_page(session_manager, file_id, page_id) + + ai_service = MagicMock() + ai_service.is_configured = True + ai_service.ocr_image = AsyncMock(return_value="Extracted text") + + file_service = MagicMock() + blob_storage = MagicMock() + blob_storage.exists = AsyncMock(return_value=True) + blob_storage.get = MagicMock(return_value=aiter_bytes(b"png_data")) + file_service.blob_storage = blob_storage + + ocr_module = OcrModule(file_service=file_service, ai_service=ai_service) + + # No prompt_resolver or prompt_hash passed + await ocr_module.process( + file_id=file_id, + session_manager=session_manager, + page_index=0, + page_id=page_id, + ) + + async with session_manager.session() as session: + stmt = select(NotePageContentDO).where( + NotePageContentDO.file_id == file_id, + NotePageContentDO.page_id == page_id, + ) + result = await session.execute(stmt) + row = result.scalar_one() + assert row.prompt_hash is None + + +# --------------------------------------------------------------------------- +# ProcessorService: passes prompt_resolver + prompt_hash to modules +# --------------------------------------------------------------------------- + + +async def test_processor_service_passes_resolver_to_modules( + prompt_config_service: PromptConfigService, +) -> None: + """ProcessorService passes prompt_resolver and prompt_hash when prompt_config_service set.""" + from supernote.server.services.processor_modules.embedding import EmbeddingModule + from supernote.server.services.processor_modules.page_hashing import ( + PageHashingModule, + ) + from supernote.server.services.processor_modules.png_conversion import ( + PngConversionModule, + ) + + event_bus = MagicMock() + session_manager = MagicMock() + file_service = MagicMock() + summary_service = MagicMock() + + # Mock the file lookup inside process_file + file_do = MagicMock() + file_do.user_id = 42 + file_do.file_name = "monthly.note" + + mock_session = AsyncMock() + mock_session.get.return_value = file_do + mock_result = MagicMock() + mock_result.all.return_value = [(0, "p0")] + mock_session.execute.return_value = mock_result + + session_manager.session.return_value.__aenter__.return_value = mock_session + + processor = ProcessorService( + event_bus=event_bus, + session_manager=session_manager, + file_service=file_service, + summary_service=summary_service, + prompt_config_service=prompt_config_service, + ) + + hashing = MagicMock(spec=PageHashingModule) + hashing.run = AsyncMock(return_value=True) + png = MagicMock(spec=PngConversionModule) + png.run = AsyncMock(return_value=True) + ocr = MagicMock(spec=OcrModule) + ocr.run = AsyncMock(return_value=True) + embedding = MagicMock(spec=EmbeddingModule) + embedding.run = AsyncMock(return_value=True) + summary = MagicMock(spec=SummaryModule) + summary.run = AsyncMock(return_value=True) + + processor.register_modules( + hashing=hashing, png=png, ocr=ocr, embedding=embedding, summary=summary + ) + + await processor.process_file(file_id=99) + + # OCR should have been called with prompt_resolver and prompt_hash kwargs + ocr_call_kwargs = ocr.run.call_args.kwargs + assert "prompt_resolver" in ocr_call_kwargs + assert "prompt_hash" in ocr_call_kwargs + assert ocr_call_kwargs["prompt_hash"] is not None + assert len(ocr_call_kwargs["prompt_hash"]) == 64 # sha256 hex + + +# --------------------------------------------------------------------------- +# reprocess_pages +# --------------------------------------------------------------------------- + + +async def test_reprocess_pages_resets_tasks( + session_manager: DatabaseSessionManager, + prompt_config_service: PromptConfigService, +) -> None: + """reprocess_pages resets SystemTaskDO status to PENDING for specified pages.""" + file_id = 10 + page_id = "P010" + + # Seed completed OCR and embedding tasks for the page + async with session_manager.session() as session: + ocr_task = SystemTaskDO( + file_id=file_id, + task_type="OCR_EXTRACTION", + key=page_id, + status=ProcessingStatus.COMPLETED, + ) + embed_task = SystemTaskDO( + file_id=file_id, + task_type="EMBEDDING_GENERATION", + key=page_id, + status=ProcessingStatus.COMPLETED, + ) + summary_task = SystemTaskDO( + file_id=file_id, + task_type="SUMMARY_GENERATION", + key="global", + status=ProcessingStatus.COMPLETED, + ) + session.add_all([ocr_task, embed_task, summary_task]) + await session.commit() + + event_bus = MagicMock() + file_service = MagicMock() + summary_service = MagicMock() + processor = ProcessorService( + event_bus=event_bus, + session_manager=session_manager, + file_service=file_service, + summary_service=summary_service, + prompt_config_service=prompt_config_service, + ) + + count = await processor.reprocess_pages(file_id=file_id, page_ids=[page_id]) + assert count == 1 + + # Check that task statuses were reset + async with session_manager.session() as session: + for task_type, key in [ + ("OCR_EXTRACTION", page_id), + ("EMBEDDING_GENERATION", page_id), + ("SUMMARY_GENERATION", "global"), + ]: + stmt = select(SystemTaskDO).where( + SystemTaskDO.file_id == file_id, + SystemTaskDO.task_type == task_type, + SystemTaskDO.key == key, + ) + result = await session.execute(stmt) + task = result.scalar_one() + assert task.status == ProcessingStatus.PENDING, ( + f"{task_type}/{key} expected PENDING got {task.status}" + ) + + +async def test_reprocess_pages_empty_list_does_nothing( + session_manager: DatabaseSessionManager, + prompt_config_service: PromptConfigService, +) -> None: + """reprocess_pages with empty list returns 0.""" + event_bus = MagicMock() + processor = ProcessorService( + event_bus=event_bus, + session_manager=session_manager, + file_service=MagicMock(), + summary_service=MagicMock(), + prompt_config_service=prompt_config_service, + ) + count = await processor.reprocess_pages(file_id=99, page_ids=[]) + assert count == 0 + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +async def aiter_bytes_gen(data: bytes): # type: ignore[no-untyped-def] + yield data + + +def aiter_bytes(data: bytes): # type: ignore[no-untyped-def] + return aiter_bytes_gen(data) diff --git a/tests/server/services/test_prompt_config_service.py b/tests/server/services/test_prompt_config_service.py new file mode 100644 index 00000000..cfe94187 --- /dev/null +++ b/tests/server/services/test_prompt_config_service.py @@ -0,0 +1,324 @@ +"""Tests for PromptConfigService — CRUD, prompt resolution, hash computation.""" + +import pytest + +from supernote.server.db.session import DatabaseSessionManager +from supernote.server.services.prompt_config import NotFoundError, PromptConfigService +from supernote.server.utils.prompt_loader import ( + DEFAULT_OCR_PROMPT, + DEFAULT_SUMMARY_COMMON_PROMPT, + DEFAULT_SUMMARY_PROMPT, + PromptId, + PromptLoader, +) + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def prompt_loader() -> PromptLoader: + return PromptLoader() + + +@pytest.fixture +def service( + session_manager: DatabaseSessionManager, prompt_loader: PromptLoader +) -> PromptConfigService: + """Create a PromptConfigService with in-process DB.""" + return PromptConfigService(session_manager, prompt_loader) + + +# --------------------------------------------------------------------------- +# list_configs +# --------------------------------------------------------------------------- + + +async def test_list_configs_empty(service: PromptConfigService) -> None: + """New user has no configs.""" + configs = await service.list_configs(user_id=1) + assert configs == [] + + +async def test_list_configs_after_upsert(service: PromptConfigService) -> None: + """Configs appear after upsert.""" + await service.upsert_config( + user_id=1, category="ocr", layer="common", content="My OCR" + ) + configs = await service.list_configs(user_id=1) + assert len(configs) == 1 + assert configs[0].category == "ocr" + assert configs[0].layer == "common" + assert configs[0].content == "My OCR" + + +async def test_list_configs_user_isolation(service: PromptConfigService) -> None: + """Each user sees only their own configs.""" + await service.upsert_config( + user_id=1, category="ocr", layer="default", content="User1 OCR" + ) + await service.upsert_config( + user_id=2, category="ocr", layer="default", content="User2 OCR" + ) + user1_configs = await service.list_configs(user_id=1) + user2_configs = await service.list_configs(user_id=2) + assert len(user1_configs) == 1 + assert len(user2_configs) == 1 + assert user1_configs[0].content == "User1 OCR" + assert user2_configs[0].content == "User2 OCR" + + +# --------------------------------------------------------------------------- +# upsert_config +# --------------------------------------------------------------------------- + + +async def test_upsert_config_creates_row(service: PromptConfigService) -> None: + """Upsert creates a new row when none exists.""" + row = await service.upsert_config( + user_id=1, category="summary", layer="monthly", content="Monthly prompt" + ) + assert row.user_id == 1 + assert row.category == "summary" + assert row.layer == "monthly" + assert row.content == "Monthly prompt" + + +async def test_upsert_config_updates_row(service: PromptConfigService) -> None: + """Upsert updates the row when one exists.""" + await service.upsert_config( + user_id=1, category="ocr", layer="common", content="First" + ) + updated = await service.upsert_config( + user_id=1, category="ocr", layer="common", content="Second" + ) + assert updated.content == "Second" + all_configs = await service.list_configs(user_id=1) + assert len(all_configs) == 1 + + +async def test_upsert_config_invalid_category(service: PromptConfigService) -> None: + """Upsert raises ValueError for unknown category.""" + with pytest.raises(ValueError, match="category"): + await service.upsert_config( + user_id=1, category="invalid", layer="default", content="Some text" + ) + + +async def test_upsert_config_empty_content(service: PromptConfigService) -> None: + """Upsert raises ValueError for empty/whitespace content.""" + with pytest.raises(ValueError, match="content"): + await service.upsert_config( + user_id=1, category="ocr", layer="default", content=" " + ) + + +async def test_upsert_config_invalid_layer(service: PromptConfigService) -> None: + """Upsert raises ValueError for invalid layer name.""" + with pytest.raises(ValueError, match="layer"): + await service.upsert_config( + user_id=1, category="ocr", layer="invalid layer!", content="Some text" + ) + + +# --------------------------------------------------------------------------- +# delete_config +# --------------------------------------------------------------------------- + + +async def test_delete_config_existing(service: PromptConfigService) -> None: + """Delete removes the row.""" + await service.upsert_config( + user_id=1, category="ocr", layer="monthly", content="Monthly OCR" + ) + await service.delete_config(user_id=1, category="ocr", layer="monthly") + configs = await service.list_configs(user_id=1) + assert configs == [] + + +async def test_delete_config_not_found(service: PromptConfigService) -> None: + """Delete raises NotFoundError when no DB row exists for that layer.""" + with pytest.raises(NotFoundError): + await service.delete_config(user_id=1, category="ocr", layer="nonexistent") + + +async def test_delete_config_protected_layer_raises( + service: PromptConfigService, +) -> None: + """Protected layers cannot be deleted even when a user override exists.""" + await service.upsert_config( + user_id=1, category="ocr", layer="default", content="custom" + ) + with pytest.raises(ValueError, match="protected"): + await service.delete_config(user_id=1, category="ocr", layer="default") + + with pytest.raises(ValueError, match="protected"): + await service.delete_config(user_id=1, category="summary", layer="common") + + +async def test_delete_config_user_isolation(service: PromptConfigService) -> None: + """Delete only removes the specified user's config.""" + await service.upsert_config( + user_id=1, category="ocr", layer="project", content="User1" + ) + await service.upsert_config( + user_id=2, category="ocr", layer="project", content="User2" + ) + await service.delete_config(user_id=1, category="ocr", layer="project") + assert await service.list_configs(user_id=1) == [] + assert len(await service.list_configs(user_id=2)) == 1 + + +# --------------------------------------------------------------------------- +# get_effective_prompt +# --------------------------------------------------------------------------- + + +async def test_get_effective_prompt_ocr_falls_back_to_loader( + service: PromptConfigService, +) -> None: + """With no user override, OCR returns the hardcoded default.""" + prompt = await service.get_effective_prompt( + user_id=1, prompt_id=PromptId.OCR_TRANSCRIPTION, note_type=None + ) + assert prompt == DEFAULT_OCR_PROMPT + + +async def test_get_effective_prompt_summary_composes_common_and_default( + service: PromptConfigService, +) -> None: + """With no user overrides, summary composes common + default.""" + prompt = await service.get_effective_prompt( + user_id=1, prompt_id=PromptId.SUMMARY_GENERATION, note_type=None + ) + assert DEFAULT_SUMMARY_COMMON_PROMPT in prompt + assert DEFAULT_SUMMARY_PROMPT in prompt + + +async def test_get_effective_prompt_uses_user_default_override( + service: PromptConfigService, +) -> None: + """User override for 'default' replaces the hardcoded default.""" + await service.upsert_config( + user_id=1, category="ocr", layer="default", content="My custom OCR" + ) + prompt = await service.get_effective_prompt( + user_id=1, prompt_id=PromptId.OCR_TRANSCRIPTION, note_type=None + ) + assert prompt == "My custom OCR" + + +async def test_get_effective_prompt_note_type_override_wins( + service: PromptConfigService, +) -> None: + """User override for a specific note type takes priority over 'default'.""" + await service.upsert_config( + user_id=1, category="ocr", layer="default", content="My default OCR" + ) + await service.upsert_config( + user_id=1, category="ocr", layer="monthly", content="My monthly OCR" + ) + prompt = await service.get_effective_prompt( + user_id=1, prompt_id=PromptId.OCR_TRANSCRIPTION, note_type="monthly" + ) + assert prompt == "My monthly OCR" + + +async def test_get_effective_prompt_note_type_falls_back_to_default_override( + service: PromptConfigService, +) -> None: + """When no type-specific override exists, uses the 'default' override.""" + await service.upsert_config( + user_id=1, category="ocr", layer="default", content="My default OCR" + ) + prompt = await service.get_effective_prompt( + user_id=1, prompt_id=PromptId.OCR_TRANSCRIPTION, note_type="weekly" + ) + assert prompt == "My default OCR" + + +# --------------------------------------------------------------------------- +# compute_combined_prompt_hash +# --------------------------------------------------------------------------- + + +async def test_compute_combined_prompt_hash_returns_hex( + service: PromptConfigService, +) -> None: + """compute_combined_prompt_hash returns a 64-char hex string.""" + h = await service.compute_combined_prompt_hash(user_id=1, note_type=None) + assert len(h) == 64 + assert all(c in "0123456789abcdef" for c in h) + + +async def test_compute_combined_prompt_hash_changes_on_override( + service: PromptConfigService, +) -> None: + """Hash changes when a user override is added.""" + h1 = await service.compute_combined_prompt_hash(user_id=1, note_type=None) + await service.upsert_config( + user_id=1, category="ocr", layer="default", content="Changed OCR" + ) + h2 = await service.compute_combined_prompt_hash(user_id=1, note_type=None) + assert h1 != h2 + + +async def test_compute_combined_prompt_hash_user_isolation( + service: PromptConfigService, +) -> None: + """Hash is per-user; different user overrides produce different hashes.""" + h_baseline = await service.compute_combined_prompt_hash(user_id=1, note_type=None) + await service.upsert_config( + user_id=2, category="ocr", layer="default", content="User2 OCR" + ) + h_user1 = await service.compute_combined_prompt_hash(user_id=1, note_type=None) + h_user2 = await service.compute_combined_prompt_hash(user_id=2, note_type=None) + assert h_user1 == h_baseline + assert h_user2 != h_baseline + + +# --------------------------------------------------------------------------- +# get_all_configs_with_defaults +# --------------------------------------------------------------------------- + + +async def test_get_all_configs_with_defaults_includes_all_layers( + service: PromptConfigService, +) -> None: + """Returns both default layers from the loader (no user overrides).""" + configs = await service.get_all_configs_with_defaults(user_id=1) + categories = {c.category for c in configs} + assert "ocr" in categories + assert "summary" in categories + layers = {c.layer for c in configs if c.category == "ocr"} + assert "default" in layers + + +async def test_get_all_configs_with_defaults_marks_overrides( + service: PromptConfigService, +) -> None: + """is_override is True only for layers with user DB rows.""" + await service.upsert_config( + user_id=1, category="ocr", layer="default", content="My custom OCR" + ) + configs = await service.get_all_configs_with_defaults(user_id=1) + override_configs = [c for c in configs if c.is_override] + non_override_configs = [c for c in configs if not c.is_override] + assert len(override_configs) == 1 + assert override_configs[0].category == "ocr" + assert override_configs[0].layer == "default" + assert override_configs[0].content == "My custom OCR" + assert all(c.default_content for c in non_override_configs) + + +async def test_get_all_configs_with_defaults_default_content_always_present( + service: PromptConfigService, +) -> None: + """default_content always holds the hardcoded default text.""" + await service.upsert_config( + user_id=1, category="ocr", layer="default", content="Override text" + ) + configs = await service.get_all_configs_with_defaults(user_id=1) + for c in configs: + assert c.default_content, f"Missing default_content for {c.category}/{c.layer}" diff --git a/tests/server/services/test_prompt_loader.py b/tests/server/services/test_prompt_loader.py index 7e3665ee..7b5dc2ff 100644 --- a/tests/server/services/test_prompt_loader.py +++ b/tests/server/services/test_prompt_loader.py @@ -1,71 +1,56 @@ -from pathlib import Path - import pytest -from supernote.server.utils.prompt_loader import PromptId, PromptLoader +from supernote.server.utils.prompt_loader import ( + DEFAULT_OCR_PROMPT, + DEFAULT_SUMMARY_COMMON_PROMPT, + DEFAULT_SUMMARY_PROMPT, + PromptId, + PromptLoader, +) @pytest.fixture -def mock_resources_dir(tmp_path: Path) -> Path: - """Create a mock resources directory structure.""" - resources_dir = tmp_path / "resources" - prompts_dir = resources_dir / "prompts" - prompts_dir.mkdir(parents=True) - - # 1. OCR Category setup - ocr_dir = prompts_dir / "ocr" - ocr_dir.mkdir(parents=True) - - # Common - (ocr_dir / "common").mkdir() - (ocr_dir / "common" / "base.md").write_text("Common OCR Base", encoding="utf-8") - - # Default - (ocr_dir / "default").mkdir() - (ocr_dir / "default" / "ocr_transcription.md").write_text( - "Default Ocr", encoding="utf-8" - ) +def loader() -> PromptLoader: + return PromptLoader() - # Custom type_a - (ocr_dir / "type_a").mkdir() - (ocr_dir / "type_a" / "custom1.md").write_text("Custom A", encoding="utf-8") - # 2. Summary Category setup (No common) - summary_dir = prompts_dir / "summary" - summary_dir.mkdir(parents=True) +def test_get_prompt_ocr_returns_default(loader: PromptLoader) -> None: + assert loader.get_prompt(PromptId.OCR_TRANSCRIPTION) == DEFAULT_OCR_PROMPT + + +def test_get_prompt_summary_composes_common_and_default(loader: PromptLoader) -> None: + """Summary prompt is common + default joined.""" + prompt = loader.get_prompt(PromptId.SUMMARY_GENERATION) + assert DEFAULT_SUMMARY_COMMON_PROMPT in prompt + assert DEFAULT_SUMMARY_PROMPT in prompt - # Default - (summary_dir / "default").mkdir() - (summary_dir / "default" / "summary_generation.md").write_text( - "Default Summary", encoding="utf-8" - ) - return prompts_dir +def test_get_prompt_custom_type_ignored(loader: PromptLoader) -> None: + """custom_type is ignored — loader always returns the hardcoded default.""" + assert ( + loader.get_prompt(PromptId.OCR_TRANSCRIPTION, custom_type="daily") + == DEFAULT_OCR_PROMPT + ) -def test_get_prompt_ocr_default(mock_resources_dir: Path) -> None: - """Test OCR: Has Common + Default""" - loader = PromptLoader(resources_dir=mock_resources_dir) - prompt = loader.get_prompt(PromptId.OCR_TRANSCRIPTION) - assert prompt == "Common OCR Base\n\nDefault Ocr" +def test_get_prompt_unknown_id_raises(loader: PromptLoader) -> None: + with pytest.raises(ValueError): + loader.get_prompt("nonexistent") # type: ignore[arg-type] -def test_get_prompt_ocr_custom(mock_resources_dir: Path) -> None: - """Test OCR: Has Common + Custom""" - loader = PromptLoader(resources_dir=mock_resources_dir) - prompt = loader.get_prompt(PromptId.OCR_TRANSCRIPTION, custom_type="type_a") - assert prompt == "Common OCR Base\n\nCustom A" +def test_get_all_known_layers_ocr(loader: PromptLoader) -> None: + layers = loader.get_all_known_layers() + assert layers["ocr_transcription"]["default"] == DEFAULT_OCR_PROMPT + assert "common" not in layers["ocr_transcription"] -def test_get_prompt_ocr_fallback(mock_resources_dir: Path) -> None: - """Test OCR: Custom type missing -> Fallback to Common + Default""" - loader = PromptLoader(resources_dir=mock_resources_dir) - prompt = loader.get_prompt(PromptId.OCR_TRANSCRIPTION, custom_type="unknown_type") - assert prompt == "Common OCR Base\n\nDefault Ocr" +def test_get_all_known_layers_summary(loader: PromptLoader) -> None: + layers = loader.get_all_known_layers() + assert layers["summary_generation"]["common"] == DEFAULT_SUMMARY_COMMON_PROMPT + assert layers["summary_generation"]["default"] == DEFAULT_SUMMARY_PROMPT -def test_get_prompt_summary(mock_resources_dir: Path) -> None: - """Test Summary: No Common, only Default""" - loader = PromptLoader(resources_dir=mock_resources_dir) - prompt = loader.get_prompt(PromptId.SUMMARY_GENERATION) - assert prompt == "Default Summary" +def test_default_prompts_are_nonempty() -> None: + assert DEFAULT_OCR_PROMPT.strip() + assert DEFAULT_SUMMARY_COMMON_PROMPT.strip() + assert DEFAULT_SUMMARY_PROMPT.strip() diff --git a/uv.lock b/uv.lock index fbb07474..80f39c97 100644 --- a/uv.lock +++ b/uv.lock @@ -1447,7 +1447,7 @@ wheels = [ [[package]] name = "supernote" -version = "0.14.8" +version = "1.0.1" source = { editable = "." } dependencies = [ { name = "colour" }, From 48088288251a231b347118a0959b193e9a4cb792 Mon Sep 17 00:00:00 2001 From: Damir Dulic Date: Tue, 17 Mar 2026 16:44:02 +0000 Subject: [PATCH 2/3] chore: bump minor version to 1.1.0 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 514afe76..eaf31a68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ requires = ["setuptools>=77.0"] [project] name = "supernote" -version = "1.0.1" +version = "1.1.0" license = "Apache-2.0" license-files = ["LICENSE"] description = "All-in-one toolkit for Supernote devices: parse notebooks, self-host services, access services" From 6ed0824b6b162c80e01834a60fbc7a06925922ab Mon Sep 17 00:00:00 2001 From: Damir Dulic Date: Tue, 17 Mar 2026 17:02:06 +0000 Subject: [PATCH 3/3] test: add missing coverage for prompts routes and prompt config service Cover user-not-found branches, exception handlers, invalid file_id paths, reprocess-all handler, stale page reprocessing with page_ids filtering, already-processing 409, page-level reprocess success/failure, and PromptConfigService edge cases (no-content ValueError, unknown prompt_id skip, make_prompt_resolver). --- tests/server/routes/test_prompts.py | 578 ++++++++++++++++++ .../services/test_prompt_config_service.py | 72 +++ 2 files changed, 650 insertions(+) diff --git a/tests/server/routes/test_prompts.py b/tests/server/routes/test_prompts.py index 3605afb6..d83c0e00 100644 --- a/tests/server/routes/test_prompts.py +++ b/tests/server/routes/test_prompts.py @@ -1,5 +1,7 @@ """Tests for prompt configuration routes.""" +from unittest.mock import AsyncMock, patch + from aiohttp.test_utils import TestClient from sqlalchemy import select @@ -383,3 +385,579 @@ async def test_reprocess_page_not_stale_returns_400( assert resp.status == 400 data = await resp.json() assert data["success"] is False + + +# --------------------------------------------------------------------------- +# User-not-found branches (line 59, 83, 123, 162, 239, 337, 411) +# --------------------------------------------------------------------------- + + +async def test_get_prompts_user_not_found( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """GET /api/extended/prompts returns 404 when user lookup returns None.""" + with patch.object( + client.app["user_service"], "get_user_id", new=AsyncMock(return_value=None) + ): + resp = await client.get("/api/extended/prompts", headers=auth_headers) + assert resp.status == 404 + + +async def test_put_prompt_user_not_found( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """PUT /api/extended/prompts returns 404 when user lookup returns None.""" + with patch.object( + client.app["user_service"], "get_user_id", new=AsyncMock(return_value=None) + ): + resp = await client.put( + "/api/extended/prompts", + headers=auth_headers, + json={"category": "ocr", "layer": "default", "content": "x"}, + ) + assert resp.status == 404 + + +async def test_delete_prompt_user_not_found( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """DELETE /api/extended/prompts returns 404 when user lookup returns None.""" + with patch.object( + client.app["user_service"], "get_user_id", new=AsyncMock(return_value=None) + ): + resp = await client.delete( + "/api/extended/prompts/ocr/default", headers=auth_headers + ) + assert resp.status == 404 + + +async def test_staleness_user_not_found( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """GET /files/{file_id}/staleness returns 404 when user lookup returns None.""" + with patch.object( + client.app["user_service"], "get_user_id", new=AsyncMock(return_value=None) + ): + resp = await client.get("/api/extended/files/1/staleness", headers=auth_headers) + assert resp.status == 404 + + +async def test_reprocess_file_user_not_found( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """POST /files/{file_id}/reprocess returns 404 when user lookup returns None.""" + with patch.object( + client.app["user_service"], "get_user_id", new=AsyncMock(return_value=None) + ): + resp = await client.post( + "/api/extended/files/1/reprocess", headers=auth_headers + ) + assert resp.status == 404 + + +async def test_reprocess_page_user_not_found( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """POST /files/{file_id}/pages/{page_id}/reprocess returns 404 when user is None.""" + with patch.object( + client.app["user_service"], "get_user_id", new=AsyncMock(return_value=None) + ): + resp = await client.post( + "/api/extended/files/1/pages/P001/reprocess", headers=auth_headers + ) + assert resp.status == 404 + + +# --------------------------------------------------------------------------- +# Invalid file_id (non-integer) branches +# --------------------------------------------------------------------------- + + +async def test_staleness_invalid_file_id( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """GET /files/abc/staleness returns 400 for non-integer file_id.""" + resp = await client.get("/api/extended/files/abc/staleness", headers=auth_headers) + assert resp.status == 400 + data = await resp.json() + assert data["success"] is False + + +async def test_reprocess_file_invalid_file_id( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """POST /files/abc/reprocess returns 400 for non-integer file_id.""" + resp = await client.post("/api/extended/files/abc/reprocess", headers=auth_headers) + assert resp.status == 400 + data = await resp.json() + assert data["success"] is False + + +async def test_reprocess_page_invalid_file_id( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """POST /files/abc/pages/P001/reprocess returns 400 for non-integer file_id.""" + resp = await client.post( + "/api/extended/files/abc/pages/P001/reprocess", headers=auth_headers + ) + assert resp.status == 400 + data = await resp.json() + assert data["success"] is False + + +# --------------------------------------------------------------------------- +# Exception-handler branches (uncaught exceptions → 500) +# --------------------------------------------------------------------------- + + +async def test_get_prompts_service_exception( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """GET /api/extended/prompts returns 500 when service raises.""" + with patch.object( + client.app["prompt_config_service"], + "get_all_configs_with_defaults", + new=AsyncMock(side_effect=RuntimeError("db exploded")), + ): + resp = await client.get("/api/extended/prompts", headers=auth_headers) + assert resp.status == 500 + + +async def test_put_prompt_service_exception( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """PUT /api/extended/prompts returns 500 when service raises unexpectedly.""" + with patch.object( + client.app["prompt_config_service"], + "upsert_config", + new=AsyncMock(side_effect=RuntimeError("db exploded")), + ): + resp = await client.put( + "/api/extended/prompts", + headers=auth_headers, + json={"category": "ocr", "layer": "default", "content": "x"}, + ) + assert resp.status == 500 + + +async def test_delete_prompt_service_exception( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """DELETE /api/extended/prompts returns 500 when service raises unexpectedly.""" + with patch.object( + client.app["prompt_config_service"], + "delete_config", + new=AsyncMock(side_effect=RuntimeError("db exploded")), + ): + resp = await client.delete( + "/api/extended/prompts/ocr/nonexistent-layer", headers=auth_headers + ) + assert resp.status == 500 + + +async def test_staleness_service_exception( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """GET staleness returns 500 when prompt_config_service raises.""" + user_id = await _get_user_id(session_manager) + async with session_manager.session() as session: + file_do = UserFileDO( + id=9001, user_id=user_id, file_name="f.note", is_folder="N" + ) + session.add(file_do) + await session.commit() + + with patch.object( + client.app["prompt_config_service"], + "compute_combined_prompt_hash", + new=AsyncMock(side_effect=RuntimeError("hash error")), + ): + resp = await client.get( + "/api/extended/files/9001/staleness", headers=auth_headers + ) + assert resp.status == 500 + + +# --------------------------------------------------------------------------- +# DELETE protected layer +# --------------------------------------------------------------------------- + + +async def test_delete_prompt_protected_layer_returns_400( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """DELETE returns 400 when attempting to delete a protected layer.""" + resp = await client.delete( + "/api/extended/prompts/ocr/default", headers=auth_headers + ) + assert resp.status == 400 + data = await resp.json() + assert data["success"] is False + + +# --------------------------------------------------------------------------- +# PUT invalid JSON body +# --------------------------------------------------------------------------- + + +async def test_put_prompt_invalid_json( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """PUT /api/extended/prompts returns 400 for malformed request body.""" + resp = await client.put( + "/api/extended/prompts", + headers={**auth_headers, "Content-Type": "application/json"}, + data=b"not-json", + ) + assert resp.status == 400 + data = await resp.json() + assert data["success"] is False + + +# --------------------------------------------------------------------------- +# POST /files/{file_id}/reprocess — stale pages + page_ids body +# --------------------------------------------------------------------------- + + +async def test_reprocess_file_with_stale_pages( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """Reprocess queues stale pages and returns queued_page_count > 0.""" + user_id = await _get_user_id(session_manager) + + async with session_manager.session() as session: + file_do = UserFileDO( + id=10001, user_id=user_id, file_name="monthly.note", is_folder="N" + ) + session.add(file_do) + page = NotePageContentDO( + file_id=10001, page_index=0, page_id="P10001", prompt_hash="old-hash" + ) + session.add(page) + await session.commit() + + with patch.object( + client.app["processor_service"], + "reprocess_pages", + new=AsyncMock(return_value=1), + ): + resp = await client.post( + "/api/extended/files/10001/reprocess", headers=auth_headers + ) + assert resp.status == 200 + data = await resp.json() + assert data["success"] is True + assert data["queuedPageCount"] == 1 + + +async def test_reprocess_file_with_page_ids_filter( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """Reprocess with explicit page_ids filters to only stale pages in the list.""" + user_id = await _get_user_id(session_manager) + + async with session_manager.session() as session: + file_do = UserFileDO( + id=10002, user_id=user_id, file_name="weekly.note", is_folder="N" + ) + session.add(file_do) + page = NotePageContentDO( + file_id=10002, page_index=0, page_id="P10002", prompt_hash="old-hash" + ) + session.add(page) + await session.commit() + + with patch.object( + client.app["processor_service"], + "reprocess_pages", + new=AsyncMock(return_value=1), + ): + resp = await client.post( + "/api/extended/files/10002/reprocess", + headers=auth_headers, + json={"pageIds": ["P10002"]}, + ) + assert resp.status == 200 + data = await resp.json() + assert data["success"] is True + assert data["queuedPageCount"] == 1 + + +async def test_reprocess_file_already_processing( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """Reprocess returns 409 when file is already being processed.""" + user_id = await _get_user_id(session_manager) + + async with session_manager.session() as session: + file_do = UserFileDO( + id=10003, user_id=user_id, file_name="daily.note", is_folder="N" + ) + session.add(file_do) + page = NotePageContentDO( + file_id=10003, page_index=0, page_id="P10003", prompt_hash="old-hash" + ) + session.add(page) + await session.commit() + + client.app["processor_service"].processing_files.add(10003) + try: + resp = await client.post( + "/api/extended/files/10003/reprocess", headers=auth_headers + ) + assert resp.status == 409 + data = await resp.json() + assert data["success"] is False + finally: + client.app["processor_service"].processing_files.discard(10003) + + +# --------------------------------------------------------------------------- +# POST /files/{file_id}/pages/{page_id}/reprocess — file not found + success +# --------------------------------------------------------------------------- + + +async def test_reprocess_page_file_not_found( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """Page reprocess returns 403 when file doesn't belong to user.""" + resp = await client.post( + "/api/extended/files/99999/pages/P001/reprocess", headers=auth_headers + ) + assert resp.status == 403 + + +async def test_reprocess_page_page_not_found( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """Page reprocess returns 404 when the page_id doesn't exist in the file.""" + user_id = await _get_user_id(session_manager) + + async with session_manager.session() as session: + file_do = UserFileDO( + id=11001, user_id=user_id, file_name="monthly.note", is_folder="N" + ) + session.add(file_do) + await session.commit() + + resp = await client.post( + "/api/extended/files/11001/pages/NONEXISTENT/reprocess", headers=auth_headers + ) + assert resp.status == 404 + data = await resp.json() + assert data["success"] is False + + +async def test_reprocess_page_success( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """Page reprocess succeeds for a stale page.""" + user_id = await _get_user_id(session_manager) + + async with session_manager.session() as session: + file_do = UserFileDO( + id=11002, user_id=user_id, file_name="monthly.note", is_folder="N" + ) + session.add(file_do) + page = NotePageContentDO( + file_id=11002, page_index=0, page_id="P11002", prompt_hash="old-hash" + ) + session.add(page) + await session.commit() + + with patch.object( + client.app["processor_service"], + "reprocess_pages", + new=AsyncMock(return_value=1), + ): + resp = await client.post( + "/api/extended/files/11002/pages/P11002/reprocess", headers=auth_headers + ) + assert resp.status == 200 + data = await resp.json() + assert data["success"] is True + assert data["queuedPageCount"] == 1 + + +# --------------------------------------------------------------------------- +# POST /api/extended/reprocess-all +# --------------------------------------------------------------------------- + + +async def test_reprocess_all_requires_auth(client: TestClient) -> None: + """POST /api/extended/reprocess-all returns 401 without auth.""" + resp = await client.post("/api/extended/reprocess-all") + assert resp.status == 401 + + +async def test_reprocess_all_user_not_found( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """POST /api/extended/reprocess-all returns 401 when user lookup returns None.""" + with patch.object( + client.app["user_service"], "get_user_id", new=AsyncMock(return_value=None) + ): + resp = await client.post("/api/extended/reprocess-all", headers=auth_headers) + assert resp.status == 401 + + +async def test_reprocess_all_success( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """POST /api/extended/reprocess-all returns queued_page_count.""" + with patch.object( + client.app["processor_service"], + "reprocess_all", + new=AsyncMock(return_value=5), + ): + resp = await client.post("/api/extended/reprocess-all", headers=auth_headers) + assert resp.status == 200 + data = await resp.json() + assert data["success"] is True + assert data["queuedPageCount"] == 5 + + +async def test_reprocess_all_service_exception( + client: TestClient, + auth_headers: dict[str, str], +) -> None: + """POST /api/extended/reprocess-all returns 500 on unexpected exception.""" + with patch.object( + client.app["processor_service"], + "reprocess_all", + new=AsyncMock(side_effect=RuntimeError("exploded")), + ): + resp = await client.post("/api/extended/reprocess-all", headers=auth_headers) + assert resp.status == 500 + + +async def test_reprocess_file_invalid_body_ignored( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """Reprocess with an invalid JSON body ignores it and uses all stale pages.""" + user_id = await _get_user_id(session_manager) + + async with session_manager.session() as session: + file_do = UserFileDO( + id=12001, user_id=user_id, file_name="daily.note", is_folder="N" + ) + session.add(file_do) + page = NotePageContentDO( + file_id=12001, page_index=0, page_id="P12001", prompt_hash="stale" + ) + session.add(page) + await session.commit() + + with patch.object( + client.app["processor_service"], + "reprocess_pages", + new=AsyncMock(return_value=1), + ): + resp = await client.post( + "/api/extended/files/12001/reprocess", + headers={**auth_headers, "Content-Type": "application/json"}, + data=b"not-valid-json", + ) + assert resp.status == 200 + data = await resp.json() + assert data["success"] is True + + +async def test_reprocess_file_service_exception( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """POST /files/{file_id}/reprocess returns 500 on unexpected exception.""" + user_id = await _get_user_id(session_manager) + + async with session_manager.session() as session: + file_do = UserFileDO( + id=12002, user_id=user_id, file_name="daily.note", is_folder="N" + ) + session.add(file_do) + page = NotePageContentDO( + file_id=12002, page_index=0, page_id="P12002", prompt_hash="stale" + ) + session.add(page) + await session.commit() + + with patch.object( + client.app["processor_service"], + "reprocess_pages", + new=AsyncMock(side_effect=RuntimeError("exploded")), + ): + resp = await client.post( + "/api/extended/files/12002/reprocess", headers=auth_headers + ) + assert resp.status == 500 + + +async def test_reprocess_page_service_exception( + client: TestClient, + auth_headers: dict[str, str], + session_manager: DatabaseSessionManager, + create_test_user: None, +) -> None: + """POST /files/{file_id}/pages/{page_id}/reprocess returns 500 on exception.""" + user_id = await _get_user_id(session_manager) + + async with session_manager.session() as session: + file_do = UserFileDO( + id=12003, user_id=user_id, file_name="monthly.note", is_folder="N" + ) + session.add(file_do) + page = NotePageContentDO( + file_id=12003, page_index=0, page_id="P12003", prompt_hash="stale" + ) + session.add(page) + await session.commit() + + with patch.object( + client.app["processor_service"], + "reprocess_pages", + new=AsyncMock(side_effect=RuntimeError("exploded")), + ): + resp = await client.post( + "/api/extended/files/12003/pages/P12003/reprocess", headers=auth_headers + ) + assert resp.status == 500 diff --git a/tests/server/services/test_prompt_config_service.py b/tests/server/services/test_prompt_config_service.py index cfe94187..c5f36104 100644 --- a/tests/server/services/test_prompt_config_service.py +++ b/tests/server/services/test_prompt_config_service.py @@ -322,3 +322,75 @@ async def test_get_all_configs_with_defaults_default_content_always_present( configs = await service.get_all_configs_with_defaults(user_id=1) for c in configs: assert c.default_content, f"Missing default_content for {c.category}/{c.layer}" + + +async def test_get_all_configs_with_defaults_custom_user_layer( + service: PromptConfigService, +) -> None: + """User-saved layers not in the file loader appear as custom overrides.""" + await service.upsert_config( + user_id=1, category="ocr", layer="my-project", content="Project-specific OCR" + ) + configs = await service.get_all_configs_with_defaults(user_id=1) + custom = next( + (c for c in configs if c.category == "ocr" and c.layer == "my-project"), None + ) + assert custom is not None + assert custom.is_override is True + assert custom.content == "Project-specific OCR" + + +async def test_get_effective_prompt_raises_when_no_content( + service: PromptConfigService, prompt_loader: PromptLoader +) -> None: + """get_effective_prompt raises ValueError when no prompt content is resolvable.""" + from unittest.mock import patch + + # Return an empty layer map so neither common nor specific has content + with patch.object(prompt_loader, "get_all_known_layers", return_value={}): + with pytest.raises(ValueError, match="No prompt content"): + await service.get_effective_prompt( + user_id=1, prompt_id=PromptId.OCR_TRANSCRIPTION, note_type=None + ) + + +async def test_make_prompt_resolver_returns_effective_prompt( + service: PromptConfigService, +) -> None: + """make_prompt_resolver returns a callable that resolves the prompt.""" + resolver = service.make_prompt_resolver(user_id=1, note_type=None) + result = await resolver(PromptId.OCR_TRANSCRIPTION) # type: ignore[misc, call-arg] + assert isinstance(result, str) + assert len(result) > 0 + + +async def test_make_prompt_resolver_custom_type_overrides_note_type( + service: PromptConfigService, +) -> None: + """make_prompt_resolver resolver uses custom_type when provided.""" + await service.upsert_config( + user_id=1, category="ocr", layer="weekly", content="Weekly OCR text" + ) + resolver = service.make_prompt_resolver(user_id=1, note_type="monthly") + result = await resolver(PromptId.OCR_TRANSCRIPTION, "weekly") # type: ignore[misc] + assert result == "Weekly OCR text" + + +async def test_get_all_configs_skips_unknown_prompt_id( + service: PromptConfigService, prompt_loader: PromptLoader +) -> None: + """get_all_configs_with_defaults skips entries whose prompt_id has no category.""" + from unittest.mock import patch + + # Inject an unknown prompt_id_value that has no mapping in _PROMPT_ID_TO_CATEGORY + original = prompt_loader.get_all_known_layers() + extended = {**original, "unknown-prompt-id": {"default": "some text"}} + with patch.object(prompt_loader, "get_all_known_layers", return_value=extended): + configs = await service.get_all_configs_with_defaults(user_id=1) + # The unknown entry should be silently skipped + layers_for_unknown = [ + c + for c in configs + if c.layer == "default" and c.category not in ("ocr", "summary") + ] + assert layers_for_unknown == []