-
Notifications
You must be signed in to change notification settings - Fork 19
Add explicit staged import sealing #47
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -39,13 +39,16 @@ def run_import( | |
| ) -> ImportSummary: | ||
| """Execute one deterministic import pass. | ||
|
|
||
| Candidates are persisted only as import records. Promotion into assertions, | ||
| projects, tasks, or decisions is deliberately deferred to reconciliation. | ||
| Candidates are persisted only as import records. Dry runs finish in the | ||
| ``previewed`` state. Real imports finish in the ``staged`` state and require | ||
| an explicit validation seal before any later promotion step. | ||
| """ | ||
| if resume_run_id: | ||
| existing = get_import_run(resume_run_id) | ||
| if existing is None: | ||
| raise ValueError(f"import run not found: {resume_run_id}") | ||
| if existing.get("status") in {"sealed", "rolled_back"}: | ||
| raise ValueError(f"{existing['status']} imports cannot be resumed") | ||
| run_id = UUID(str(existing["id"])) | ||
| config = existing.get("config") or {} | ||
| if isinstance(config, str): | ||
|
|
@@ -102,9 +105,10 @@ def run_import( | |
| records_rejected=counters["rejected"], | ||
| ) | ||
|
|
||
| final_status = "previewed" if dry_run else "staged" | ||
| update_import_run( | ||
| run_id, | ||
| status="completed", | ||
| status=final_status, | ||
| cursor_value=cursor, | ||
|
Comment on lines
109
to
112
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
If a resume starts just before an operator calls Useful? React with 👍 / 👎. |
||
| records_seen=counters["seen"], | ||
| records_imported=counters["imported"], | ||
|
|
@@ -126,11 +130,11 @@ def run_import( | |
|
|
||
| return ImportSummary( | ||
| run_id=run_id, | ||
| status="completed", | ||
| status=final_status, | ||
| dry_run=dry_run, | ||
| records_seen=counters["seen"], | ||
| records_imported=counters["imported"], | ||
| records_merged=counters["merged"], | ||
| records_rejected=counters["rejected"], | ||
| cursor=cursor, | ||
| ) | ||
| ) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,73 @@ | ||
| """Validation and sealing controls for staged import runs.""" | ||
|
|
||
| from __future__ import annotations | ||
|
|
||
| import json | ||
| from typing import Any | ||
| from uuid import UUID | ||
|
|
||
| from db.connection import get_db_cursor | ||
|
|
||
|
|
||
| def seal_import_run(run_id: UUID, *, actor: str, expected_records: int | None = None) -> dict[str, Any]: | ||
| """Seal a completed staging run after validating its immutable candidate set. | ||
|
|
||
| Sealing is a deliberate operator boundary between discovery and promotion. A | ||
| sealed run cannot accept further candidates or be resumed. | ||
| """ | ||
| with get_db_cursor() as cursor: | ||
| cursor.execute("SELECT * FROM import_run WHERE id = %s FOR UPDATE", (run_id,)) | ||
| run = cursor.fetchone() | ||
| if run is None: | ||
| raise ValueError(f"import run not found: {run_id}") | ||
|
|
||
| config = run.get("config") or {} | ||
| if isinstance(config, str): | ||
| config = json.loads(config) | ||
| if bool(config.get("dry_run")): | ||
| raise ValueError("dry-run imports cannot be sealed") | ||
| if run.get("rolled_back_at") is not None or run.get("status") == "rolled_back": | ||
| raise ValueError("rolled-back imports cannot be sealed") | ||
| if run.get("status") == "sealed": | ||
| return dict(run) | ||
| if run.get("status") != "staged": | ||
| raise ValueError("only staged imports can be sealed") | ||
|
|
||
| cursor.execute( | ||
| """ | ||
| SELECT COUNT(*) AS total, | ||
| COUNT(*) FILTER (WHERE result = 'staged' AND rolled_back_at IS NULL) AS valid, | ||
| COUNT(*) FILTER (WHERE error IS NOT NULL) AS failed | ||
| FROM import_record | ||
| WHERE import_run_id = %s | ||
| """, | ||
| (run_id,), | ||
| ) | ||
| counts = cursor.fetchone() | ||
| total = int(counts["total"]) | ||
| valid = int(counts["valid"]) | ||
| failed = int(counts["failed"]) | ||
| if failed or valid != total: | ||
| raise ValueError("import contains invalid, failed, or rolled-back records") | ||
| if expected_records is not None and total != expected_records: | ||
| raise ValueError( | ||
| f"staged record count changed: expected {expected_records}, found {total}" | ||
| ) | ||
|
|
||
| seal = { | ||
| "actor": actor, | ||
| "record_count": total, | ||
| "source_fingerprint": config.get("source_fingerprint"), | ||
| } | ||
| cursor.execute( | ||
| """ | ||
| UPDATE import_run | ||
| SET status = 'sealed', | ||
| config = config || jsonb_build_object('seal', %s::jsonb), | ||
| completed_at = COALESCE(completed_at, NOW()) | ||
| WHERE id = %s | ||
| RETURNING * | ||
| """, | ||
| (json.dumps(seal), run_id), | ||
| ) | ||
| return dict(cursor.fetchone()) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Because
update_import_run()only fillscompleted_atforcompleted,failed, andcancelled, the new terminal statuses assigned here leave finished dry runs and staged imports withcompleted_at = NULLuntil sealing (and previewed runs forever). That makes the persisted lifecycle metadata inconsistent with the run having finished; includepreviewed/stagedin the completion timestamp path when introducing these final states.AGENTS.md reference: AGENTS.md:L9-L9
Useful? React with 👍 / 👎.