diff --git a/migrations/0013_mesa_v4_native_publisher_contract.sql b/migrations/0013_mesa_v4_native_publisher_contract.sql new file mode 100644 index 0000000..69abcce --- /dev/null +++ b/migrations/0013_mesa_v4_native_publisher_contract.sql @@ -0,0 +1,10 @@ +-- Native MESA V4 delivery contract: session routes and durable session identity. +-- Forward-only and safe for existing data roots. + +ALTER TABLE mesa_target_settings ADD COLUMN session_start_path TEXT NOT NULL DEFAULT '/v4/sessions/start'; +ALTER TABLE mesa_target_settings ADD COLUMN session_end_path_template TEXT NOT NULL DEFAULT '/v4/sessions/{session_id}/end'; +ALTER TABLE mesa_deliveries ADD COLUMN remote_session_id TEXT; + +-- Existing stored route assumptions were deliberately marked unknown by 0008. +-- New defaults make the authoritative V4 routes available once an operator +-- explicitly configures/verifies the rest of the target contract. diff --git a/src/mesa_legal_data/publisher/client.py b/src/mesa_legal_data/publisher/client.py index 2297556..58cab4b 100644 --- a/src/mesa_legal_data/publisher/client.py +++ b/src/mesa_legal_data/publisher/client.py @@ -39,20 +39,18 @@ def __init__( def is_api_key_configured(self) -> bool: return bool(self._api_key and self._api_key.strip()) - def _get_headers(self, idempotency_key: str | None = None) -> dict[str, str]: + def _get_headers(self) -> dict[str, str]: headers = { "Content-Type": "application/json", "Accept": "application/json", "User-Agent": "MESA-Legal-Data-Publisher/1.0", } if self._api_key: - headers["Authorization"] = f"Bearer {self._api_key}" - if idempotency_key: - headers["Idempotency-Key"] = idempotency_key + headers["X-API-Key"] = self._api_key return headers def target_safety_error(self) -> str | None: - """Validate the target before a request can carry Authorization.""" + """Validate the target before a request can carry the API key.""" try: parsed = urlparse(self.settings.base_url) except ValueError: @@ -80,6 +78,7 @@ def is_contract_configured(self) -> bool: self.settings.contract_source in ("configured", "live_verified") and self.settings.base_url and self.settings.health_path.startswith("/") + and self.settings.session_start_path.startswith("/") and self.settings.publish_path.startswith("/") and self.settings.mutation_status_path_template.startswith("/") and "{mutation_id}" in self.settings.mutation_status_path_template @@ -93,6 +92,18 @@ def _normalize_mutation_state(raw_state: Any) -> tuple[str, str | None]: state_upper = raw_state.strip().upper() if state_upper in ("ACCEPTED", "RECEIVED"): return MutationState.QUEUED.value, None + if state_upper in ( + "EXTRACTED", + "VALIDATED", + "SQL_APPLIED", + "VECTOR_APPLIED", + "GRAPH_APPLIED", + "RETRY_PENDING", + "ROLLING_BACK", + ): + return MutationState.PROCESSING.value, None + if state_upper in ("DEAD_LETTER", "BLOCKED", "ROLLED_BACK"): + return MutationState.FAILED.value, None if state_upper in MutationState.__members__: return state_upper, None return MutationState.FAILED.value, f"Unknown MESA mutation state: {raw_state}" @@ -324,33 +335,106 @@ def run_preflight_checks( total_canonical_bytes=total_canonical_bytes, ) - def publish_source_chunk( + def start_session(self) -> dict[str, Any]: + """Create one MESA V4 session scoped to this delivery target.""" + if not self.is_contract_configured: + return {"session_id": None, "error": "MESA HTTP contract is unknown"} + target_error = self.target_safety_error() + if target_error: + return {"session_id": None, "error": target_error} + payload = { + "tenant_id": self.settings.tenant_id, + "workspace_id": self.settings.workspace_id, + "dataset_ids": [self.settings.dataset_id], + "agent_id": self.settings.agent_id, + } + try: + with httpx.Client(timeout=self.timeout_seconds) as client: + response = client.post( + f"{self.settings.base_url.rstrip('/')}{self.settings.session_start_path}", + json=payload, + headers=self._get_headers(), + ) + if response.status_code == 201: + data = response.json() + session_id = data.get("session_id") + if isinstance(session_id, str) and session_id: + return {"session_id": session_id, "message": data.get("status", "started")} + return {"session_id": None, "error": "MESA session start response omitted session_id"} + return {"session_id": None, "error": f"HTTP {response.status_code}: {response.text[:200]}"} + except Exception as exc: + return {"session_id": None, "error": f"Transport failure starting session: {exc}"} + + def end_session(self, session_id: str) -> dict[str, Any]: + """End a session only once no mutation needs it for status access.""" + if not session_id or not self.settings.session_end_path_template: + return {"ended": False, "error": "Missing session_id or session end route"} + path = self.settings.session_end_path_template.replace("{session_id}", session_id) + try: + with httpx.Client(timeout=self.timeout_seconds) as client: + response = client.post(f"{self.settings.base_url.rstrip('/')}{path}", headers=self._get_headers()) + return { + "ended": response.status_code in (200, 202), + "error": None + if response.status_code in (200, 202) + else f"HTTP {response.status_code}: {response.text[:200]}", + } + except Exception as exc: + return {"ended": False, "error": f"Transport failure ending session: {exc}"} + + def build_memory_insert_payload( self, chunk: SourceChunk, + *, + session_id: str, idempotency_key: str, + finalize_revision: bool, ) -> dict[str, Any]: - """ - Submits a single source chunk mutation to MESA v4 HTTP API. - Returns dictionary with mutation_id, state (COMMITTED, QUEUED, etc.), and message. - """ - payload = { - "tenant_id": self.settings.tenant_id, - "workspace_id": self.settings.workspace_id, + """Map a frozen source chunk to the strict V4MemoryInsertRequest shape.""" + metadata = { + **chunk.metadata, + "mesa_data_chunk_type": chunk.chunk_type, + "mesa_data_char_start": chunk.char_start, + "mesa_data_char_end": chunk.char_end, + "mesa_data_content_hash": chunk.content_hash, + } + source_ref = metadata.get("authoritative_source_ref") or metadata.get("source_url") + if not isinstance(source_ref, str) or not source_ref.strip(): + source_ref = f"mesa-data://releases/{metadata.get('release_id', 'current')}/documents/{chunk.document_id}/revisions/{chunk.version_id}/chunks/{chunk.chunk_id}" + return { + "session_id": session_id, "dataset_id": self.settings.dataset_id, - "agent_id": self.settings.agent_id, "document_id": chunk.document_id, - "version_id": chunk.version_id, + "revision_id": chunk.version_id, "chunk_id": chunk.chunk_id, - "chunk_type": chunk.chunk_type, - "title": chunk.title, - "char_start": chunk.char_start, - "char_end": chunk.char_end, - "ordinal": chunk.ordinal, + "title": chunk.title or f"Document {chunk.document_id} chunk {chunk.ordinal}", + "source_ref": source_ref, "content": chunk.content, - "content_hash": chunk.content_hash, - "metadata": chunk.metadata, + "evidence_span": "", + "revision_number": int(metadata.get("revision_number", 1)), + "chunk_ordinal": chunk.ordinal, + "finalize_revision": finalize_revision, + "supersedes_revision_id": metadata.get("supersedes_revision_id"), + "metadata": metadata, + "idempotency_key": idempotency_key, } + def publish_source_chunk( + self, + chunk: SourceChunk, + idempotency_key: str, + *, + session_id: str, + finalize_revision: bool, + ) -> dict[str, Any]: + """Submit one strict V4MemoryInsertRequest to MESA.""" + payload = self.build_memory_insert_payload( + chunk, + session_id=session_id, + idempotency_key=idempotency_key, + finalize_revision=finalize_revision, + ) + if not self.is_contract_configured: return { "mutation_id": None, @@ -363,7 +447,7 @@ def publish_source_chunk( return {"mutation_id": None, "state": MutationState.FAILED.value, "message": target_error} url = f"{self.settings.base_url.rstrip('/')}{self.settings.publish_path}" - headers = self._get_headers(idempotency_key=idempotency_key) + headers = self._get_headers() try: with httpx.Client(timeout=self.timeout_seconds) as client: diff --git a/src/mesa_legal_data/publisher/engine.py b/src/mesa_legal_data/publisher/engine.py index 09bb843..e7065c8 100644 --- a/src/mesa_legal_data/publisher/engine.py +++ b/src/mesa_legal_data/publisher/engine.py @@ -17,6 +17,7 @@ insert_delivery_item, is_chunk_already_committed, list_failed_delivery_items, + set_delivery_remote_session, update_delivery_item_state, update_delivery_progress, ) @@ -47,8 +48,10 @@ def target_config_sha256(settings) -> str: "content_limit_chars": settings.content_limit_chars, "contract_source": settings.contract_source, "health_path": settings.health_path, + "session_start_path": settings.session_start_path, "publish_path": settings.publish_path, "mutation_status_path_template": settings.mutation_status_path_template, + "session_end_path_template": settings.session_end_path_template, } canonical = json.dumps(payload, sort_keys=True, separators=(",", ":")) return hashlib.sha256(canonical.encode("utf-8")).hexdigest() @@ -285,6 +288,25 @@ def build_delivery_plan( content_limit_chars=target_settings.content_limit_chars, ) + provenance = conn.execute( + """SELECT v.revision_number, v.supersedes_version_id, a.source_url, a.source_id, a.sha256 + FROM versions v JOIN artifacts a ON a.artifact_id = v.artifact_id + WHERE v.version_id = ?""", + (v_id,), + ).fetchone() + for chunk_index, chunk in enumerate(chunks): + chunk.metadata.update( + { + "release_id": release_id, + "revision_number": provenance[0] if provenance else 1, + "supersedes_revision_id": provenance[1] if provenance else None, + "source_url": provenance[2] if provenance else None, + "source_id": provenance[3] if provenance else None, + "artifact_sha256": provenance[4] if provenance else None, + "is_final_chunk": chunk_index == len(chunks) - 1, + } + ) + for chunk in chunks: total_bytes += len(chunk.content.encode("utf-8")) is_committed = is_chunk_already_committed( @@ -376,6 +398,26 @@ def execute_publish_delivery( release_manifest_sha256=release_manifest_sha256, ) + remote_session_id: str | None = None + if any(not is_already_done for _, is_already_done in chunk_tuples): + session_res = client.start_session() + remote_session_id = session_res.get("session_id") + if not remote_session_id: + error = session_res.get("error") or "MESA session start failed" + update_delivery_progress( + conn, + delivery_id=delivery_id, + status=DeliveryStatus.FAILED.value, + committed_items=0, + failed_items=0, + skipped_items=0, + last_error=error, + finished=True, + ) + conn.close() + raise MesaClientError(f"MESA session start failed: {error}") + set_delivery_remote_session(conn, delivery_id=delivery_id, remote_session_id=remote_session_id) + update_delivery_progress( conn, delivery_id=delivery_id, @@ -446,7 +488,12 @@ def execute_publish_delivery( ) # Submit chunk to MESA v4 - pub_res = client.publish_source_chunk(chunk, idempotency_key=idemp_key) + pub_res = client.publish_source_chunk( + chunk, + idempotency_key=idemp_key, + session_id=remote_session_id or "", + finalize_revision=bool(chunk.metadata.get("is_final_chunk")), + ) remote_mutation_id = pub_res.get("mutation_id") initial_state = pub_res.get("state", MutationState.FAILED.value) @@ -558,6 +605,10 @@ def execute_publish_delivery( last_error=last_err, finished=final_delivery_status not in (DeliveryStatus.AWAITING_MUTATION.value, DeliveryStatus.SENDING.value), ) + if final_delivery_status == DeliveryStatus.COMMITTED.value and remote_session_id: + # Current MESA checks mutation status through the session, so end only + # after every item is terminal and no retry/poll needs it. + client.end_session(remote_session_id) conn.close() return { "delivery_id": delivery_id, @@ -611,6 +662,15 @@ def retry_delivery_failures( conn.close() return {"delivery_id": delivery_id, "retried_count": 0, "message": "No failed items eligible for retry"} + remote_session_id = delivery.get("remote_session_id") + if not remote_session_id: + session_res = client.start_session() + remote_session_id = session_res.get("session_id") + if not remote_session_id: + conn.close() + raise MesaClientError(f"MESA session start failed: {session_res.get('error') or 'missing session_id'}") + set_delivery_remote_session(conn, delivery_id=delivery_id, remote_session_id=remote_session_id) + retried_success = 0 retried_failed = 0 last_err = None @@ -632,7 +692,12 @@ def retry_delivery_failures( pub_res = client.get_mutation_status(remote_mutation_id) final_state = pub_res.get("state", MutationState.FAILED.value) else: - pub_res = client.publish_source_chunk(chunk, idempotency_key=idemp_key) + pub_res = client.publish_source_chunk( + chunk, + idempotency_key=idemp_key, + session_id=remote_session_id, + finalize_revision=bool(chunk.metadata.get("is_final_chunk")), + ) remote_mutation_id = pub_res.get("mutation_id") final_state = pub_res.get("state", MutationState.FAILED.value) @@ -728,6 +793,8 @@ def retry_delivery_failures( last_error=last_err, finished=new_status != DeliveryStatus.AWAITING_MUTATION.value, ) + if new_status == DeliveryStatus.COMMITTED.value and remote_session_id: + client.end_session(remote_session_id) conn.close() return { diff --git a/src/mesa_legal_data/publisher/ledger.py b/src/mesa_legal_data/publisher/ledger.py index 7f797b8..e6cebdb 100644 --- a/src/mesa_legal_data/publisher/ledger.py +++ b/src/mesa_legal_data/publisher/ledger.py @@ -15,7 +15,8 @@ def get_mesa_target_settings(conn: sqlite3.Connection, target_key: str = "defaul cursor = conn.cursor() cursor.execute( """SELECT target_key, base_url, tenant_id, workspace_id, dataset_id, agent_id, content_limit_chars, updated_at, - contract_source, health_path, publish_path, mutation_status_path_template + contract_source, health_path, session_start_path, publish_path, + mutation_status_path_template, session_end_path_template FROM mesa_target_settings WHERE target_key = ?""", (target_key,), ) @@ -32,8 +33,10 @@ def get_mesa_target_settings(conn: sqlite3.Connection, target_key: str = "defaul updated_at=row[7], contract_source=row[8], health_path=row[9], - publish_path=row[10], - mutation_status_path_template=row[11], + session_start_path=row[10], + publish_path=row[11], + mutation_status_path_template=row[12], + session_end_path_template=row[13], ) # Return sensible default return MesaTargetSettings(target_key=target_key) @@ -47,9 +50,10 @@ def upsert_mesa_target_settings(conn: sqlite3.Connection, settings: MesaTargetSe """INSERT INTO mesa_target_settings ( target_key, base_url, tenant_id, workspace_id, dataset_id, agent_id, content_limit_chars, updated_at, contract_source, health_path, - publish_path, mutation_status_path_template + session_start_path, publish_path, mutation_status_path_template, + session_end_path_template ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(target_key) DO UPDATE SET base_url = excluded.base_url, tenant_id = excluded.tenant_id, @@ -60,8 +64,10 @@ def upsert_mesa_target_settings(conn: sqlite3.Connection, settings: MesaTargetSe updated_at = excluded.updated_at, contract_source = excluded.contract_source, health_path = excluded.health_path, + session_start_path = excluded.session_start_path, publish_path = excluded.publish_path, - mutation_status_path_template = excluded.mutation_status_path_template""", + mutation_status_path_template = excluded.mutation_status_path_template, + session_end_path_template = excluded.session_end_path_template""", ( settings.target_key, settings.base_url, @@ -73,8 +79,10 @@ def upsert_mesa_target_settings(conn: sqlite3.Connection, settings: MesaTargetSe now_iso, settings.contract_source, settings.health_path, + settings.session_start_path, settings.publish_path, settings.mutation_status_path_template, + settings.session_end_path_template, ), ) @@ -137,6 +145,15 @@ def update_delivery_progress( ) +def set_delivery_remote_session(conn: sqlite3.Connection, *, delivery_id: str, remote_session_id: str) -> None: + """Persist the non-secret remote session identity for safe retry recovery.""" + with transaction(conn): + conn.execute( + "UPDATE mesa_deliveries SET remote_session_id = ? WHERE delivery_id = ?", + (remote_session_id, delivery_id), + ) + + def insert_delivery_item( conn: sqlite3.Connection, *, @@ -227,7 +244,7 @@ def list_deliveries(conn: sqlite3.Connection, limit: int = 20, offset: int = 0) cursor.execute( """SELECT delivery_id, release_id, target_key, status, started_at, finished_at, total_items, committed_items, failed_items, skipped_items, last_error, created_at, - target_config_sha256, release_manifest_sha256 + target_config_sha256, release_manifest_sha256, remote_session_id FROM mesa_deliveries ORDER BY created_at DESC LIMIT ? OFFSET ?""", (limit, offset), ) @@ -248,6 +265,7 @@ def list_deliveries(conn: sqlite3.Connection, limit: int = 20, offset: int = 0) "created_at": r[11], "target_config_sha256": r[12], "release_manifest_sha256": r[13], + "remote_session_id": r[14], } for r in rows ] @@ -258,7 +276,7 @@ def get_delivery(conn: sqlite3.Connection, delivery_id: str) -> dict[str, Any] | cursor.execute( """SELECT delivery_id, release_id, target_key, status, started_at, finished_at, total_items, committed_items, failed_items, skipped_items, last_error, created_at, - target_config_sha256, release_manifest_sha256 + target_config_sha256, release_manifest_sha256, remote_session_id FROM mesa_deliveries WHERE delivery_id = ?""", (delivery_id,), ) @@ -280,6 +298,7 @@ def get_delivery(conn: sqlite3.Connection, delivery_id: str) -> dict[str, Any] | "created_at": r[11], "target_config_sha256": r[12], "release_manifest_sha256": r[13], + "remote_session_id": r[14], } diff --git a/src/mesa_legal_data/publisher/models.py b/src/mesa_legal_data/publisher/models.py index 53f5752..f8118c8 100644 --- a/src/mesa_legal_data/publisher/models.py +++ b/src/mesa_legal_data/publisher/models.py @@ -34,11 +34,13 @@ class MesaTargetSettings(BaseModel): workspace_id: str = Field(default="legal") dataset_id: str = Field(default="tr_legislation") agent_id: str = Field(default="mesa_data_publisher") - content_limit_chars: int = Field(default=32768, ge=1024, le=1048576) + content_limit_chars: int = Field(default=32768, ge=1024, le=32768) contract_source: Literal["unknown", "configured", "live_verified"] = "unknown" - health_path: str = "" - publish_path: str = "" - mutation_status_path_template: str = "" + health_path: str = "/health" + session_start_path: str = "/v4/sessions/start" + publish_path: str = "/v4/memory/insert" + mutation_status_path_template: str = "/v4/mutations/{mutation_id}" + session_end_path_template: str = "/v4/sessions/{session_id}/end" api_key_configured: bool = Field(default=False) updated_at: str | None = None diff --git a/src/mesa_legal_data/web/api.py b/src/mesa_legal_data/web/api.py index 88ab70f..7df7261 100644 --- a/src/mesa_legal_data/web/api.py +++ b/src/mesa_legal_data/web/api.py @@ -2151,12 +2151,17 @@ def update_publisher_settings_endpoint(req: MesaTargetSettingsUpdateRequest, tar content_limit_chars=req.content_limit_chars, contract_source=( "configured" - if req.health_path and req.publish_path and "{mutation_id}" in req.mutation_status_path_template + if req.health_path + and req.session_start_path + and req.publish_path + and "{mutation_id}" in req.mutation_status_path_template else "unknown" ), health_path=req.health_path, + session_start_path=req.session_start_path, publish_path=req.publish_path, mutation_status_path_template=req.mutation_status_path_template, + session_end_path_template=req.session_end_path_template, ) upsert_mesa_target_settings(conn, new_settings) conn.close() diff --git a/src/mesa_legal_data/web/schemas.py b/src/mesa_legal_data/web/schemas.py index e1f12de..dcc82f3 100644 --- a/src/mesa_legal_data/web/schemas.py +++ b/src/mesa_legal_data/web/schemas.py @@ -84,10 +84,12 @@ class MesaTargetSettingsUpdateRequest(BaseModel): workspace_id: str = Field(min_length=1, max_length=100) dataset_id: str = Field(min_length=1, max_length=100) agent_id: str = Field(min_length=1, max_length=100) - content_limit_chars: int = Field(default=32768, ge=1024, le=1048576) - health_path: str = Field(default="", max_length=300, pattern=r"^(|/.*)$") - publish_path: str = Field(default="", max_length=300, pattern=r"^(|/.*)$") - mutation_status_path_template: str = Field(default="", max_length=300, pattern=r"^(|/.*)$") + content_limit_chars: int = Field(default=32768, ge=1024, le=32768) + health_path: str = Field(default="/health", max_length=300, pattern=r"^/.*$") + session_start_path: str = Field(default="/v4/sessions/start", max_length=300, pattern=r"^/.*$") + publish_path: str = Field(default="/v4/memory/insert", max_length=300, pattern=r"^/.*$") + mutation_status_path_template: str = Field(default="/v4/mutations/{mutation_id}", max_length=300, pattern=r"^/.*$") + session_end_path_template: str = Field(default="/v4/sessions/{session_id}/end", max_length=300, pattern=r"^/.*$") class MesaPublishRequest(BaseModel): diff --git a/src/mesa_legal_data/web/static/app.js b/src/mesa_legal_data/web/static/app.js index 4f5ca5e..97ca0c3 100644 --- a/src/mesa_legal_data/web/static/app.js +++ b/src/mesa_legal_data/web/static/app.js @@ -1512,8 +1512,10 @@ async function loadExportView() { const elAgent = document.getElementById("mesa-target-agent"); const elLimit = document.getElementById("mesa-target-limit"); const elHealthPath = document.getElementById("mesa-health-path"); + const elSessionStartPath = document.getElementById("mesa-session-start-path"); const elPublishPath = document.getElementById("mesa-publish-path"); const elMutationPath = document.getElementById("mesa-mutation-path"); + const elSessionEndPath = document.getElementById("mesa-session-end-path"); const badgeKey = document.getElementById("badge-mesa-key"); if (elUrl && targetSettings.base_url) elUrl.value = targetSettings.base_url; @@ -1523,8 +1525,10 @@ async function loadExportView() { if (elAgent && targetSettings.agent_id) elAgent.value = targetSettings.agent_id; if (elLimit && targetSettings.content_limit_chars) elLimit.value = targetSettings.content_limit_chars; if (elHealthPath) elHealthPath.value = targetSettings.health_path || ""; + if (elSessionStartPath) elSessionStartPath.value = targetSettings.session_start_path || ""; if (elPublishPath) elPublishPath.value = targetSettings.publish_path || ""; if (elMutationPath) elMutationPath.value = targetSettings.mutation_status_path_template || ""; + if (elSessionEndPath) elSessionEndPath.value = targetSettings.session_end_path_template || ""; if (badgeKey) { if (targetSettings.api_key_configured) { @@ -1634,8 +1638,10 @@ async function handleMesaSaveSettings() { const agent = document.getElementById("mesa-target-agent")?.value.trim(); const limit = parseInt(document.getElementById("mesa-target-limit")?.value || "32768", 10); const healthPath = document.getElementById("mesa-health-path")?.value.trim() || ""; + const sessionStartPath = document.getElementById("mesa-session-start-path")?.value.trim() || ""; const publishPath = document.getElementById("mesa-publish-path")?.value.trim() || ""; const mutationPath = document.getElementById("mesa-mutation-path")?.value.trim() || ""; + const sessionEndPath = document.getElementById("mesa-session-end-path")?.value.trim() || ""; if (!url || !tenant || !ws || !ds || !agent) { showToast("Lütfen tüm zorunlu hedef alanlarını doldurunuz.", "warning"); @@ -1655,8 +1661,10 @@ async function handleMesaSaveSettings() { agent_id: agent, content_limit_chars: limit, health_path: healthPath, + session_start_path: sessionStartPath, publish_path: publishPath, mutation_status_path_template: mutationPath, + session_end_path_template: sessionEndPath, }), }); showToast("MESA hedef ayarları kaydedildi.", "success"); diff --git a/src/mesa_legal_data/web/static/index.html b/src/mesa_legal_data/web/static/index.html index 90798b9..536962d 100644 --- a/src/mesa_legal_data/web/static/index.html +++ b/src/mesa_legal_data/web/static/index.html @@ -607,7 +607,7 @@
Madde 1- Yayınlanacak madde metni.
""" @@ -529,11 +529,15 @@ def test_master_j_release_bound_delivery_plan_and_cancellation(tmp_path, monkeyp dataset_id="tr_legislation", agent_id="publisher", contract_source="configured", - health_path="/v4/health", - publish_path="/v4/sources/chunks", + health_path="/health", + session_start_path="/v4/sessions/start", + publish_path="/v4/memory/insert", mutation_status_path_template="/v4/mutations/{mutation_id}", ) upsert_mesa_target_settings(conn, settings) + respx.post("https://mock-mesa.internal/v4/sessions/start").mock( + return_value=httpx.Response(201, json={"status": "started", "session_id": "sess-cancel"}) + ) ver = get_version_for_artifact(conn, art_id) assert ver is not None diff --git a/tests/integration/test_mesa_v4_publisher_e2e.py b/tests/integration/test_mesa_v4_publisher_e2e.py index e903e4a..bbb72ab 100644 --- a/tests/integration/test_mesa_v4_publisher_e2e.py +++ b/tests/integration/test_mesa_v4_publisher_e2e.py @@ -40,8 +40,9 @@ def setup_publisher_env(tmp_path, monkeypatch): dataset_id="tr_legislation", agent_id="publisher", contract_source="configured", - health_path="/v4/health", - publish_path="/v4/sources/chunks", + health_path="/health", + session_start_path="/v4/sessions/start", + publish_path="/v4/memory/insert", mutation_status_path_template="/v4/mutations/{mutation_id}", ) upsert_mesa_target_settings(conn, settings) @@ -146,10 +147,17 @@ def setup_publisher_env(tmp_path, monkeypatch): @respx.mock def test_full_delivery_success_and_cross_release_dedup(setup_publisher_env): # Mock MESA endpoints - respx.get("https://mock-mesa.internal/v4/health").respond(200, json={"status": "ok"}) - respx.post("https://mock-mesa.internal/v4/sources/chunks").respond( - 200, json={"mutation_id": "mut-101", "state": "COMMITTED", "message": "Committed directly"} + respx.get("https://mock-mesa.internal/health").respond(200, json={"status": "ok"}) + starts = respx.post("https://mock-mesa.internal/v4/sessions/start").respond( + 201, json={"status": "started", "session_id": "sess-run-1"} ) + inserts = respx.post("https://mock-mesa.internal/v4/memory/insert").respond( + 202, json={"mutation_id": "mut-101", "status": "accepted"} + ) + respx.get("https://mock-mesa.internal/v4/mutations/mut-101").respond( + 200, json={"mutation_id": "mut-101", "candidate_id": "cand", "state": "COMMITTED"} + ) + respx.post("https://mock-mesa.internal/v4/sessions/sess-run-1/end").respond(200, json={"status": "ended"}) # First delivery del1 = execute_publish_delivery(delivery_id="del-run-1") @@ -157,6 +165,8 @@ def test_full_delivery_success_and_cross_release_dedup(setup_publisher_env): assert del1["committed_items"] == 2 assert del1["skipped_items"] == 0 assert del1["failed_items"] == 0 + assert starts.call_count == 1 + assert inserts.call_count == 2 # Second delivery (Cross-release dedup check: same content must be SKIPPED) del2 = execute_publish_delivery(delivery_id="del-run-2") @@ -168,7 +178,10 @@ def test_full_delivery_success_and_cross_release_dedup(setup_publisher_env): @respx.mock def test_partial_failure_and_retry_workflow(setup_publisher_env): - respx.get("https://mock-mesa.internal/v4/health").respond(200, json={"status": "ok"}) + respx.get("https://mock-mesa.internal/health").respond(200, json={"status": "ok"}) + respx.post("https://mock-mesa.internal/v4/sessions/start").respond( + 201, json={"status": "started", "session_id": "sess-partial"} + ) # First call succeeds for chunk 1, fails for chunk 2 call_count = 0 @@ -177,12 +190,15 @@ def chunk_handler(request): nonlocal call_count call_count += 1 payload = json.loads(request.content) - if payload.get("ordinal") == 1: - return httpx.Response(200, json={"mutation_id": "mut-1", "state": "COMMITTED"}) + if payload.get("chunk_ordinal") == 1: + return httpx.Response(202, json={"mutation_id": "mut-1", "status": "accepted"}) else: return httpx.Response(500, text="Internal server error") - respx.post("https://mock-mesa.internal/v4/sources/chunks").mock(side_effect=chunk_handler) + respx.post("https://mock-mesa.internal/v4/memory/insert").mock(side_effect=chunk_handler) + respx.get("https://mock-mesa.internal/v4/mutations/mut-1").respond( + 200, json={"mutation_id": "mut-1", "candidate_id": "cand", "state": "COMMITTED"} + ) # 1. Delivery results in PARTIAL del_res = execute_publish_delivery(delivery_id="del-partial-1") @@ -191,9 +207,13 @@ def chunk_handler(request): assert del_res["failed_items"] == 1 # 2. Fix the server for retry - respx.post("https://mock-mesa.internal/v4/sources/chunks").mock( - return_value=httpx.Response(200, json={"mutation_id": "mut-2", "state": "COMMITTED"}) + respx.post("https://mock-mesa.internal/v4/memory/insert").mock( + return_value=httpx.Response(202, json={"mutation_id": "mut-2", "status": "accepted"}) ) + respx.get("https://mock-mesa.internal/v4/mutations/mut-2").respond( + 200, json={"mutation_id": "mut-2", "candidate_id": "cand", "state": "COMMITTED"} + ) + respx.post("https://mock-mesa.internal/v4/sessions/sess-partial/end").respond(200, json={"status": "ended"}) # 3. Retry only failed items retry_res = retry_delivery_failures(delivery_id="del-partial-1") @@ -205,19 +225,26 @@ def chunk_handler(request): @respx.mock def test_response_loss_retry_reuses_exact_idempotency_key(setup_publisher_env): - respx.get("https://mock-mesa.internal/v4/health").respond(200, json={"status": "ok"}) + respx.get("https://mock-mesa.internal/health").respond(200, json={"status": "ok"}) + respx.post("https://mock-mesa.internal/v4/sessions/start").respond( + 201, json={"status": "started", "session_id": "sess-loss"} + ) seen_keys = [] call_count = 0 def response_loss_then_commit(request): nonlocal call_count call_count += 1 - seen_keys.append(request.headers["Idempotency-Key"]) + seen_keys.append(json.loads(request.content)["idempotency_key"]) if call_count == 1: raise httpx.ReadTimeout("response lost after request was sent", request=request) - return httpx.Response(200, json={"mutation_id": "mut-after-loss", "state": "COMMITTED"}) + return httpx.Response(202, json={"mutation_id": "mut-after-loss", "status": "accepted"}) - respx.post("https://mock-mesa.internal/v4/sources/chunks").mock(side_effect=response_loss_then_commit) + respx.post("https://mock-mesa.internal/v4/memory/insert").mock(side_effect=response_loss_then_commit) + respx.get("https://mock-mesa.internal/v4/mutations/mut-after-loss").respond( + 200, json={"mutation_id": "mut-after-loss", "candidate_id": "cand", "state": "COMMITTED"} + ) + respx.post("https://mock-mesa.internal/v4/sessions/sess-loss/end").respond(200, json={"status": "ended"}) first = execute_publish_delivery(delivery_id="del-response-loss") assert first["status"] == "PARTIAL" diff --git a/tests/integration/test_post_master_independent_audit.py b/tests/integration/test_post_master_independent_audit.py index 173632d..0ca9f2f 100644 --- a/tests/integration/test_post_master_independent_audit.py +++ b/tests/integration/test_post_master_independent_audit.py @@ -916,15 +916,16 @@ def test_control_44_to_53_mesa_publisher_all_mutation_states_and_security(tmp_pa assert "MESA target host is not the explicitly allowed host" in err # Control 44: Auth failures 401/403 -> reachable=True, authenticated=False - respx.get("https://mock-mesa.internal/v4/health").mock( + respx.get("https://mock-mesa.internal/health").mock( return_value=httpx.Response(401, json={"detail": "Unauthorized"}) ) valid_settings = MesaTargetSettings( target_key="default", base_url="https://mock-mesa.internal", contract_source="configured", - health_path="/v4/health", - publish_path="/v4/sources/chunks", + health_path="/health", + session_start_path="/v4/sessions/start", + publish_path="/v4/memory/insert", mutation_status_path_template="/v4/mutations/{mutation_id}", ) client = MesaClient(valid_settings) @@ -970,9 +971,18 @@ def test_control_44_to_53_mesa_publisher_all_mutation_states_and_security(tmp_pa build_release(release_id=rel_id) # Mock health 200, publish 200 COMMITTED - respx.get("https://mock-mesa.internal/v4/health").mock(return_value=httpx.Response(200, json={"status": "ok"})) - respx.post("https://mock-mesa.internal/v4/sources/chunks").mock( - return_value=httpx.Response(200, json={"mutation_id": "mut-1", "state": "COMMITTED"}) + respx.get("https://mock-mesa.internal/health").mock(return_value=httpx.Response(200, json={"status": "ok"})) + respx.post("https://mock-mesa.internal/v4/sessions/start").mock( + return_value=httpx.Response(201, json={"status": "started", "session_id": "sess-audit"}) + ) + respx.post("https://mock-mesa.internal/v4/memory/insert").mock( + return_value=httpx.Response(202, json={"mutation_id": "mut-1", "status": "accepted"}) + ) + respx.get("https://mock-mesa.internal/v4/mutations/mut-1").mock( + return_value=httpx.Response(200, json={"mutation_id": "mut-1", "candidate_id": "cand", "state": "COMMITTED"}) + ) + respx.post("https://mock-mesa.internal/v4/sessions/sess-audit/end").mock( + return_value=httpx.Response(200, json={"status": "ended"}) ) del_id_1 = f"del-1-{uuid.uuid4().hex[:6]}" @@ -1040,8 +1050,13 @@ def test_control_44_to_53_mesa_publisher_all_mutation_states_and_security(tmp_pa conn.close() # Retry only retries the failed item - respx.post("https://mock-mesa.internal/v4/sources/chunks").mock( - return_value=httpx.Response(200, json={"mutation_id": "mut-retry", "state": "COMMITTED"}) + respx.post("https://mock-mesa.internal/v4/memory/insert").mock( + return_value=httpx.Response(202, json={"mutation_id": "mut-retry", "status": "accepted"}) + ) + respx.get("https://mock-mesa.internal/v4/mutations/mut-retry").mock( + return_value=httpx.Response( + 200, json={"mutation_id": "mut-retry", "candidate_id": "cand", "state": "COMMITTED"} + ) ) res_retry = retry_delivery_failures(del_partial_id) assert res_retry["status"] == DeliveryStatus.COMMITTED.value diff --git a/tests/integration/test_prompt4_integration_audit.py b/tests/integration/test_prompt4_integration_audit.py index 0ec58bb..d096ffc 100644 --- a/tests/integration/test_prompt4_integration_audit.py +++ b/tests/integration/test_prompt4_integration_audit.py @@ -717,8 +717,9 @@ def test_kontrol_7_8_9_publisher_contract_and_committed_truth(audit_env, monkeyp dataset_id="tr_legislation", agent_id="publisher", contract_source="configured", - health_path="/v4/health", - publish_path="/v4/sources/chunks", + health_path="/health", + session_start_path="/v4/sessions/start", + publish_path="/v4/memory/insert", mutation_status_path_template="/v4/mutations/{mutation_id}", ) upsert_mesa_target_settings(conn, settings) @@ -784,10 +785,17 @@ def test_kontrol_7_8_9_publisher_contract_and_committed_truth(audit_env, monkeyp conn.close() # Mock MESA HTTP endpoints - respx.get("https://mock-mesa.test/v4/health").respond(200, json={"status": "ok"}) - respx.post("https://mock-mesa.test/v4/sources/chunks").respond( - 200, json={"mutation_id": "mut-hmk-1", "state": "COMMITTED"} + respx.get("https://mock-mesa.test/health").respond(200, json={"status": "ok"}) + respx.post("https://mock-mesa.test/v4/sessions/start").respond( + 201, json={"status": "started", "session_id": "sess-hmk"} ) + respx.post("https://mock-mesa.test/v4/memory/insert").respond( + 202, json={"mutation_id": "mut-hmk-1", "status": "accepted"} + ) + respx.get("https://mock-mesa.test/v4/mutations/mut-hmk-1").respond( + 200, json={"mutation_id": "mut-hmk-1", "candidate_id": "cand", "state": "COMMITTED"} + ) + respx.post("https://mock-mesa.test/v4/sessions/sess-hmk/end").respond(200, json={"status": "ended"}) # 1. First publish -> COMMITTED del1 = execute_publish_delivery(delivery_id="del-audit-1") diff --git a/tests/integration/test_web_panel_publisher.py b/tests/integration/test_web_panel_publisher.py index 60c0e26..d9465fe 100644 --- a/tests/integration/test_web_panel_publisher.py +++ b/tests/integration/test_web_panel_publisher.py @@ -39,8 +39,9 @@ def client(tmp_path, monkeypatch): dataset_id="tr_legislation", agent_id="publisher", contract_source="configured", - health_path="/v4/health", - publish_path="/v4/sources/chunks", + health_path="/health", + session_start_path="/v4/sessions/start", + publish_path="/v4/memory/insert", mutation_status_path_template="/v4/mutations/{mutation_id}", ), ) @@ -147,9 +148,10 @@ def test_publisher_settings_and_preflight_endpoints(client): "workspace_id": "legal", "dataset_id": "tr_legislation", "agent_id": "publisher_v4", - "content_limit_chars": 65536, - "health_path": "/v4/health", - "publish_path": "/v4/sources/chunks", + "content_limit_chars": 32768, + "health_path": "/health", + "session_start_path": "/v4/sessions/start", + "publish_path": "/v4/memory/insert", "mutation_status_path_template": "/v4/mutations/{mutation_id}", }, ) @@ -157,7 +159,7 @@ def test_publisher_settings_and_preflight_endpoints(client): upd_data = res_post.json()["data"] assert upd_data["base_url"] == "https://mesa-updated.internal" assert upd_data["tenant_id"] == "corp" - assert upd_data["content_limit_chars"] == 65536 + assert upd_data["content_limit_chars"] == 32768 # 3. GET Ready Summary res_sum = client.get("/api/publisher/ready-summary", headers=headers) diff --git a/tests/unit/test_publisher_client_and_ledger.py b/tests/unit/test_publisher_client_and_ledger.py index 5c60df1..64d055a 100644 --- a/tests/unit/test_publisher_client_and_ledger.py +++ b/tests/unit/test_publisher_client_and_ledger.py @@ -1,3 +1,5 @@ +import json + import pytest import respx @@ -11,6 +13,7 @@ insert_delivery_item, is_chunk_already_committed, list_failed_delivery_items, + set_delivery_remote_session, update_delivery_progress, upsert_mesa_target_settings, ) @@ -45,6 +48,8 @@ def test_target_settings_persistence(db_conn): assert loaded.dataset_id == "turkey_laws" assert loaded.agent_id == "agent_mesa_01" assert loaded.content_limit_chars == 16384 + assert loaded.session_start_path == "/v4/sessions/start" + assert loaded.session_end_path_template == "/v4/sessions/{session_id}/end" def test_api_key_secrecy_and_isolation(monkeypatch): @@ -53,9 +58,10 @@ def test_api_key_secrecy_and_isolation(monkeypatch): client = MesaClient(settings=settings) assert client.is_api_key_configured is True - # Verify the client headers contain the secret + # MESA bootstrap API keys use the exact X-API-Key contract, never Bearer. headers = client._get_headers() - assert headers["Authorization"] == "Bearer secret_token_12345" + assert headers["X-API-Key"] == "secret_token_12345" + assert "Authorization" not in headers # Verify settings dump does NOT leak the API key dumped = settings.model_dump() @@ -86,11 +92,125 @@ def test_committed_truth_requires_explicit_committed(response_json): ) respx.post("https://mesa-contract.test/publish-contract").respond(200, json=response_json) - result = client.publish_source_chunk(chunk, "stable-key") + result = client.publish_source_chunk(chunk, "stable-key", session_id="session-1", finalize_revision=True) assert result["state"] != MutationState.COMMITTED.value +@respx.mock +def test_native_v4_session_and_insert_payload_are_exact(monkeypatch): + monkeypatch.setenv("MESA_DATA_MESA_ALLOWED_HOST", "mesa-contract.test") + settings = MesaTargetSettings(base_url="https://mesa-contract.test", contract_source="configured") + client = MesaClient(settings=settings, api_key="secret") + chunk = SourceChunk( + chunk_id="v1:chunk:3", + document_id="doc-1", + version_id="v1", + chunk_type="article", + title="Madde 3", + char_start=10, + char_end=22, + ordinal=3, + content="canonical text", + content_hash="content-hash", + metadata={"revision_number": 2, "source_url": "https://authority.test/doc-1"}, + ) + start = respx.post("https://mesa-contract.test/v4/sessions/start").respond( + 201, json={"status": "started", "session_id": "sess-1"} + ) + + def assert_insert(request): + assert request.headers["X-API-Key"] == "secret" + assert "Authorization" not in request.headers + assert "Idempotency-Key" not in request.headers + payload = json.loads(request.content) + assert payload == { + "session_id": "sess-1", + "dataset_id": "tr_legislation", + "document_id": "doc-1", + "revision_id": "v1", + "chunk_id": "v1:chunk:3", + "title": "Madde 3", + "source_ref": "https://authority.test/doc-1", + "content": "canonical text", + "evidence_span": "", + "revision_number": 2, + "chunk_ordinal": 3, + "finalize_revision": True, + "supersedes_revision_id": None, + "metadata": { + "revision_number": 2, + "source_url": "https://authority.test/doc-1", + "mesa_data_chunk_type": "article", + "mesa_data_char_start": 10, + "mesa_data_char_end": 22, + "mesa_data_content_hash": "content-hash", + }, + "idempotency_key": "stable-key", + } + return httpx.Response(202, json={"status": "accepted", "mutation_id": "mut-1"}) + + import httpx + + insert = respx.post("https://mesa-contract.test/v4/memory/insert").mock(side_effect=assert_insert) + assert client.start_session()["session_id"] == "sess-1" + result = client.publish_source_chunk(chunk, "stable-key", session_id="sess-1", finalize_revision=True) + assert result["state"] == MutationState.QUEUED.value + assert start.called and insert.called + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("ACCEPTED", "QUEUED"), + ("RECEIVED", "QUEUED"), + ("EXTRACTED", "PROCESSING"), + ("VALIDATED", "PROCESSING"), + ("SQL_APPLIED", "PROCESSING"), + ("VECTOR_APPLIED", "PROCESSING"), + ("GRAPH_APPLIED", "PROCESSING"), + ("RETRY_PENDING", "PROCESSING"), + ("COMMITTED", "COMMITTED"), + ("REJECTED", "REJECTED"), + ("DEAD_LETTER", "FAILED"), + ("BLOCKED", "FAILED"), + ("ROLLED_BACK", "FAILED"), + ("UNKNOWN_FUTURE_STATE", "FAILED"), + ], +) +def test_current_mesa_v4_mutation_state_normalization(raw, expected): + state, _ = MesaClient._normalize_mutation_state(raw) + assert state == expected + + +def test_multi_chunk_revision_finalizes_only_last_chunk(): + client = MesaClient(MesaTargetSettings(), api_key="secret") + chunks = [ + SourceChunk( + chunk_id=f"v1:chunk:{ordinal}", + document_id="doc-1", + version_id="v1", + chunk_type="general", + char_start=ordinal, + char_end=ordinal + 1, + ordinal=ordinal, + content=f"chunk {ordinal}", + content_hash=f"hash-{ordinal}", + ) + for ordinal in (1, 2, 3) + ] + payloads = [ + client.build_memory_insert_payload( + chunk, + session_id="sess-1", + idempotency_key=f"key-{chunk.ordinal}", + finalize_revision=chunk.ordinal == 3, + ) + for chunk in chunks + ] + assert [payload["finalize_revision"] for payload in payloads] == [False, False, True] + + def test_unknown_contract_never_guesses_routes(): client = MesaClient(settings=MesaTargetSettings(base_url="https://mesa-contract.test"), api_key="secret") assert client.is_contract_configured is False @@ -100,6 +220,7 @@ def test_unknown_contract_never_guesses_routes(): def test_delivery_ledger_and_cross_release_dedup(db_conn): delivery_id = "del-test-01" create_delivery(db_conn, delivery_id=delivery_id, release_id="rel-1", target_key="default", total_items=2) + set_delivery_remote_session(db_conn, delivery_id=delivery_id, remote_session_id="sess-persisted") insert_delivery_item( db_conn, @@ -142,6 +263,7 @@ def test_delivery_ledger_and_cross_release_dedup(db_conn): assert del_info["status"] == "PARTIAL" assert del_info["committed_items"] == 1 assert del_info["failed_items"] == 1 + assert del_info["remote_session_id"] == "sess-persisted" # Cross-release dedup check assert (