diff --git a/.gitignore b/.gitignore index e4d3908..d6f5566 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,17 @@ dist/ *.log *.jsonl *.egg-info/ + +# Trace-derived validation sets are local-only / private: generated corpora hold +# raw conversation content and must never be committed. The in-repo convenience +# dir is ignored; raw corpora also match the *.jsonl rule above. +.contextpilot_validation/ +# ...but committed SYNTHETIC fixtures (no real trace data) are kept. +!tests/fixtures/trace_validation/synthetic_cases.jsonl +!datasets/ +!datasets/provenance_linking/ +!datasets/provenance_linking/synthetic_v1.jsonl +!datasets/provenance_linking/synthetic_v1.jsonl.manifest.json */.DS_Store *.DS_Store diff --git a/__init__.py b/__init__.py index eac1f08..be18cf5 100644 --- a/__init__.py +++ b/__init__.py @@ -53,6 +53,11 @@ def _load_submodule(name: str, file_path: Path): _hermes_sanitizer_patched = False _bootstrap_attempted = False +# Cache for the directly-loaded hermes_opportunities canary modules. ``None`` +# means "not yet attempted"; ``False`` means "attempted and unavailable"; a dict +# means "loaded". +_canary_modules: Any = None + def _import_contextpilot_submodules(): global dedup_chat_completions @@ -217,6 +222,326 @@ def _write_telemetry(record: Dict[str, Any]) -> None: logger.debug("[ContextPilot] telemetry write skipped: %s", e) +def _iter_message_text(messages: List[Dict[str, Any]]): + """Yield text fragments from an LLM-bound payload for in-memory measurement. + + Used only to *size* the payload (chars / exact tokens). Fragments are never + stored or emitted -- callers consume them immediately to produce integer + counts, then discard them. + """ + for msg in messages: + if not isinstance(msg, dict): + continue + content = msg.get("content") + if isinstance(content, str): + yield content + elif isinstance(content, list): + for block in content: + if isinstance(block, str): + yield block + elif isinstance(block, dict): + text = block.get("text") + if isinstance(text, str): + yield text + inner = block.get("content") + if isinstance(inner, str): + yield inner + + +def _payload_chars(messages: List[Dict[str, Any]]) -> int: + """Total character count of an LLM-bound payload (metadata-only measure).""" + return sum(len(frag) for frag in _iter_message_text(messages)) + + +# Sentinel so the (possibly None) tokenizer is resolved at most once per process. +_exact_tokenizer_cache: Any = "unset" + + +def _get_exact_tokenizer(): + """Return a callable ``(text) -> int`` for EXACT token counting, or None. + + Optional and best-effort: an exact tokenizer is used only when a backend is + installed and not disabled. This never raises and never installs anything; + when no backend is available the caller records an ``unavailable`` status + rather than emitting a fake (chars/4) token count. + + Backend selection via ``CONTEXTPILOT_EXACT_TOKENIZER`` = ``tiktoken`` + (default) | ``off``. Background/accounting tasks must use tokenizer counts; + when a tokenizer is unavailable they record ``unavailable`` rather than + substituting a chars/4 proxy. The separate disable environment flag also + returns ``None`` immediately. + """ + + global _exact_tokenizer_cache + if _exact_tokenizer_cache != "unset": + return _exact_tokenizer_cache + _exact_tokenizer_cache = None + if os.environ.get("CONTEXTPILOT_DISABLE_EXACT_TOKENIZER") == "1": + return None + backend = os.environ.get("CONTEXTPILOT_EXACT_TOKENIZER", "tiktoken").lower() + if backend in ("off", "none", "disabled", "auto"): + return None + if backend == "tiktoken": + try: + import tiktoken # optional dependency; never a hard requirement + + encoding_name = os.environ.get( + "CONTEXTPILOT_TIKTOKEN_ENCODING", "cl100k_base" + ) + enc = tiktoken.get_encoding(encoding_name) + + def _count(text: str, _enc=enc) -> int: + return len(_enc.encode(text, disallowed_special=())) + + _count._backend = f"tiktoken:{encoding_name}" # type: ignore[attr-defined] + _exact_tokenizer_cache = _count + except Exception as e: # noqa: BLE001 - tokenizer is strictly optional + logger.debug("[ContextPilot] exact tokenizer unavailable: %s", e) + _exact_tokenizer_cache = None + return _exact_tokenizer_cache + + +def _measure_actual_tokens( + original_messages: List[Dict[str, Any]], + optimized_messages: List[Dict[str, Any]], +) -> Dict[str, Any]: + """Metadata-only EXACT before/after token measurement of the payload. + + Returns a dict carrying ``actual_token_status`` of ``available`` or + ``unavailable``. When unavailable (no exact tokenizer backend), it emits NO + token numbers -- callers must not substitute a chars/4 estimate for these + fields. Raw text is counted in-memory only and never stored. + """ + counter = _get_exact_tokenizer() + if counter is None: + return {"actual_token_status": "unavailable"} + try: + before = sum(counter(frag) for frag in _iter_message_text(original_messages)) + after = sum(counter(frag) for frag in _iter_message_text(optimized_messages)) + except Exception as e: # noqa: BLE001 - a measurement must never break optimization + logger.debug("[ContextPilot] exact token measurement failed: %s", e) + return {"actual_token_status": "unavailable"} + return { + "actual_token_status": "available", + "actual_tokenizer_backend": getattr(counter, "_backend", "unknown"), + "actual_tokens_before": before, + "actual_tokens_after": after, + "actual_tokens_saved": before - after, + } + + +def _load_canary_modules(): + """Load the hermes_opportunities canary modules without importing the + ``contextpilot`` package ``__init__``. + + ``from contextpilot.hermes_opportunities.* import ...`` would first execute + ``contextpilot/__init__.py``, which pulls in the pipeline / live-index stack + (numpy/scipy). Those are unavailable in the Hermes/plugin runtime, so the + package import fails and both canaries silently fall back to "off". Instead + we load the four pure-Python modules + (``models``/``privacy``/``prompt_dedup_canary``/``artifact_dedup_canary``) + directly from their files under a lightweight private package + (``_contextpilot_canary``) so their relative imports (``from .models``, + ``from .privacy``) resolve without touching the heavy package ``__init__``. + + Returns a dict with ``models``/``prompt_dedup_canary``/``artifact_dedup_canary`` + module objects, or ``None`` when the files cannot be loaded. + """ + global _canary_modules + if _canary_modules is not None: + return _canary_modules or None + + try: + pkg_name = "_contextpilot_canary" + ho_dir = _REPO_ROOT / "contextpilot" / "hermes_opportunities" + + pkg = sys.modules.get(pkg_name) + if pkg is None: + pkg_spec = _ilu.spec_from_loader(pkg_name, loader=None, is_package=True) + pkg = _ilu.module_from_spec(pkg_spec) + pkg.__path__ = [str(ho_dir)] + sys.modules[pkg_name] = pkg + + def _load(sub: str): + full = f"{pkg_name}.{sub}" + cached = sys.modules.get(full) + if cached is not None: + return cached + spec = _ilu.spec_from_file_location(full, str(ho_dir / f"{sub}.py")) + if spec is None or spec.loader is None: + raise ImportError(f"Cannot load {full}") + mod = _ilu.module_from_spec(spec) + # Register before exec so the canary modules' relative imports + # (``from .models``/``from .privacy``) resolve to these entries. + sys.modules[full] = mod + spec.loader.exec_module(mod) + return mod + + # Dependencies first: the canary modules import from these. + _load("models") + _load("privacy") + _canary_modules = { + "models": sys.modules[f"{pkg_name}.models"], + "prompt_dedup_canary": _load("prompt_dedup_canary"), + "artifact_dedup_canary": _load("artifact_dedup_canary"), + } + return _canary_modules + except Exception as e: # noqa: BLE001 - canary must never break requests + _canary_modules = False + logger.debug("[ContextPilot] canary modules unavailable: %s", e) + return None + + +def _classify_prompt_content_for_canary(text: str) -> str: + """Conservatively classify runtime system text for prompt-dedup canary. + + Runtime API payloads usually expose both system and skill instructions as + role='system' messages. The canary may only rewrite clearly skill-like text; + ordinary/unclear system content stays system_prompt and is therefore never + eligible for the same_type_skill_prompt_only canary class. + """ + low = text.lower() + stripped = low.lstrip() + if stripped.startswith("---") and "name:" in low[:300]: + return "skill_prompt" + # Runtime canary is stricter than the offline analyzer: only obvious skill + # documents whose leading text says "use this skill" are writable. Broader + # cues such as "available skills" remain system_prompt at runtime. + if "use this skill" in low[:500]: + return "skill_prompt" + return "system_prompt" + + +def _apply_prompt_dedup_canary_to_api_messages( + api_messages: List[Dict[str, Any]], *, salt: str = "contextpilot-runtime-prompt-dedup-v1" +): + """Apply the default-off skill-prompt canary to runtime API messages. + + This is a narrow adapter from Hermes/OpenAI-style messages to the analyzer + package's in-memory _LLMContent carrier. It mutates api_messages only when + CONTEXTPILOT_PROMPT_DEDUP_MODE=canary and the canary module replaces a + same_type_skill_prompt_only duplicate. User/assistant/tool and ordinary + system content are never passed as writable skill_prompt items. + """ + mods = _load_canary_modules() + if mods is None: + return None + _LLMContent = mods["models"]._LLMContent + apply_prompt_dedup_canary = mods["prompt_dedup_canary"].apply_prompt_dedup_canary + + llm_items = [] + message_indexes = [] + for idx, msg in enumerate(api_messages): + if not isinstance(msg, dict) or msg.get("role") != "system": + continue + content = msg.get("content") + if not isinstance(content, str): + continue + block_type = _classify_prompt_content_for_canary(content) + llm_items.append(_LLMContent(block_type=block_type, content=content)) + message_indexes.append(idx) + + if not llm_items: + return None + + result = apply_prompt_dedup_canary( + llm_items, + salt=salt, + min_block_chars=40, + ) + if result and result.mutated: + for item, idx in zip(llm_items, message_indexes): + if item.block_type == "skill_prompt": + api_messages[idx]["content"] = item.content + return result + + +# Telemetry class for the runtime artifact-dedup path. The analyzer module's +# ARTIFACT_DEDUP_CLASS is its own internal enum; the runtime path reports this +# stable, provenance-flavored class string in its telemetry/stats. +_ARTIFACT_DEDUP_RUNTIME_CLASS = "same_payload_exact_artifact_body" + + +def _apply_artifact_dedup_canary_to_api_messages( + api_messages: List[Dict[str, Any]], *, salt: str = "contextpilot-runtime-artifact-dedup-v1" +): + """Apply the default-off artifact-dedup canary to runtime API messages. + + This is a narrow adapter from Hermes/OpenAI-style messages to the analyzer + package's in-memory _LLMContent carrier. Only ``role=tool`` (mapped to + ``tool_result``) and ``role=assistant`` (mapped to ``assistant_context``) + messages are passed as mutable artifact bodies; user/system/skill content is + never scanned or rewritten. It mutates api_messages only when + CONTEXTPILOT_ARTIFACT_DEDUP_MODE=canary and the canary module replaces a + later exact-duplicate artifact body with a strictly shorter reference. + """ + mods = _load_canary_modules() + if mods is None: + return None + _LLMContent = mods["models"]._LLMContent + ArtifactSpanLink = mods["artifact_dedup_canary"].ArtifactSpanLink + apply_artifact_dedup_canary = mods["artifact_dedup_canary"].apply_artifact_dedup_canary + + llm_items = [] + message_indexes = [] + for idx, msg in enumerate(api_messages): + if not isinstance(msg, dict): + continue + role = msg.get("role") + if role == "tool": + block_type = "tool_result" + elif role == "assistant": + block_type = "assistant_context" + else: + continue + content = msg.get("content") + if not isinstance(content, str): + continue + llm_items.append(_LLMContent(block_type=block_type, content=content)) + message_indexes.append(idx) + + if not llm_items: + return None + + llm_index_by_message_index = {msg_idx: llm_idx for llm_idx, msg_idx in enumerate(message_indexes)} + span_links = [] + for msg_idx, msg in enumerate(api_messages): + raw_links = msg.get("contextpilot_span_links") if isinstance(msg, dict) else None + if isinstance(msg, dict): + msg.pop("contextpilot_span_links", None) + if not isinstance(raw_links, list): + continue + for raw in raw_links: + if not isinstance(raw, dict): + continue + try: + src_msg = int(raw["source_message_index"]) + tgt_msg = int(raw.get("target_message_index", msg_idx)) + span_links.append( + ArtifactSpanLink( + source_index=llm_index_by_message_index[src_msg], + source_start=int(raw["source_start"]), + source_end=int(raw["source_end"]), + target_index=llm_index_by_message_index[tgt_msg], + target_start=int(raw["target_start"]), + target_end=int(raw["target_end"]), + ) + ) + except (KeyError, TypeError, ValueError, IndexError): + continue + + result = apply_artifact_dedup_canary( + llm_items, + salt=salt, + min_block_chars=40, + span_links=span_links, + ) + if result and result.mutated: + for item, idx in zip(llm_items, message_indexes): + api_messages[idx]["content"] = item.content + return result + + def _reorder_docs(docs: List[str], alpha: float = 0.001) -> List[str]: global _intercept_index if len(docs) < 2: @@ -629,7 +954,26 @@ def _tool_chars(msgs): except Exception as e: logger.debug("[ContextPilot] Extract/reorder failed: %s", e) - # Step 5: Block-level dedup + # Step 5: Optional prompt-dedup canary (default off). This is the only + # runtime prompt mutation path and is limited to same_type_skill_prompt_only. + prompt_dedup_result = _apply_prompt_dedup_canary_to_api_messages(api_messages) + prompt_dedup_chars_saved = ( + prompt_dedup_result.chars_saved + if prompt_dedup_result is not None and prompt_dedup_result.mutated + else 0 + ) + + # Step 5b: Optional artifact-dedup canary (default off). The second + # runtime mutation path, limited to exact-duplicate tool_result / + # assistant_context artifact bodies (provenance-aware reference). + artifact_dedup_result = _apply_artifact_dedup_canary_to_api_messages(api_messages) + artifact_dedup_chars_saved = ( + artifact_dedup_result.chars_saved + if artifact_dedup_result is not None and artifact_dedup_result.mutated + else 0 + ) + + # Step 6: Block-level dedup sys_content = None for msg in api_messages: if isinstance(msg, dict) and msg.get("role") == "system": @@ -642,49 +986,114 @@ def _tool_chars(msgs): {"messages": api_messages}, system_content=sys_content, ) - turn_chars_saved = doc_chars_saved + dedup_result.chars_saved + turn_chars_saved = ( + doc_chars_saved + + dedup_result.chars_saved + + prompt_dedup_chars_saved + + artifact_dedup_chars_saved + ) self._total_chars_saved += turn_chars_saved + # Actual before/after of the full LLM-bound payload (chars). These are + # measured directly from the original input vs the optimized output, so + # they reflect the realized processed-payload delta -- not a duplicate + # opportunity count. Cheap (string length only); always computed. + payload_chars_before = _payload_chars(original_messages) + payload_chars_after = _payload_chars(api_messages) + payload_chars_saved = payload_chars_before - payload_chars_after + # Step 6: Cache for next turn self._cached_messages = copy.deepcopy(api_messages) self._cached_original_messages = original_messages if turn_chars_saved > 0: logger.info( - "[ContextPilot] Turn %d: saved %d chars (~%d tokens) | cumulative: %d chars (~%d tokens)", + "[ContextPilot] Turn %d: saved %d chars by processing | cumulative: %d chars", self._optimize_count, turn_chars_saved, - turn_chars_saved // 4, self._total_chars_saved, - self._total_chars_saved // 4, ) # Metadata-only telemetry so the monitor does not depend solely on # gateway log lines. No content, prompts, or tool payloads here. - _write_telemetry( - { - "ts": time.time(), - "type": "turn", - "session_hash": ( - _hash_text(str(self._session_id)) - if self._session_id is not None else None - ), - "turn": self._optimize_count, - "chars_saved": turn_chars_saved, - "tokens_saved": turn_chars_saved // 4, - "doc_chars_saved": doc_chars_saved, - "block_chars_saved": dedup_result.chars_saved, - "blocks_deduped": dedup_result.blocks_deduped, - "blocks_total": dedup_result.blocks_total, - "docs_deduped": self._total_docs_deduped, - "system_blocks_matched": dedup_result.system_blocks_matched, - "cumulative_chars_saved": self._total_chars_saved, - } + # + # Token fields use tokenizer measurements only: + # * ``actual_tokens_*`` come from a tokenizer backend and are present + # only when ``actual_token_status == "available"``. + # * When no tokenizer backend is configured or available, status is + # ``unavailable`` and no token numbers are emitted; background + # accounting must not substitute chars/4. + telemetry_record = { + "ts": time.time(), + "type": "turn", + "session_hash": ( + _hash_text(str(self._session_id)) + if self._session_id is not None else None + ), + "turn": self._optimize_count, + # Actual processed-payload char delta (doc + block dedup). + "chars_saved": turn_chars_saved, + # Actual before/after of the full LLM-bound payload (chars). + "payload_chars_before": payload_chars_before, + "payload_chars_after": payload_chars_after, + "payload_chars_saved": payload_chars_saved, + "doc_chars_saved": doc_chars_saved, + "block_chars_saved": dedup_result.chars_saved, + "prompt_dedup_mode": ( + prompt_dedup_result.mode if prompt_dedup_result is not None else "off" + ), + "prompt_dedup_class": ( + prompt_dedup_result.prompt_dedup_class + if prompt_dedup_result is not None else "same_type_skill_prompt_only" + ), + "prompt_dedup_blocks_replaced": ( + prompt_dedup_result.blocks_replaced + if prompt_dedup_result is not None and prompt_dedup_result.mutated else 0 + ), + "prompt_dedup_chars_saved": prompt_dedup_chars_saved, + "artifact_dedup_mode": ( + artifact_dedup_result.mode if artifact_dedup_result is not None else "off" + ), + "artifact_dedup_class": _ARTIFACT_DEDUP_RUNTIME_CLASS, + "artifact_dedup_blocks_replaced": ( + artifact_dedup_result.blocks_replaced + if artifact_dedup_result is not None and artifact_dedup_result.mutated else 0 + ), + "artifact_dedup_chars_saved": artifact_dedup_chars_saved, + "blocks_deduped": dedup_result.blocks_deduped, + "blocks_total": dedup_result.blocks_total, + "docs_deduped": self._total_docs_deduped, + "system_blocks_matched": dedup_result.system_blocks_matched, + "cumulative_chars_saved": self._total_chars_saved, + } + # Optional EXACT token measurement (only computed on a saving turn). + telemetry_record.update( + _measure_actual_tokens(original_messages, api_messages) ) + _write_telemetry(telemetry_record) return api_messages, { "chars_saved": turn_chars_saved, + "payload_chars_before": payload_chars_before, + "payload_chars_after": payload_chars_after, + "payload_chars_saved": payload_chars_saved, "doc_chars_saved": doc_chars_saved, "block_chars_saved": dedup_result.chars_saved, + "prompt_dedup_mode": ( + prompt_dedup_result.mode if prompt_dedup_result is not None else "off" + ), + "prompt_dedup_chars_saved": prompt_dedup_chars_saved, + "prompt_dedup_blocks_replaced": ( + prompt_dedup_result.blocks_replaced + if prompt_dedup_result is not None and prompt_dedup_result.mutated else 0 + ), + "artifact_dedup_mode": ( + artifact_dedup_result.mode if artifact_dedup_result is not None else "off" + ), + "artifact_dedup_chars_saved": artifact_dedup_chars_saved, + "artifact_dedup_blocks_replaced": ( + artifact_dedup_result.blocks_replaced + if artifact_dedup_result is not None and artifact_dedup_result.mutated else 0 + ), "blocks_deduped": dedup_result.blocks_deduped, "blocks_total": dedup_result.blocks_total, "docs_deduped": self._total_docs_deduped, @@ -720,11 +1129,10 @@ def on_session_end(self, session_id: str, messages: List[Dict[str, Any]]) -> Non self._compressor.on_session_end(session_id, messages) if self._total_chars_saved > 0: logger.info( - "[ContextPilot] Session %s: %d turns, %d chars saved (~%d tokens)", + "[ContextPilot] Session %s: %d turns, %d chars saved by processing", session_id, self._optimize_count, self._total_chars_saved, - self._total_chars_saved // 4, ) def on_session_reset(self) -> None: diff --git a/contextpilot/__init__.py b/contextpilot/__init__.py index a2523d7..3052c6d 100644 --- a/contextpilot/__init__.py +++ b/contextpilot/__init__.py @@ -17,70 +17,87 @@ >>> results = pipeline.run(queries=["What is AI?"]) See docs/reference/api.md for detailed documentation. -""" - -from .pipeline import ( - RAGPipeline, - RetrieverConfig, - OptimizerConfig, - InferenceConfig, - PipelineConfig, -) - -from .context_index import ( - ContextIndex, - IndexResult, -) - -from .context_ordering import ( - IntraContextOrderer, -) -from .server.live_index import ContextPilot - -from .dedup import ( - dedup_chat_completions, - dedup_responses_api, - DedupResult, -) - -from .api import optimize, optimize_batch +Imports are lazy (PEP 562): the heavy RAG stack (``pipeline`` -> ``context_index`` +-> ``scipy``) is only pulled in when one of its names is first accessed. This +keeps lightweight, dependency-free consumers -- such as the standalone token +monitor / provenance profiler in :mod:`contextpilot.hermes_opportunities` -- +importable inside minimal environments where SciPy and friends are absent. +""" +from __future__ import annotations -from .retriever import ( - BM25Retriever, - FAISSRetriever, - FAISS_AVAILABLE, - Mem0Retriever, - create_mem0_corpus_map, - MEM0_AVAILABLE, -) +import importlib +from typing import TYPE_CHECKING __version__ = "0.4.1" -__all__ = [ +# Map each public name to the submodule that defines it. Submodules are imported +# on first attribute access, so importing ``contextpilot`` (or any lightweight +# subpackage like ``hermes_opportunities``) never eagerly drags in SciPy/NumPy. +_LAZY_EXPORTS = { # High-level pipeline API - "RAGPipeline", - "RetrieverConfig", - "OptimizerConfig", - "InferenceConfig", - "PipelineConfig", + "RAGPipeline": ".pipeline", + "RetrieverConfig": ".pipeline", + "OptimizerConfig": ".pipeline", + "InferenceConfig": ".pipeline", + "PipelineConfig": ".pipeline", # Core components - "ContextIndex", - "IndexResult", - "IntraContextOrderer", - "ContextPilot", + "ContextIndex": ".context_index", + "IndexResult": ".context_index", + "IntraContextOrderer": ".context_ordering", + "ContextPilot": ".server.live_index", # Deduplication - "dedup_chat_completions", - "dedup_responses_api", - "DedupResult", + "dedup_chat_completions": ".dedup", + "dedup_responses_api": ".dedup", + "DedupResult": ".dedup", # Convenience functions - "optimize", - "optimize_batch", + "optimize": ".api", + "optimize_batch": ".api", # Retrievers - "BM25Retriever", - "FAISSRetriever", - "FAISS_AVAILABLE", - "Mem0Retriever", - "create_mem0_corpus_map", - "MEM0_AVAILABLE", -] + "BM25Retriever": ".retriever", + "FAISSRetriever": ".retriever", + "FAISS_AVAILABLE": ".retriever", + "Mem0Retriever": ".retriever", + "create_mem0_corpus_map": ".retriever", + "MEM0_AVAILABLE": ".retriever", +} + +__all__ = list(_LAZY_EXPORTS) + + +def __getattr__(name: str): + """Lazily resolve a public name to its (heavy) submodule on first access.""" + module_name = _LAZY_EXPORTS.get(name) + if module_name is None: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + module = importlib.import_module(module_name, __name__) + value = getattr(module, name) + globals()[name] = value # cache so subsequent lookups skip the import machinery + return value + + +def __dir__(): + return sorted(list(globals()) + __all__) + + +if TYPE_CHECKING: # pragma: no cover - import-time hints for type checkers only + from .api import optimize, optimize_batch + from .context_index import ContextIndex, IndexResult + from .context_ordering import IntraContextOrderer + from .dedup import DedupResult, dedup_chat_completions, dedup_responses_api + from .pipeline import ( + InferenceConfig, + OptimizerConfig, + PipelineConfig, + RAGPipeline, + RetrieverConfig, + ) + from .retriever import ( + FAISS_AVAILABLE, + MEM0_AVAILABLE, + BM25Retriever, + FAISSRetriever, + Mem0Retriever, + create_mem0_corpus_map, + ) + from .server.live_index import ContextPilot diff --git a/contextpilot/_openai_hook.py b/contextpilot/_openai_hook.py index a64e684..52fd039 100644 --- a/contextpilot/_openai_hook.py +++ b/contextpilot/_openai_hook.py @@ -336,13 +336,12 @@ def _optimize_messages(kwargs): if chars_saved > 0 or docs_reordered > 0: logger.info( "[ContextPilot] Call #%d: %d chars saved, %d blocks deduped, " - "%d docs reordered (cumulative: %d chars ≈ %d tokens)", + "%d docs reordered (cumulative: %d chars; tokenizer accounting required for tokens)", _total_calls, chars_saved, dedup_result.blocks_deduped, docs_reordered, _total_chars_saved, - _total_chars_saved // 4, ) @@ -369,12 +368,11 @@ def _optimize_responses(kwargs): if dedup_result.chars_saved > 0: logger.info( "[ContextPilot] Responses call #%d: %d chars saved, %d blocks deduped " - "(cumulative: %d chars ≈ %d tokens)", + "(cumulative: %d chars; tokenizer accounting required for tokens)", _total_calls, dedup_result.chars_saved, dedup_result.blocks_deduped, _total_chars_saved, - _total_chars_saved // 4, ) diff --git a/contextpilot/hermes_opportunities/__init__.py b/contextpilot/hermes_opportunities/__init__.py new file mode 100644 index 0000000..9ca912f --- /dev/null +++ b/contextpilot/hermes_opportunities/__init__.py @@ -0,0 +1,208 @@ +"""Privacy-safe Hermes context opportunity analyzer for ContextPilot. + +Unlike ``hermes_contextpilot_monitor.py`` (which never reads message bodies), +this analyzer *does* inspect message content and tool outputs in order to find +concrete token-reduction opportunities: exact duplicate tool outputs, repeated +line/block fingerprints, oversized tool outputs per tool, heavy sessions, and +ContextPilot telemetry coverage. + +It reads content only in-memory to compute salted hashes and aggregate +counters. Reports never contain raw message/tool text, system prompts, or raw +session ids -- only salted SHA-256 fingerprints and numeric aggregates. This +makes it safe to run continuously from a cron job and ship the reports. + +The package is split into focused modules: + +* :mod:`.models` -- privacy-safe dataclasses, tunables, in-memory carriers +* :mod:`.privacy` -- salted hashing + the forbidden-output guard +* :mod:`.db` -- read-only Hermes state-DB loaders +* :mod:`.telemetry` -- metadata-only ContextPilot telemetry parsing +* :mod:`.detection` -- content-aware redundancy detection +* :mod:`.dedup_ab` -- offline prompt-dedup A/B simulation (no mutation) +* :mod:`.prompt_dedup_canary` -- default-off runtime canary, skill-only +* :mod:`.routing` -- Worker Context Routing shadow mode (P0) +* :mod:`.aggregation` -- Parent Aggregation Artifacts shadow mode (P0) +* :mod:`.report` -- report assembly + serialization +* :mod:`.cli` -- command-line entry point + +Safety contract: everything here is reporting/measurement only **except** +:mod:`.prompt_dedup_canary`, which is the single default-off runtime mutation +path. That canary is limited to same-type skill-prompt exact duplicates and only +runs when explicitly enabled by environment variable. +""" +from __future__ import annotations + +from .aggregation import ( + ARTIFACT_KINDS, + DEFAULT_MIN_ARTIFACT_CHARS, + PARENT_AGGREGATION_SOURCE_TYPES, + analyze_parent_aggregation_artifacts, + classify_artifact_kind, +) +from .cli import main +from .dedup_ab import simulate_prompt_dedup_ab +from .db import ( + load_heavy_sessions, + load_llm_bound_content, + load_tool_messages, + total_input_tokens, +) +from .detection import ( + analyze_llm_bound_blocks, + detect_exact_duplicate_tool_outputs, + detect_prompt_duplicate_blocks, + detect_repeated_blocks, + summarize_tool_sizes, +) +from .models import ( + BLOCK_TYPES, + DEFAULT_LARGE_OUTPUT_CHARS, + DEFAULT_MIN_BLOCK_CHARS, + DEFAULT_MIN_BLOCK_REPEAT, + DEFAULT_TOP_N, + EST_CHARS_PER_TOKEN, + PROMPT_DEDUP_AB_CLASSES, + PROMPT_DEDUP_AB_REFERENCE_TEMPLATE, + PROMPT_DUPLICATE_BLOCK_TYPES, + ArtifactKindStat, + ArtifactSourceCount, + BlockTypeStat, + CrossTypeBlockGroup, + DuplicateToolOutput, + HeavySession, + OpportunityReport, + ParentAggregationArtifacts, + ParentAggregationGroup, + PromptDedupABClass, + PromptDedupABSimulation, + PromptDuplicateBlock, + PromptDuplicateShadow, + PromptDuplicateTypeCount, + ProvenanceProfile, + ProvenanceSourceStat, + RepeatedBlock, + RouterCandidateBlock, + RouterLabelCount, + RouterReasonCount, + TelemetryCoverage, + ToolSizeStat, + TypeCount, + UNKNOWN_SOURCE, + WorkerRoutingShadow, + _est_tokens, + _LLMContent, + _ToolMessage, +) +from .provenance import build_provenance_profile +from .privacy import ( + FORBIDDEN_OUTPUT_KEYS, + _assert_no_forbidden_keys, + _salt_fingerprint, + _salted_hash, +) +from .prompt_dedup_canary import ( + CANARY_DEDUP_CLASS, + DEFAULT_PROMPT_DEDUP_MODE, + PROMPT_DEDUP_CANARY_REFERENCE_TEMPLATE, + PROMPT_DEDUP_DISABLE_ENV, + PROMPT_DEDUP_MODE_ENV, + PROMPT_DEDUP_MODES, + SAFETY_DENYLIST, + PromptDedupCanaryResult, + apply_prompt_dedup_canary, + build_canary_telemetry_record, + resolve_prompt_dedup_mode, +) +from .report import build_report, write_report +from .routing import ( + ROUTER_LABELS, + _ROUTABLE_LABELS, + analyze_worker_routing_shadow, + classify_router_label, +) +from .telemetry import parse_telemetry +from .tokenizer import TokenizerBackend, resolve_tokenizer + +__all__ = [ + # tunables / enums + "DEFAULT_MIN_BLOCK_CHARS", + "DEFAULT_MIN_BLOCK_REPEAT", + "DEFAULT_LARGE_OUTPUT_CHARS", + "DEFAULT_TOP_N", + "DEFAULT_MIN_ARTIFACT_CHARS", + "EST_CHARS_PER_TOKEN", + "BLOCK_TYPES", + "PROMPT_DUPLICATE_BLOCK_TYPES", + "ROUTER_LABELS", + "ARTIFACT_KINDS", + "PARENT_AGGREGATION_SOURCE_TYPES", + "FORBIDDEN_OUTPUT_KEYS", + # prompt-dedup canary (runtime; default off) + "PROMPT_DEDUP_MODE_ENV", + "PROMPT_DEDUP_DISABLE_ENV", + "PROMPT_DEDUP_MODES", + "DEFAULT_PROMPT_DEDUP_MODE", + "CANARY_DEDUP_CLASS", + "PROMPT_DEDUP_CANARY_REFERENCE_TEMPLATE", + "SAFETY_DENYLIST", + # dataclasses + "DuplicateToolOutput", + "RepeatedBlock", + "TypeCount", + "BlockTypeStat", + "CrossTypeBlockGroup", + "ToolSizeStat", + "HeavySession", + "TelemetryCoverage", + "PromptDedupABClass", + "PromptDedupABSimulation", + "PromptDuplicateBlock", + "PromptDuplicateTypeCount", + "PromptDuplicateShadow", + "RouterLabelCount", + "RouterReasonCount", + "RouterCandidateBlock", + "WorkerRoutingShadow", + "ArtifactSourceCount", + "ParentAggregationGroup", + "ArtifactKindStat", + "ParentAggregationArtifacts", + "ProvenanceSourceStat", + "ProvenanceProfile", + "OpportunityReport", + # provenance profile (token-monitor view) + "UNKNOWN_SOURCE", + "build_provenance_profile", + # loaders + "load_tool_messages", + "load_llm_bound_content", + "load_heavy_sessions", + "total_input_tokens", + # telemetry + "parse_telemetry", + # detection + "detect_exact_duplicate_tool_outputs", + "detect_repeated_blocks", + "summarize_tool_sizes", + "analyze_llm_bound_blocks", + "detect_prompt_duplicate_blocks", + "simulate_prompt_dedup_ab", + "TokenizerBackend", + "resolve_tokenizer", + # prompt-dedup canary (runtime; default off) + "PromptDedupCanaryResult", + "resolve_prompt_dedup_mode", + "apply_prompt_dedup_canary", + "build_canary_telemetry_record", + # routing (shadow) + "classify_router_label", + "analyze_worker_routing_shadow", + # aggregation (shadow) + "classify_artifact_kind", + "analyze_parent_aggregation_artifacts", + # report + "build_report", + "write_report", + # cli + "main", +] diff --git a/contextpilot/hermes_opportunities/aggregation.py b/contextpilot/hermes_opportunities/aggregation.py new file mode 100644 index 0000000..8f59adc --- /dev/null +++ b/contextpilot/hermes_opportunities/aggregation.py @@ -0,0 +1,312 @@ +"""Parent Aggregation Artifacts — SHADOW MODE (P0 telemetry only). + +When a parent/orchestrator aggregates results from several workers, the same +artifact body (a test log, a diff, a file dump, a review summary, ...) is often +carried into the parent's LLM context once per worker and again in the parent's +own roll-up -- paying for the same tokens several times. This section collects +*telemetry only* so a future parent-aggregation dedup can be evaluated offline: +it groups EXACT artifact bodies by salted content hash, classifies each body +with a deterministic heuristic kind, and emits low-cardinality metadata + +counters. It NEVER drops, summarizes, replaces, or mutates any context, and it +NEVER emits raw artifact text, worker text, tool output, session ids, or +system prompts. +""" +from __future__ import annotations + +from typing import Iterable + +from .models import ( + ArtifactKindStat, + ArtifactSourceCount, + ParentAggregationArtifacts, + ParentAggregationGroup, + _est_tokens, + _LLMContent, +) +from .privacy import _salted_hash + +# Heuristic P0 artifact kinds. Low-cardinality enums describing the *shape* of an +# aggregation artifact, never its text. Classification is deterministic. +ARTIFACT_KINDS = ( + "test_log", + "terminal_output", + "file_content", + "diff", + "error_trace", + "review_findings", + "benchmark_result", + "worker_summary", + "unknown_large_block", +) + +# Conservative floor: only sizeable blocks are treated as candidate aggregation +# artifacts, so short prompts/hints never enter parent-aggregation telemetry. +DEFAULT_MIN_ARTIFACT_CHARS = 400 + +# Parent aggregation P0 focuses on content produced by workers/tools and then +# carried into the parent context. System/skill/user prompts are analyzed by the +# LLM-bound redundancy and worker-routing sections, but excluding them here keeps +# parent artifact telemetry from being polluted by prompt boilerplate. +PARENT_AGGREGATION_SOURCE_TYPES = ("assistant_context", "tool_result") + + +def classify_artifact_kind(content: str) -> str: + """Deterministically classify a candidate aggregation artifact body. + + Pure P0 heuristic over in-memory text; returns a low-cardinality enum from + ``ARTIFACT_KINDS`` and never the text. The check order is fixed so the same + body always yields the same kind (first match wins). + """ + low = content.lower() + stripped = content.lstrip() + + # 1. Unified diff / patch. + if ( + stripped.startswith("diff --git") + or stripped.startswith("--- a/") + or stripped.startswith("@@ ") + or "\n@@ " in content + or ("\n--- " in content and "\n+++ " in content) + ): + return "diff" + + # 2. Test/pytest log (checked before error_trace: a failing test log may + # embed a traceback but is still fundamentally a test log). + if ( + "pytest" in low + or "test session starts" in low + or " passed in " in low + or " failed in " in low + or ("passed" in low and "failed" in low) + or "=== " in content + ): + return "test_log" + + # 3. Error / exception trace. + if ( + "traceback (most recent call last)" in low + or "\n at " in content + or "stack trace" in low + or ("exception" in low and "error" in low) + ): + return "error_trace" + + # 4. Benchmark / perf result. + if ( + "benchmark" in low + or "ops/sec" in low + or "ops/s" in low + or "req/sec" in low + or "throughput" in low + or "latency" in low + or "iterations/sec" in low + ): + return "benchmark_result" + + # 5. Code-review findings. + if ( + "code review" in low + or "review findings" in low + or "severity:" in low + or "vulnerab" in low + or "## findings" in low + ): + return "review_findings" + + # 6. File content / source dump (cat -n style numbering or code cues). + if ( + "\n 1\t" in content + or "\n 1\t" in content + or "def " in content + or "class " in content + or "\nimport " in content + or "#include" in content + or "function " in content + ): + return "file_content" + + # 7. Worker / aggregation summary. Checked after source-code cues so files + # mentioning workers are still labeled as file_content. + if ( + "## summary" in low + or "in summary" in low + or "summary:" in low + or "tl;dr" in low + or "aggregat" in low + or "worker" in low + ): + return "worker_summary" + + # 8. Terminal / shell session output. + if ( + "\n$ " in content + or stripped.startswith("$ ") + or "\n# " in content + or "user@" in low + or "bash-" in low + or "exit code" in low + ): + return "terminal_output" + + # 9. Fallback: a large block we could not confidently classify. + return "unknown_large_block" + + +def analyze_parent_aggregation_artifacts( + contents: Iterable[_LLMContent], + *, + salt: str, + min_artifact_chars: int, + top_n: int, + enabled: bool = True, +) -> ParentAggregationArtifacts: + """Group EXACT aggregation-artifact bodies and emit provenance telemetry. + + P0 telemetry/advisory only: no context is dropped, summarized, replaced, or + mutated. Each sizeable LLM-bound block is fingerprinted by EXACT salted + content hash (near-duplicates never group), classified with a deterministic + heuristic kind, and rolled up into low-cardinality metadata + counters. + ``est_duplicate_tokens`` is an advisory upper bound on what a *future* parent + dedup might save -- never a realized saving. No raw artifact/worker/tool/ + system text, and no raw session ids, are ever emitted. + """ + if not enabled: + return ParentAggregationArtifacts( + enabled=False, + item_count=0, + artifact_body_count=0, + total_occurrences=0, + duplicate_group_count=0, + est_total_tokens=0, + est_duplicate_tokens=0, + by_kind=[], + source_type_counts=[], + top_duplicate_groups=[], + notes=["parent-aggregation artifact analysis disabled via flag"], + ) + + # --- group sizeable bodies by EXACT salted content hash ---------------- + groups: dict[str, dict] = {} + item_count = 0 + source_totals: dict[str, int] = {} + for item in contents: + content = item.content + bt = item.block_type + if bt not in PARENT_AGGREGATION_SOURCE_TYPES: + continue + if not content or len(content) < min_artifact_chars: + continue + item_count += 1 + source_totals[bt] = source_totals.get(bt, 0) + 1 + h = _salted_hash(content, salt) + g = groups.get(h) + if g is None: + groups[h] = { + "char_length": len(content), + "occurrences": 1, + "sources": {bt: 1}, + # classify once from in-memory text; never stored/emitted. + "kind": classify_artifact_kind(content), + } + else: + g["occurrences"] += 1 + g["sources"][bt] = g["sources"].get(bt, 0) + 1 + + # --- per-kind rollup + per-group records ------------------------------- + kind_agg: dict[str, dict] = {} + group_records: list[ParentAggregationGroup] = [] + total_occurrences = 0 + est_total_tokens = 0 + est_duplicate_tokens = 0 + duplicate_group_count = 0 + + for h, g in groups.items(): + occ = g["occurrences"] + char_len = g["char_length"] + est = _est_tokens(char_len) + dup_tokens = est * (occ - 1) + kind = g["kind"] + is_dup = occ >= 2 + + total_occurrences += occ + est_total_tokens += est + est_duplicate_tokens += dup_tokens + if is_dup: + duplicate_group_count += 1 + + ka = kind_agg.setdefault( + kind, + {"groups": 0, "occ": 0, "dups": 0, "est": 0, "dup_tokens": 0}, + ) + ka["groups"] += 1 + ka["occ"] += occ + ka["est"] += est + ka["dup_tokens"] += dup_tokens + if is_dup: + ka["dups"] += 1 + + if is_dup: + # Provenance counts, sorted by source_type for determinism. + source_counts = [ + ArtifactSourceCount(source_type=st, count=c) + for st, c in sorted(g["sources"].items()) + ] + # Canonical source: dominant origin, tie-broken alphabetically. + canonical = min( + g["sources"].items(), key=lambda kv: (-kv[1], kv[0]) + )[0] + group_records.append( + ParentAggregationGroup( + content_hash=h, + artifact_kind=kind, + canonical_source_type=canonical, + occurrences=occ, + char_length=char_len, + est_tokens=est, + est_duplicate_tokens=dup_tokens, + source_type_counts=source_counts, + ) + ) + + by_kind = [ + ArtifactKindStat( + artifact_kind=kind, + group_count=kind_agg[kind]["groups"], + occurrence_count=kind_agg[kind]["occ"], + duplicate_group_count=kind_agg[kind]["dups"], + est_tokens=kind_agg[kind]["est"], + est_duplicate_tokens=kind_agg[kind]["dup_tokens"], + ) + for kind in ARTIFACT_KINDS + if kind in kind_agg + ] + source_type_counts = [ + ArtifactSourceCount(source_type=st, count=c) + for st, c in sorted(source_totals.items()) + ] + group_records.sort( + key=lambda g: (g.est_duplicate_tokens, g.occurrences, g.content_hash), + reverse=True, + ) + + notes = [ + "SHADOW MODE P0: telemetry only -- no aggregation artifact was deduped, replaced, summarized, or mutated", + "artifact_kind/source_type/canonical_source_type are low-cardinality enums; content_hash is a salted SHA-256 fingerprint", + "grouping is EXACT (same salted content hash): near-duplicate artifacts never group", + "est_duplicate_tokens is ADVISORY ((occurrences-1) * est_tokens), an upper bound for a FUTURE parent dedup -- not a realized saving", + "provenance source_type_counts show how many copies came from each parent/worker output origin (assistant_context, tool_result)", + ] + + return ParentAggregationArtifacts( + enabled=True, + item_count=item_count, + artifact_body_count=len(groups), + total_occurrences=total_occurrences, + duplicate_group_count=duplicate_group_count, + est_total_tokens=est_total_tokens, + est_duplicate_tokens=est_duplicate_tokens, + by_kind=by_kind, + source_type_counts=source_type_counts, + top_duplicate_groups=group_records[:top_n], + notes=notes, + ) diff --git a/contextpilot/hermes_opportunities/artifact_dedup_canary.py b/contextpilot/hermes_opportunities/artifact_dedup_canary.py new file mode 100644 index 0000000..fca203f --- /dev/null +++ b/contextpilot/hermes_opportunities/artifact_dedup_canary.py @@ -0,0 +1,599 @@ +"""Default-OFF provenance-aware tool-artifact reuse canary. + +This is the *second* runtime mutation path in ContextPilot and, like +:mod:`.prompt_dedup_canary`, it is default-OFF and narrowly scoped. Where the +prompt-dedup canary rewrites duplicate ``skill_prompt`` *lines*, this canary +dedups whole **artifact bodies** carried by ``tool_result`` and +``assistant_context`` items: it keeps the FIRST full artifact body verbatim and +replaces a later EXACT duplicate body -- regardless of which of the two mutable +artifact block types it appears in (provenance-aware) -- with a deterministic, +strictly shorter reference string that records the canonical body's provenance +(type + salted hash). + +Risk gate (all conditions must hold before a single character is changed): + +* Mode must be ``canary``. The mode is read from + ``CONTEXTPILOT_ARTIFACT_DEDUP_MODE`` (``off`` | ``shadow`` | ``canary``) and + defaults to ``off``. ``off`` and ``shadow`` never mutate the payload. +* The escape-hatch env ``CONTEXTPILOT_ARTIFACT_DEDUP_DISABLE`` (any truthy + value) forces ``off`` regardless of the mode var -- an immediate kill switch. +* Only ``tool_result`` / ``assistant_context`` artifact bodies are mutable. + ``system_prompt`` / ``user_prompt`` / ``skill_prompt`` / ``unknown`` (and any + other non-artifact content) are protected and never scanned or rewritten. +* Only EXACT duplicate full bodies are eligible. The first occurrence (within + the mutable artifact types) is the canonical body and is always kept verbatim; + only later exact occurrences are replaced, and only when the deterministic + reference string is strictly shorter than the body it replaces (never grows + the payload). +* A reference may only point at an EARLIER canonical full body in the same + payload; :func:`dangling_artifact_references` exposes a check used by the + validation gate to reject a dangling reference. + +The reference string carries only a low-cardinality artifact-type enum and a +salted body hash -- never raw artifact content. Telemetry is metadata-only: +mode/class enums and integer counters; no artifact text and no realized-savings +claim unless an actual mutation occurred. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Iterable + +from .models import _LLMContent +from .privacy import _assert_no_forbidden_keys, _salted_hash + +# Environment controls. ``off`` is the default and the safe state. +ARTIFACT_DEDUP_MODE_ENV = "CONTEXTPILOT_ARTIFACT_DEDUP_MODE" +ARTIFACT_DEDUP_DISABLE_ENV = "CONTEXTPILOT_ARTIFACT_DEDUP_DISABLE" +ARTIFACT_DEDUP_MODES = ("off", "shadow", "canary") +DEFAULT_ARTIFACT_DEDUP_MODE = "off" + +# The only block types whose artifact bodies this canary may dedup. Both are +# mutable; the duplicate may span them (provenance-aware, cross-type). +MUTABLE_ARTIFACT_BLOCK_TYPES = ("tool_result", "assistant_context") + +# The only duplicate class this canary acts on: an exact-duplicate full artifact +# body across the mutable artifact types. +ARTIFACT_DEDUP_CLASS = "same_payload_exact_artifact_body" +ARTIFACT_SPAN_PROVENANCE_CLASS = "declared_source_span_backref" + +# Deterministic placeholder left in place of a later duplicate body. ```` +# is the CANONICAL (first) body's provenance and ```` its salted +# fingerprint -- both low-cardinality, never raw artifact content. +ARTIFACT_DEDUP_CANARY_REFERENCE_TEMPLATE = ( + "[ContextPilot artifact dedup: duplicate artifact body omitted; " + "ref=:]" +) + +# Fixed head of the reference string (everything before the first placeholder), +# used to recognize a reference line without re-rendering it. +_REF_HEAD = ARTIFACT_DEDUP_CANARY_REFERENCE_TEMPLATE.split("", 1)[0] + + +@dataclass +class ArtifactSpanLink: + """Declared source-span provenance edge, in Python string offsets. + + The canary treats this as untrusted metadata: it rewrites only when the + declared target slice byte-equals the earlier source slice and all scope / + line-alignment / never-grow gates pass. + """ + + source_index: int + source_start: int + source_end: int + target_index: int + target_start: int + target_end: int + + +@dataclass +class ArtifactDedupCanaryResult: + """Metadata-only outcome of an artifact-dedup canary pass. No raw text, ever. + + ``chars_saved`` / ``blocks_replaced`` are REALIZED figures and are non-zero + only when ``mode == 'canary'`` and an actual replacement occurred. The + ``candidate_*`` fields are advisory (what a canary *would* replace) and are + populated in ``shadow`` mode for visibility without mutating anything. + """ + + mode: str # off | shadow | canary + artifact_dedup_class: str # always ARTIFACT_DEDUP_CLASS + mutated: bool # True only if a real replacement happened + item_count: int # mutable artifact items scanned + candidate_group_count: int # eligible exact-duplicate body groups + candidate_chars: int # advisory chars later occurrences occupy + blocks_replaced: int # REALIZED replacements (canary only) + chars_saved: int # REALIZED chars saved (canary only) + span_candidate_count: int = 0 # advisory declared span replacements + span_candidate_chars: int = 0 # advisory chars later declared spans occupy + span_blocks_replaced: int = 0 # REALIZED source-span replacements + span_chars_saved: int = 0 # REALIZED source-span chars saved + notes: list[str] = field(default_factory=list) + + +def _truthy(value: str | None) -> bool: + return bool(value) and value.strip().lower() not in ("", "0", "false", "no", "off") + + +def resolve_artifact_dedup_mode(env: dict | None = None) -> str: + """Resolve the active artifact-dedup mode, defaulting to the safe ``off``. + + Unknown values fall back to ``off``. The escape-hatch disable variable, when + truthy, forces ``off`` regardless of the mode variable. + """ + source = os.environ if env is None else env + if _truthy(source.get(ARTIFACT_DEDUP_DISABLE_ENV)): + return "off" + raw = ( + source.get(ARTIFACT_DEDUP_MODE_ENV) or DEFAULT_ARTIFACT_DEDUP_MODE + ).strip().lower() + return raw if raw in ARTIFACT_DEDUP_MODES else DEFAULT_ARTIFACT_DEDUP_MODE + + +def _artifact_reference_string(canonical_type: str, body_hash: str) -> str: + """Render the reference that points at a canonical artifact body. + + Carries only the canonical provenance enum and the salted hash -- never the + artifact body itself. + """ + return ARTIFACT_DEDUP_CANARY_REFERENCE_TEMPLATE.replace( + "", canonical_type + ).replace("", body_hash) + + +def _parse_artifact_reference(line: str) -> str | None: + """Return the salted hash a reference line encodes, or ``None`` if not one.""" + if not (line.startswith(_REF_HEAD) and "ref=" in line and line.endswith("]")): + return None + after = line.rsplit("ref=", 1)[1][:-1] # strip the trailing ']' + _type, sep, body_hash = after.partition(":") + if not sep or not body_hash: + return None + return body_hash + + +def _segment_fenced_blocks(body: str) -> list[tuple[str, str]]: + """Split ``body`` into reversible prose/fence segments. + + Only closed triple-backtick fences are marked as ``"fence"``. Unterminated + fences are deliberately treated as prose so the canary never guesses a block + boundary. Concatenating the segment text always reproduces ``body`` exactly. + """ + segments: list[tuple[str, str]] = [] + pos = 0 + n = len(body) + while pos < n: + start = body.find("```", pos) + if start == -1: + if pos < n: + segments.append(("prose", body[pos:])) + break + close = body.find("```", start + 3) + if close == -1: + if pos < n: + segments.append(("prose", body[pos:])) + break + if start > pos: + segments.append(("prose", body[pos:start])) + end = close + 3 + segments.append(("fence", body[start:end])) + pos = end + return segments + + +def _scan_fenced_block_candidates( + contents: list[_LLMContent], *, salt: str, min_block_chars: int +) -> tuple[int, int]: + """Advisory duplicate count for exact fenced sub-artifacts.""" + agg: dict[str, dict] = {} + for item in contents: + if item.block_type not in MUTABLE_ARTIFACT_BLOCK_TYPES: + continue + # Whole-body references are not canonical sources for sub-blocks. + if _parse_artifact_reference(item.content) is not None: + continue + for kind, text in _segment_fenced_blocks(item.content): + if kind != "fence" or len(text) < min_block_chars: + continue + if _parse_artifact_reference(text) is not None: + continue + h = _salted_hash(text, salt) + entry = agg.get(h) + if entry is None: + agg[h] = {"canonical_type": f"{item.block_type}#block", "char_length": len(text), "occ": 1} + else: + entry["occ"] += 1 + return _eligible_groups(agg) + + +def _in_range(text: str, start: int, end: int) -> bool: + return 0 <= start < end <= len(text) + + +def _line_aligned(text: str, start: int, end: int) -> bool: + """Require standalone line spans so emitted refs are standalone lines.""" + return ( + _in_range(text, start, end) + and (start == 0 or text[start - 1] == "\n") + and (end == len(text) or text[end] == "\n") + ) + + +def _valid_span_link( + items: list[_LLMContent], link: ArtifactSpanLink, *, min_block_chars: int, salt: str +) -> tuple[str, str, str] | None: + """Return ``(target_text, ref, source_hash)`` if a declared link is safe.""" + if not (0 <= link.source_index < len(items) and 0 <= link.target_index < len(items)): + return None + if link.source_index >= link.target_index: + return None + source = items[link.source_index] + target = items[link.target_index] + if source.block_type != "tool_result" or target.block_type != "assistant_context": + return None + if not _line_aligned(source.content, link.source_start, link.source_end): + return None + if not _line_aligned(target.content, link.target_start, link.target_end): + return None + source_text = source.content[link.source_start:link.source_end] + target_text = target.content[link.target_start:link.target_end] + if len(target_text) < min_block_chars or target_text != source_text: + return None + h = _salted_hash(source_text, salt) + ref = _artifact_reference_string(f"{source.block_type}#span", h) + if len(ref) >= len(target_text): + return None + return target_text, ref, h + + +def _scan_span_candidates( + items: list[_LLMContent], span_links: Iterable[ArtifactSpanLink], *, salt: str, min_block_chars: int +) -> tuple[int, int]: + count = 0 + chars = 0 + seen_targets: set[tuple[int, int, int]] = set() + for link in span_links: + valid = _valid_span_link(items, link, min_block_chars=min_block_chars, salt=salt) + key = (link.target_index, link.target_start, link.target_end) + if valid is None or key in seen_targets: + continue + seen_targets.add(key) + target_text, _ref, _h = valid + count += 1 + chars += len(target_text) + return count, chars + + +def _apply_span_links( + items: list[_LLMContent], span_links: Iterable[ArtifactSpanLink], *, salt: str, min_block_chars: int +) -> tuple[int, int, set[int], set[int]]: + """Apply safe declared source-span replacements right-to-left per target.""" + by_target: dict[int, dict[tuple[int, int], tuple[int, int, str, int, int]]] = {} + for link in span_links: + valid = _valid_span_link(items, link, min_block_chars=min_block_chars, salt=salt) + if valid is None: + continue + target_text, ref, _h = valid + by_target.setdefault(link.target_index, {})[(link.target_start, link.target_end)] = ( + link.target_start, link.target_end, ref, len(target_text), link.source_index + ) + + blocks = 0 + saved = 0 + mutated_targets: set[int] = set() + preserved_sources: set[int] = set() + for target_index, replacement_map in by_target.items(): + # First-cut validation scope certifies one line-aligned source-span swap + # per target body. Keep multi-span targets in shadow/advisory until the + # gate can prove several replacements in one message. + if len(replacement_map) != 1: + continue + replacements = list(replacement_map.values()) + replacements.sort(key=lambda r: r[0], reverse=True) + ordered = sorted(replacements, key=lambda r: r[0]) + if any(a[1] > b[0] for a, b in zip(ordered, ordered[1:])): + continue + body = items[target_index].content + for start, end, ref, old_len, source_index in replacements: + body = body[:start] + ref + body[end:] + blocks += 1 + saved += old_len - len(ref) + preserved_sources.add(source_index) + items[target_index].content = body + mutated_targets.add(target_index) + return blocks, saved, mutated_targets, preserved_sources + + +def _scan_artifacts( + contents: list[_LLMContent], *, salt: str, min_block_chars: int +) -> tuple[dict[str, dict], int]: + """Fingerprint mutable artifact bodies in order. + + Returns ``(agg, item_count)`` where ``agg`` maps a body hash to + ``{canonical_type, char_length, occ}`` (``canonical_type`` is the FIRST + occurrence's provenance) and ``item_count`` is the number of mutable + artifact items seen. + """ + agg: dict[str, dict] = {} + item_count = 0 + for item in contents: + if item.block_type not in MUTABLE_ARTIFACT_BLOCK_TYPES: + continue + item_count += 1 + body = item.content + if len(body) < min_block_chars: + continue + # A reference left by an earlier pass is not itself a canonical body. + if _parse_artifact_reference(body) is not None: + continue + h = _salted_hash(body, salt) + entry = agg.get(h) + if entry is None: + agg[h] = { + "canonical_type": item.block_type, + "char_length": len(body), + "occ": 1, + } + else: + entry["occ"] += 1 + return agg, item_count + + +def _eligible_groups(agg: dict[str, dict]) -> tuple[int, int]: + """Advisory measurement of duplicate body groups that would actually shrink. + + Returns ``(candidate_group_count, candidate_chars)`` where ``candidate_chars`` + is the chars the later (replaceable) occurrences currently occupy. + """ + group_count = 0 + candidate_chars = 0 + for h, entry in agg.items(): + if entry["occ"] < 2: + continue # not a duplicate -> nothing to replace + ref = _artifact_reference_string(entry["canonical_type"], h) + if len(ref) >= entry["char_length"]: + continue # replacement would grow the payload -> skip + group_count += 1 + candidate_chars += (entry["occ"] - 1) * entry["char_length"] + return group_count, candidate_chars + + +def apply_artifact_dedup_canary( + contents: Iterable[_LLMContent], + *, + salt: str, + min_block_chars: int, + mode: str | None = None, + env: dict | None = None, + span_links: Iterable[ArtifactSpanLink] | None = None, +) -> ArtifactDedupCanaryResult: + """Run the artifact-dedup canary over LLM-bound content. + + ``contents`` are the in-memory ``_LLMContent`` items bound for the LLM. In + ``canary`` mode this MUTATES the ``content`` of eligible mutable artifact + items in place (keeping the first canonical body, replacing later exact + duplicates with a deterministic, strictly shorter reference). In ``off`` and + ``shadow`` modes nothing is mutated. + + ``mode`` overrides the resolved environment mode (used by tests); otherwise + the mode comes from :func:`resolve_artifact_dedup_mode`. + """ + items = list(contents) + links = list(span_links or []) + resolved = mode if mode is not None else resolve_artifact_dedup_mode(env) + if resolved not in ARTIFACT_DEDUP_MODES: + resolved = DEFAULT_ARTIFACT_DEDUP_MODE + + if resolved == "off": + # Safe default: no scan, no candidates, no savings. + return ArtifactDedupCanaryResult( + mode="off", + artifact_dedup_class=ARTIFACT_DEDUP_CLASS, + mutated=False, + item_count=0, + candidate_group_count=0, + candidate_chars=0, + blocks_replaced=0, + chars_saved=0, + notes=["artifact-dedup canary off (default): payload unchanged"], + ) + + agg, item_count = _scan_artifacts(items, salt=salt, min_block_chars=min_block_chars) + candidate_group_count, candidate_chars = _eligible_groups(agg) + block_group_count, block_candidate_chars = _scan_fenced_block_candidates( + items, salt=salt, min_block_chars=min_block_chars + ) + span_candidate_count, span_candidate_chars = _scan_span_candidates( + items, links, salt=salt, min_block_chars=min_block_chars + ) + candidate_group_count += block_group_count + span_candidate_count + candidate_chars += block_candidate_chars + span_candidate_chars + + if resolved == "shadow": + # Measure what a canary would replace, but never touch the payload. + return ArtifactDedupCanaryResult( + mode="shadow", + artifact_dedup_class=ARTIFACT_DEDUP_CLASS, + mutated=False, + item_count=item_count, + candidate_group_count=candidate_group_count, + candidate_chars=candidate_chars, + blocks_replaced=0, + chars_saved=0, + span_candidate_count=span_candidate_count, + span_candidate_chars=span_candidate_chars, + notes=["artifact-dedup canary shadow: candidates measured, payload unchanged"], + ) + + # --- canary: the ONLY branch that mutates LLM-bound payload --------------- + blocks_replaced = 0 + chars_saved = 0 + span_blocks_replaced, span_chars_saved, span_mutated_targets, span_preserved_sources = _apply_span_links( + items, links, salt=salt, min_block_chars=min_block_chars + ) + blocks_replaced += span_blocks_replaced + chars_saved += span_chars_saved + # hash -> canonical provenance type of the first (kept) occurrence. + canonical: dict[str, str] = {} + # hash -> canonical provenance type for exact fenced sub-artifacts. + block_canonical: dict[str, str] = {} + for idx, item in enumerate(items): + if idx in span_mutated_targets or idx in span_preserved_sources: + if item.block_type in MUTABLE_ARTIFACT_BLOCK_TYPES: + canonical[_salted_hash(item.content, salt)] = item.block_type + continue + if item.block_type not in MUTABLE_ARTIFACT_BLOCK_TYPES: + continue # protected content is never touched + body = item.content + if len(body) < min_block_chars: + continue + if _parse_artifact_reference(body) is not None: + continue + h = _salted_hash(body, salt) + already_has_whole_canonical = h in canonical + if already_has_whole_canonical: + # Later exact duplicate whole body: reference the EARLIER canonical + # body's provenance and do not also scan sub-blocks (no double count). + ref = _artifact_reference_string(canonical[h], h) + if len(ref) < len(body): # only when it actually shrinks the payload + item.content = ref + blocks_replaced += 1 + chars_saved += len(body) - len(ref) + continue + + # If the whole body is not replaced, opportunistically dedup exact + # duplicate fenced sub-artifacts within/across mutable artifact bodies. + segments = _segment_fenced_blocks(body) + if not any(kind == "fence" for kind, _text in segments): + if not already_has_whole_canonical: + canonical[h] = item.block_type # keep the first canonical body verbatim + continue + new_segments: list[str] = [] + changed = False + for kind, text in segments: + if kind != "fence" or len(text) < min_block_chars: + new_segments.append(text) + continue + if _parse_artifact_reference(text) is not None: + new_segments.append(text) + continue + bh = _salted_hash(text, salt) + if bh not in block_canonical: + block_canonical[bh] = f"{item.block_type}#block" + new_segments.append(text) + continue + ref = _artifact_reference_string(block_canonical[bh], bh) + if len(ref) < len(text): + new_segments.append(ref) + blocks_replaced += 1 + chars_saved += len(text) - len(ref) + changed = True + else: + new_segments.append(text) + if changed: + item.content = "".join(new_segments) + # Register only the post-mutation whole body as canonical. Registering + # the original pre-mutation hash would let a later whole-body + # reference point to a body no longer present in the payload. + canonical[_salted_hash(item.content, salt)] = item.block_type + elif not already_has_whole_canonical: + canonical[h] = item.block_type # keep the first canonical body verbatim + + return ArtifactDedupCanaryResult( + mode="canary", + artifact_dedup_class=ARTIFACT_DEDUP_CLASS, + mutated=blocks_replaced > 0, + item_count=item_count, + candidate_group_count=candidate_group_count, + candidate_chars=candidate_chars, + blocks_replaced=blocks_replaced, + chars_saved=chars_saved, + span_candidate_count=span_candidate_count, + span_candidate_chars=span_candidate_chars, + span_blocks_replaced=span_blocks_replaced, + span_chars_saved=span_chars_saved, + notes=["artifact-dedup canary active: exact duplicate artifact bodies only"], + ) + + +def dangling_artifact_references( + contents: Iterable[_LLMContent], *, salt: str, span_links: Iterable[ArtifactSpanLink] | None = None +) -> list[int]: + """Return indices of artifact references that do not resolve to an earlier body. + + A reference is valid only if an EARLIER mutable artifact item carries the + full canonical body whose salted hash matches the reference. A reference with + no such earlier canonical body (or one that only appears later) is dangling. + """ + seen_full: set[str] = set() # hashes of earlier full canonical artifact bodies/blocks/spans + dangling: list[int] = [] + items = list(contents) + span_by_source: dict[int, list[ArtifactSpanLink]] = {} + for link in span_links or []: + span_by_source.setdefault(link.source_index, []).append(link) + for idx, item in enumerate(items): + body = item.content + ref_hash = _parse_artifact_reference(body) + if ref_hash is not None: + if ref_hash not in seen_full: + dangling.append(idx) + continue + if item.block_type not in MUTABLE_ARTIFACT_BLOCK_TYPES: + continue + + # Whole artifact body can satisfy whole-body references. + seen_full.add(_salted_hash(body, salt)) + # Declared source spans can satisfy later #span references, but only from + # their earlier source item and only when the declared source slice is + # still byte-identical in the payload. + for link in span_by_source.get(idx, []): + if link.source_index >= link.target_index: + continue + if _line_aligned(body, link.source_start, link.source_end): + seen_full.add(_salted_hash(body[link.source_start:link.source_end], salt)) + + # Within a body, ordering matters: an earlier fenced block can satisfy a + # later reference segment in the same body, but a later block cannot. + for kind, text in _segment_fenced_blocks(body): + if kind != "fence": + # References may also appear as standalone prose lines after a + # fenced block replacement. Embedded prose around the line stays + # protected; only exact standalone reference lines are accepted. + for line in text.splitlines(): + seg_ref = _parse_artifact_reference(line.strip()) + if seg_ref is not None and seg_ref not in seen_full: + dangling.append(idx) + continue + seg_ref = _parse_artifact_reference(text) + if seg_ref is not None: + if seg_ref not in seen_full: + dangling.append(idx) + continue + seen_full.add(_salted_hash(text, salt)) + return dangling + + +def build_artifact_canary_telemetry_record(result: ArtifactDedupCanaryResult) -> dict: + """Build a metadata-only telemetry record for an artifact-dedup canary pass. + + The aggregate ``chars_saved`` counter gains the artifact-dedup contribution + ONLY when a real mutation occurred (canary). ``off``/``shadow`` contribute 0 + to the total while still reporting the separated ``artifact_dedup_*`` fields. + Contains only mode/class enums and integer counters -- never artifact text. + """ + realized = result.chars_saved if result.mutated else 0 + record = { + "artifact_dedup_mode": result.mode, + "artifact_dedup_class": result.artifact_dedup_class, + "artifact_dedup_blocks_replaced": result.blocks_replaced if result.mutated else 0, + "artifact_span_blocks_replaced": result.span_blocks_replaced if result.mutated else 0, + "artifact_span_chars_saved": result.span_chars_saved if result.mutated else 0, + # Separated field: always present, mirrors the realized artifact-dedup save. + "artifact_dedup_chars_saved": realized, + # Aggregate total: includes artifact dedup only when a mutation occurred. + "chars_saved": realized, + } + _assert_no_forbidden_keys(record) + return record diff --git a/contextpilot/hermes_opportunities/cli.py b/contextpilot/hermes_opportunities/cli.py new file mode 100644 index 0000000..1a84f00 --- /dev/null +++ b/contextpilot/hermes_opportunities/cli.py @@ -0,0 +1,174 @@ +"""Command-line entry point for the Hermes context opportunity analyzer. + +Thin orchestration layer: parse args, run the read-only loaders, build the +privacy-safe report, and write it. Hardened for unattended cron use -- on any +failure it emits only the exception class name (never a traceback that could +echo the DB path or SQL) and a non-zero exit code. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import json +from pathlib import Path + +from .aggregation import DEFAULT_MIN_ARTIFACT_CHARS +from .db import ( + load_heavy_sessions, + load_llm_bound_content, + load_tool_messages, + total_input_tokens, +) +from .models import ( + DEFAULT_LARGE_OUTPUT_CHARS, + DEFAULT_MIN_BLOCK_CHARS, + DEFAULT_MIN_BLOCK_REPEAT, + DEFAULT_TOP_N, +) +from .report import build_report, write_report +from .telemetry import parse_telemetry +from .tokenizer import resolve_tokenizer + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description="Privacy-safe Hermes context opportunity analyzer for ContextPilot." + ) + parser.add_argument("--state-db", type=Path, default=Path("/root/.hermes/state.db")) + parser.add_argument( + "--telemetry-file", + type=Path, + default=Path.home() / ".hermes" / "contextpilot" / "telemetry.jsonl", + help="metadata-only ContextPilot telemetry file", + ) + parser.add_argument( + "--out-dir", type=Path, default=Path.home() / "contextpilot" / "opportunities" + ) + parser.add_argument("--since-hours", type=int, default=24) + parser.add_argument( + "--all-sessions", + action="store_true", + help="ignore --since-hours; scan all non-archived sessions and active messages", + ) + parser.add_argument( + "--salt", + default="contextpilot-hermes-opportunity-v1", + help="salt for stable per-install content/session fingerprints", + ) + parser.add_argument("--date", default=dt.date.today().isoformat()) + parser.add_argument("--min-block-chars", type=int, default=DEFAULT_MIN_BLOCK_CHARS) + parser.add_argument("--min-block-repeat", type=int, default=DEFAULT_MIN_BLOCK_REPEAT) + parser.add_argument( + "--large-output-chars", type=int, default=DEFAULT_LARGE_OUTPUT_CHARS + ) + parser.add_argument("--top-n", type=int, default=DEFAULT_TOP_N) + parser.add_argument( + "--disable-worker-routing-shadow", + action="store_true", + help=( + "skip the shadow-mode Worker Context Routing classification " + "(P0 data collection; enabled by default, never prunes context)" + ), + ) + parser.add_argument( + "--disable-parent-aggregation", + action="store_true", + help=( + "skip the shadow-mode Parent Aggregation Artifact telemetry " + "(P0 telemetry only; enabled by default, never dedups/replaces context)" + ), + ) + parser.add_argument( + "--min-artifact-chars", type=int, default=DEFAULT_MIN_ARTIFACT_CHARS + ) + parser.add_argument( + "--disable-prompt-duplicate-shadow", + action="store_true", + help=( + "skip the advisory system/skill prompt duplicate-block scan " + "(enabled by default; advisory only, never rewrites/dedups prompts)" + ), + ) + parser.add_argument( + "--disable-prompt-dedup-ab", + action="store_true", + help=( + "skip the offline prompt-dedup A/B simulation section " + "(enabled by default; offline simulation only, never mutates prompts; " + "this is the evidence gate before any canary replace)" + ), + ) + parser.add_argument( + "--prompt-dedup-tokenizer", + default=None, + help=( + "opt-in exact tokenizer backend for the prompt-dedup A/B simulation, " + "e.g. 'tiktoken:cl100k_base' (off by default; without it the A/B " + "section reports tokenizer_status=unavailable and no actual-token fields)" + ), + ) + args = parser.parse_args(argv) + + if not args.state_db.exists(): + raise SystemExit(f"Hermes state DB not found: {args.state_db}") + + # Harden for unattended cron use: never dump a traceback (which would echo + # the DB path / SQL); emit only the exception class name and a non-zero code. + try: + # Opt-in tokenizer; off by default -> A/B simulation reports actual tokens + # as unavailable rather than fabricating chars/4 figures. + dedup_ab_tokenizer = resolve_tokenizer(args.prompt_dedup_tokenizer) + tool_messages = load_tool_messages( + args.state_db, since_hours=args.since_hours, all_sessions=args.all_sessions + ) + llm_contents = load_llm_bound_content( + args.state_db, since_hours=args.since_hours, all_sessions=args.all_sessions + ) + heavy_sessions = load_heavy_sessions( + args.state_db, + since_hours=args.since_hours, + salt=args.salt, + top_n=args.top_n, + all_sessions=args.all_sessions, + ) + total_input = total_input_tokens( + args.state_db, since_hours=args.since_hours, all_sessions=args.all_sessions + ) + telemetry = parse_telemetry( + args.telemetry_file, + since_hours=args.since_hours, + total_input_tokens=total_input, + all_sessions=args.all_sessions, + ) + report = build_report( + date=args.date, + since_hours=args.since_hours, + salt=args.salt, + tool_messages=tool_messages, + heavy_sessions=heavy_sessions, + telemetry=telemetry, + llm_contents=llm_contents, + all_sessions=args.all_sessions, + min_block_chars=args.min_block_chars, + min_block_repeat=args.min_block_repeat, + large_output_chars=args.large_output_chars, + top_n=args.top_n, + worker_routing_shadow=not args.disable_worker_routing_shadow, + parent_aggregation_shadow=not args.disable_parent_aggregation, + prompt_duplicate_shadow=not args.disable_prompt_duplicate_shadow, + prompt_dedup_ab=not args.disable_prompt_dedup_ab, + prompt_dedup_ab_tokenizer=dedup_ab_tokenizer, + min_artifact_chars=args.min_artifact_chars, + ) + json_path, md_path = write_report(report, args.out_dir) + except Exception as exc: # noqa: BLE001 - cron-safe: report class only, no payload + print(json.dumps({"ok": False, "error": type(exc).__name__})) + return 1 + + print( + json.dumps( + {"ok": True, "json": str(json_path), "markdown": str(md_path)}, + ensure_ascii=False, + ) + ) + return 0 diff --git a/contextpilot/hermes_opportunities/db.py b/contextpilot/hermes_opportunities/db.py new file mode 100644 index 0000000..5d755c9 --- /dev/null +++ b/contextpilot/hermes_opportunities/db.py @@ -0,0 +1,272 @@ +"""Read-only Hermes state-DB loaders. + +All loaders open the SQLite DB in read-only mode and return either privacy-safe +dataclasses (``HeavySession``) or in-memory content carriers (``_ToolMessage``, +``_LLMContent``) whose ``content`` is for hashing only and must never be +emitted. Schema is probed defensively so older/newer DBs degrade gracefully. +""" +from __future__ import annotations + +import datetime as dt +import sqlite3 +from pathlib import Path + +from .models import HeavySession, _LLMContent, _ToolMessage +from .privacy import _salted_hash + + +def _connect_readonly(path: Path) -> sqlite3.Connection: + uri = f"file:{path}?mode=ro" + return sqlite3.connect(uri, uri=True) + + +def _window_cutoff(since_hours: int, all_sessions: bool) -> float | None: + """Return the epoch cutoff, or ``None`` to scan all history. + + ``all_sessions=True`` disables the time window so old sessions/messages are + included regardless of ``since_hours``. + """ + if all_sessions: + return None + return dt.datetime.now(dt.timezone.utc).timestamp() - since_hours * 3600 + + +def _classify_system_prompt(text: str) -> str: + """Heuristically label a system prompt as skill material or a plain prompt. + + Operates on in-memory text only; returns a low-cardinality enum, never the + text itself. + """ + low = text.lower() + stripped = low.lstrip() + # Skill-style frontmatter block (e.g. "---\nname: ...\ndescription: ..."). + if stripped.startswith("---") and "name:" in low[:300]: + return "skill_prompt" + cues = ( + "use this skill", + "available skills", + "when to use", + "invoke it via skill", + " str: + if role == "tool" or tool_name is not None: + return "tool_result" + if role == "user": + return "user_prompt" + if role == "assistant": + return "assistant_context" + if role == "system": + return "system_prompt" + return "unknown" + + +def load_tool_messages( + db_path: Path, *, since_hours: int, all_sessions: bool = False +) -> list[_ToolMessage]: + """Load tool-output messages within the window. + + Content is returned for in-memory hashing only; callers must not emit it. + A message is treated as tool output when ``role='tool'`` or ``tool_name`` + is set. With ``all_sessions=True`` the time window is ignored. + """ + cutoff = _window_cutoff(since_hours, all_sessions) + conn = _connect_readonly(db_path) + try: + cols = {row[1] for row in conn.execute("PRAGMA table_info(messages)")} + if "content" not in cols: + return [] + has_tool_name = "tool_name" in cols + has_ts = "timestamp" in cols + select_tool = "tool_name" if has_tool_name else "NULL AS tool_name" + where = [] + params: list[object] = [] + if has_ts and cutoff is not None: + where.append("timestamp >= ?") + params.append(cutoff) + if "active" in cols: + where.append("active = 1") + tool_pred = "role = 'tool'" + if has_tool_name: + tool_pred = "(role = 'tool' OR tool_name IS NOT NULL)" + where.append(tool_pred) + sql = ( + f"SELECT {select_tool}, content FROM messages " + f"WHERE {' AND '.join(where)}" + ) + rows = conn.execute(sql, params).fetchall() + finally: + conn.close() + + out: list[_ToolMessage] = [] + for tool_name, content in rows: + if content is None: + continue + out.append(_ToolMessage(tool_name=tool_name, content=str(content))) + return out + + +def load_llm_bound_content( + db_path: Path, *, since_hours: int, all_sessions: bool = False +) -> list[_LLMContent]: + """Load only content Hermes would actually send to an LLM. + + Sources, all read in-memory for hashing (never emitted): + * ``sessions.system_prompt`` -> ``system_prompt`` or ``skill_prompt``, + * ``messages.content`` for active messages with role in + ``system``/``user``/``assistant``/``tool`` -> per-role block type, + * tool-result messages (role=tool or ``tool_name`` set) -> ``tool_result``. + + Inactive messages are skipped when an ``active`` column exists; archived + sessions (and their messages) are skipped when an ``archived`` column + exists. With ``all_sessions=True`` the time window is ignored. + """ + cutoff = _window_cutoff(since_hours, all_sessions) + conn = _connect_readonly(db_path) + out: list[_LLMContent] = [] + try: + scols = {row[1] for row in conn.execute("PRAGMA table_info(sessions)")} + mcols = {row[1] for row in conn.execute("PRAGMA table_info(messages)")} + + # --- system / skill prompts from sessions ------------------------- + if "system_prompt" in scols: + where = ["system_prompt IS NOT NULL"] + params: list[object] = [] + if cutoff is not None and "started_at" in scols: + where.append("started_at >= ?") + params.append(cutoff) + if "archived" in scols: + where.append("archived = 0") + sql = f"SELECT system_prompt FROM sessions WHERE {' AND '.join(where)}" + for (sp,) in conn.execute(sql, params): + if sp is None: + continue + text = str(sp) + out.append( + _LLMContent(block_type=_classify_system_prompt(text), content=text) + ) + + # --- active messages bound for the LLM ---------------------------- + if "content" in mcols: + has_role = "role" in mcols + has_tool_name = "tool_name" in mcols + select = [ + "messages.role" if has_role else "NULL AS role", + "messages.content", + "messages.tool_name" if has_tool_name else "NULL AS tool_name", + ] + where = ["messages.content IS NOT NULL"] + params = [] + if has_role: + where.append( + "messages.role IN ('system', 'user', 'assistant', 'tool')" + ) + if cutoff is not None and "timestamp" in mcols: + where.append("messages.timestamp >= ?") + params.append(cutoff) + if "active" in mcols: + where.append("messages.active = 1") + join = "" + if "archived" in scols and "session_id" in mcols and "id" in scols: + join = " JOIN sessions ON sessions.id = messages.session_id" + where.append("sessions.archived = 0") + sql = ( + f"SELECT {', '.join(select)} FROM messages{join} " + f"WHERE {' AND '.join(where)}" + ) + for role, content, tool_name in conn.execute(sql, params): + if content is None: + continue + out.append( + _LLMContent( + block_type=_message_block_type(role, tool_name), + content=str(content), + ) + ) + finally: + conn.close() + return out + + +def load_heavy_sessions( + db_path: Path, *, since_hours: int, salt: str, top_n: int, all_sessions: bool = False +) -> list[HeavySession]: + cutoff = _window_cutoff(since_hours, all_sessions) + conn = _connect_readonly(db_path) + try: + cols = {row[1] for row in conn.execute("PRAGMA table_info(sessions)")} + if "id" not in cols: + return [] + wanted = [ + "id", + "source", + "input_tokens", + "output_tokens", + "message_count", + "tool_call_count", + "api_call_count", + ] + select_cols = [c if c in cols else f"NULL AS {c}" for c in wanted] + where = [] + params: list[object] = [] + if cutoff is not None and "started_at" in cols: + where.append("started_at >= ?") + params.append(cutoff) + if "archived" in cols: + where.append("archived = 0") + sql = f"SELECT {', '.join(select_cols)} FROM sessions" + if where: + sql += " WHERE " + " AND ".join(where) + sql += " ORDER BY input_tokens DESC" + rows = conn.execute(sql, params).fetchall() + finally: + conn.close() + + sessions: list[HeavySession] = [] + for sid, source, inp, out_tok, msgs, tools, apis in rows: + sessions.append( + HeavySession( + session_hash=_salted_hash(str(sid), salt), + source=source, + input_tokens=int(inp or 0), + output_tokens=int(out_tok or 0), + message_count=int(msgs or 0), + tool_call_count=int(tools or 0), + api_call_count=int(apis or 0), + ) + ) + sessions.sort(key=lambda s: (s.input_tokens, s.tool_call_count), reverse=True) + return sessions[:top_n] + + +def total_input_tokens( + db_path: Path, *, since_hours: int, all_sessions: bool = False +) -> int: + """Sum input tokens across ALL in-window sessions (not just the top-N).""" + cutoff = _window_cutoff(since_hours, all_sessions) + conn = _connect_readonly(db_path) + try: + cols = {row[1] for row in conn.execute("PRAGMA table_info(sessions)")} + if "input_tokens" not in cols: + return 0 + where = [] + params: list[object] = [] + if cutoff is not None and "started_at" in cols: + where.append("started_at >= ?") + params.append(cutoff) + if "archived" in cols: + where.append("archived = 0") + sql = "SELECT COALESCE(SUM(input_tokens), 0) FROM sessions" + if where: + sql += " WHERE " + " AND ".join(where) + (total,) = conn.execute(sql, params).fetchone() + finally: + conn.close() + return int(total or 0) diff --git a/contextpilot/hermes_opportunities/dedup_ab.py b/contextpilot/hermes_opportunities/dedup_ab.py new file mode 100644 index 0000000..768eca7 --- /dev/null +++ b/contextpilot/hermes_opportunities/dedup_ab.py @@ -0,0 +1,229 @@ +"""Prompt dedup A/B simulation harness (OFFLINE simulation + measurement only). + +This is the evidence gate to evaluate *before* any canary prompt replacement. +It scans ONLY ``system_prompt`` / ``skill_prompt`` LLM-bound blocks, fingerprints +exact duplicate blocks, and simulates -- in accounting only -- keeping the first +occurrence of each duplicate while replacing every later occurrence with a +deterministic reference placeholder. + +Hard guarantees: + +* It never mutates the DB, runtime state, or any emitted prompt; it produces no + side effects beyond the privacy-safe report dataclasses below. +* It emits salted hashes / counters / low-cardinality enums only -- never raw + prompt text and never the reference placeholder filled with real content. +* Char and token deltas are SIMULATED candidate figures, explicitly NOT realized + savings. ContextPilot performs no canonicalization or replacement at runtime. +* Exact token figures appear only when an explicitly configured tokenizer backend + is available (opt-in, off by default); otherwise the status is ``unavailable`` + and no actual-token fields are populated. +""" +from __future__ import annotations + +from typing import Iterable + +from .models import ( + PROMPT_DEDUP_AB_CLASSES, + PROMPT_DEDUP_AB_REFERENCE_TEMPLATE, + PROMPT_DUPLICATE_BLOCK_TYPES, + PromptDedupABClass, + PromptDedupABSimulation, + _LLMContent, +) +from .privacy import _salted_hash +from .tokenizer import TokenizerBackend + +# Per-class risk label + advisory note. The skill-only class is the lowest-risk +# first canary candidate; the other classes are reported but flagged higher risk. +_CLASS_META = { + "same_type_skill_prompt_only": ( + "low", + "first canary candidate: exact duplicate blocks within skill prompts only", + ), + "same_type_system_prompt_only": ( + "high", + "higher risk: exact duplicate blocks within system prompts only", + ), + "cross_type_system_skill": ( + "high", + "higher risk: exact duplicate blocks shared across system and skill prompts", + ), +} + + +def _classify_group(types: dict[str, int]) -> str | None: + """Map a duplicate group's prompt-type spread to a candidate class.""" + present = set(types) + if present == {"skill_prompt"}: + return "same_type_skill_prompt_only" + if present == {"system_prompt"}: + return "same_type_system_prompt_only" + if present == {"system_prompt", "skill_prompt"}: + return "cross_type_system_skill" + return None # only system/skill are scanned; anything else is ignored + + +def _canonical_type(types: dict[str, int]) -> str: + """Deterministically pick the canonical prompt type for the reference string. + + Dominant by occurrence count; ties broken by sorted type name so the choice + is stable across runs and inputs. + """ + return sorted(types.items(), key=lambda kv: (-kv[1], kv[0]))[0][0] + + +def _reference_string(canonical_type: str, block_hash: str) -> str: + return PROMPT_DEDUP_AB_REFERENCE_TEMPLATE.replace("", canonical_type).replace( + "", block_hash + ) + + +def simulate_prompt_dedup_ab( + contents: Iterable[_LLMContent], + *, + salt: str, + min_block_chars: int, + tokenizer: TokenizerBackend | None = None, + enabled: bool = True, +) -> PromptDedupABSimulation: + """Simulate prompt-dedup replacement over system/skill prompt blocks. + + Restricted to ``system_prompt`` / ``skill_prompt`` items. Every fingerprintable + line is counted (intra- and inter-prompt), and any fingerprint seen 2+ times + is a duplicate group. Each group is assigned to exactly one candidate class and + simulated independently: the first occurrence is kept full, every later + occurrence is replaced (in accounting only) by the deterministic reference + string ``[Prompt duplicate omitted in simulation; canonical=:]``. + + Returns a privacy-safe :class:`PromptDedupABSimulation` -- hashes, counters, + and enums only. No DB/runtime/payload is touched. + """ + scanned = list(PROMPT_DUPLICATE_BLOCK_TYPES) + tok_status = "available" if tokenizer is not None else "unavailable" + tok_backend = tokenizer.name if tokenizer is not None else None + + if not enabled: + return PromptDedupABSimulation( + enabled=False, + item_count=0, + scanned_block_types=scanned, + tokenizer_status="unavailable", + tokenizer_backend=None, + reference_string_template=PROMPT_DEDUP_AB_REFERENCE_TEMPLATE, + classes=[], + notes=["prompt-dedup A/B simulation disabled"], + ) + + # block_hash -> {char_length, text (in-memory only), types: {block_type: occ}} + agg: dict[str, dict] = {} + item_count = 0 + for item in contents: + bt = item.block_type + if bt not in PROMPT_DUPLICATE_BLOCK_TYPES: + continue + item_count += 1 + for line in item.content.splitlines(): + block = line.strip() + if len(block) < min_block_chars: + continue + h = _salted_hash(block, salt) + entry = agg.get(h) + if entry is None: + # ``text`` is held in-memory only for exact token counting; it is + # never written to the report (no dataclass field carries it). + agg[h] = {"char_length": len(block), "text": block, "types": {bt: 1}} + else: + entry["types"][bt] = entry["types"].get(bt, 0) + 1 + + # Per-class running totals. + acc: dict[str, dict] = { + cls: { + "groups": 0, + "repl_occ": 0, + "chars_before": 0, + "chars_after": 0, + "tok_before": 0, + "tok_after": 0, + } + for cls in PROMPT_DEDUP_AB_CLASSES + } + + for h, entry in agg.items(): + types = entry["types"] + occ = sum(types.values()) + if occ < 2: + continue # not a duplicate -> no replacement candidate + cls = _classify_group(types) + if cls is None: + continue + char_len = entry["char_length"] + ref = _reference_string(_canonical_type(types), h) + ref_len = len(ref) + + a = acc[cls] + a["groups"] += 1 + a["repl_occ"] += occ - 1 + a["chars_before"] += occ * char_len + # Keep first occurrence full; later occurrences become the reference str. + a["chars_after"] += char_len + (occ - 1) * ref_len + if tokenizer is not None: + tb = tokenizer.count(entry["text"]) + tr = tokenizer.count(ref) + a["tok_before"] += occ * tb + a["tok_after"] += tb + (occ - 1) * tr + + classes: list[PromptDedupABClass] = [] + for cls in PROMPT_DEDUP_AB_CLASSES: + a = acc[cls] + risk, note = _CLASS_META[cls] + if tokenizer is not None: + tok_before = a["tok_before"] + tok_after = a["tok_after"] + tok_delta = tok_before - tok_after + else: + tok_before = tok_after = tok_delta = None + classes.append( + PromptDedupABClass( + candidate_class=cls, + risk_label=risk, + candidate_group_count=a["groups"], + replacement_occurrence_count=a["repl_occ"], + chars_before=a["chars_before"], + chars_after_simulated=a["chars_after"], + chars_delta_simulated=a["chars_before"] - a["chars_after"], + tokenizer_status=tok_status, + actual_tokens_before=tok_before, + actual_tokens_after=tok_after, + actual_tokens_delta=tok_delta, + note=note, + ) + ) + + notes = [ + "OFFLINE SIMULATION + MEASUREMENT ONLY: no DB/runtime/prompt is mutated; " + "ContextPilot performs no replacement or canonicalization", + "char/token deltas are SIMULATED candidate figures, NOT realized savings", + "this A/B evidence is the gate to evaluate before any canary prompt replacement", + "same_type_skill_prompt_only is the lowest-risk first canary candidate; " + "system-only and cross-type classes are higher risk", + "chars_delta_simulated is signed: negative means a short duplicate would grow " + "if replaced by the reference placeholder", + ] + if tokenizer is None: + notes.append( + "actual-token measurement unavailable (no exact tokenizer backend configured); " + "no actual-token fields are reported" + ) + if item_count == 0: + notes.append("no system/skill prompt items observed in the selected window") + + return PromptDedupABSimulation( + enabled=True, + item_count=item_count, + scanned_block_types=scanned, + tokenizer_status=tok_status, + tokenizer_backend=tok_backend, + reference_string_template=PROMPT_DEDUP_AB_REFERENCE_TEMPLATE, + classes=classes, + notes=notes, + ) diff --git a/contextpilot/hermes_opportunities/detection.py b/contextpilot/hermes_opportunities/detection.py new file mode 100644 index 0000000..0ca2433 --- /dev/null +++ b/contextpilot/hermes_opportunities/detection.py @@ -0,0 +1,372 @@ +"""Content-aware redundancy detection (reporting/measurement only). + +These detectors fingerprint in-memory message/tool/LLM-bound text with salted +hashes and emit privacy-safe dataclasses (hashes + counters + low-cardinality +enums). They measure redundancy opportunities; they never drop, summarize, or +mutate any context. +""" +from __future__ import annotations + +from typing import Iterable + +from .models import ( + PROMPT_DUPLICATE_BLOCK_TYPES, + BlockTypeStat, + CrossTypeBlockGroup, + DuplicateToolOutput, + PromptDuplicateBlock, + PromptDuplicateShadow, + PromptDuplicateTypeCount, + RepeatedBlock, + ToolSizeStat, + TypeCount, + _est_tokens, + _LLMContent, + _ToolMessage, +) +from .privacy import _salted_hash + + +def detect_exact_duplicate_tool_outputs( + messages: Iterable[_ToolMessage], *, salt: str, top_n: int +) -> list[DuplicateToolOutput]: + groups: dict[str, dict] = {} + for msg in messages: + content = msg.content + if not content: + continue + h = _salted_hash(content, salt) + g = groups.get(h) + if g is None: + groups[h] = { + "tool_name": msg.tool_name, + "occurrences": 1, + "char_length": len(content), + } + else: + g["occurrences"] += 1 + if g["tool_name"] != msg.tool_name: + g["tool_name"] = None # mixed tools produced identical output + + dups: list[DuplicateToolOutput] = [] + for h, g in groups.items(): + if g["occurrences"] < 2: + continue + est = _est_tokens(g["char_length"]) + dups.append( + DuplicateToolOutput( + content_hash=h, + tool_name=g["tool_name"], + occurrences=g["occurrences"], + char_length=g["char_length"], + est_tokens=est, + est_wasted_tokens=est * (g["occurrences"] - 1), + ) + ) + dups.sort(key=lambda d: d.est_wasted_tokens, reverse=True) + return dups[:top_n] + + +def detect_repeated_blocks( + messages: Iterable[_ToolMessage], + *, + salt: str, + min_block_chars: int, + min_repeat: int, + top_n: int, +) -> list[RepeatedBlock]: + counts: dict[str, dict] = {} + for msg in messages: + seen_in_msg: set[str] = set() + for line in msg.content.splitlines(): + block = line.strip() + if len(block) < min_block_chars: + continue + h = _salted_hash(block, salt) + # Count cross-message recurrence; collapse repeats within one + # message so a single noisy output cannot dominate. + if h in seen_in_msg: + continue + seen_in_msg.add(h) + c = counts.get(h) + if c is None: + counts[h] = {"occurrences": 1, "char_length": len(block)} + else: + c["occurrences"] += 1 + + blocks: list[RepeatedBlock] = [] + for h, c in counts.items(): + if c["occurrences"] < min_repeat: + continue + est = _est_tokens(c["char_length"]) + blocks.append( + RepeatedBlock( + block_hash=h, + occurrences=c["occurrences"], + char_length=c["char_length"], + est_tokens=est, + est_wasted_tokens=est * (c["occurrences"] - 1), + ) + ) + blocks.sort(key=lambda b: b.est_wasted_tokens, reverse=True) + return blocks[:top_n] + + +def summarize_tool_sizes( + messages: Iterable[_ToolMessage], *, large_output_chars: int, top_n: int +) -> list[ToolSizeStat]: + agg: dict[str, dict] = {} + for msg in messages: + name = msg.tool_name or "(unknown)" + length = len(msg.content) + a = agg.get(name) + if a is None: + agg[name] = { + "output_count": 1, + "total_chars": length, + "max_chars": length, + "large_output_count": 1 if length >= large_output_chars else 0, + } + else: + a["output_count"] += 1 + a["total_chars"] += length + a["max_chars"] = max(a["max_chars"], length) + if length >= large_output_chars: + a["large_output_count"] += 1 + + stats: list[ToolSizeStat] = [] + for name, a in agg.items(): + stats.append( + ToolSizeStat( + tool_name=name, + output_count=a["output_count"], + total_chars=a["total_chars"], + max_chars=a["max_chars"], + avg_chars=a["total_chars"] // a["output_count"], + total_est_tokens=_est_tokens(a["total_chars"]), + large_output_count=a["large_output_count"], + ) + ) + stats.sort(key=lambda s: s.total_chars, reverse=True) + return stats[:top_n] + + +def detect_prompt_duplicate_blocks( + contents: Iterable[_LLMContent], + *, + salt: str, + min_block_chars: int, + top_n: int, + enabled: bool = True, +) -> PromptDuplicateShadow: + """Advisory scan for EXACT duplicate blocks in system/skill prompt text. + + Restricted to ``system_prompt`` / ``skill_prompt`` items. Counts every block + instance (intra- and inter-prompt) so a block literally present multiple + times in the static prompt payload is detected. A "duplicate" is any block + fingerprint observed 2+ times. + + SHADOW/ADVISORY ONLY: output is salted hashes + counters + block-type enums; + char figures are ACTUAL duplicated chars and the token figure is an ADVISORY + chars/4 estimate. This never rewrites or dedups prompts and must never be + reported as a realized saving. + """ + scanned = list(PROMPT_DUPLICATE_BLOCK_TYPES) + if not enabled: + return PromptDuplicateShadow( + enabled=False, + item_count=0, + scanned_block_types=scanned, + duplicate_group_count=0, + total_duplicate_occurrences=0, + total_chars_duplicated=0, + advisory_est_duplicate_tokens_chars_div_4=0, + by_block_type=[], + top_duplicate_blocks=[], + notes=["prompt-duplicate shadow disabled"], + ) + + # block_hash -> {char_length, types: {block_type: occ}} + agg: dict[str, dict] = {} + item_counts: dict[str, int] = {} + for item in contents: + bt = item.block_type + if bt not in PROMPT_DUPLICATE_BLOCK_TYPES: + continue + item_counts[bt] = item_counts.get(bt, 0) + 1 + # Count every fingerprintable line (no intra-item dedup) so repeated + # blocks within a single prompt are surfaced too. + for line in item.content.splitlines(): + block = line.strip() + if len(block) < min_block_chars: + continue + h = _salted_hash(block, salt) + entry = agg.get(h) + if entry is None: + agg[h] = {"char_length": len(block), "types": {bt: 1}} + else: + entry["types"][bt] = entry["types"].get(bt, 0) + 1 + + dup_blocks: list[PromptDuplicateBlock] = [] + per_type: dict[str, dict] = {} + total_chars_dup = 0 + total_dup_occ = 0 + for h, entry in agg.items(): + types = entry["types"] + occ = sum(types.values()) + if occ < 2: + continue # not a duplicate + char_len = entry["char_length"] + chars_dup = (occ - 1) * char_len + total_chars_dup += chars_dup + total_dup_occ += occ + dup_blocks.append( + PromptDuplicateBlock( + block_hash=h, + block_types=sorted(types.keys()), + occurrences=occ, + char_length=char_len, + chars_duplicated=chars_dup, + advisory_est_duplicate_tokens_chars_div_4=_est_tokens(chars_dup), + ) + ) + for bt, type_occ in types.items(): + t = per_type.setdefault( + bt, {"blocks": 0, "occ": 0, "chars_dup": 0} + ) + t["blocks"] += 1 + t["occ"] += type_occ + # Attribute duplicated chars within this type (occ-1 of the in-type + # instances are duplicates); cross-type-only blocks contribute 0 here. + t["chars_dup"] += max(type_occ - 1, 0) * char_len + + by_block_type = [ + PromptDuplicateTypeCount( + block_type=bt, + duplicate_block_count=per_type.get(bt, {}).get("blocks", 0), + occurrence_count=per_type.get(bt, {}).get("occ", 0), + chars_duplicated=per_type.get(bt, {}).get("chars_dup", 0), + ) + for bt in scanned + ] + + dup_blocks.sort(key=lambda b: b.chars_duplicated, reverse=True) + notes: list[str] = [] + if not item_counts: + notes.append("no system/skill prompt items observed in the selected window") + return PromptDuplicateShadow( + enabled=True, + item_count=sum(item_counts.values()), + scanned_block_types=scanned, + duplicate_group_count=len(dup_blocks), + total_duplicate_occurrences=total_dup_occ, + total_chars_duplicated=total_chars_dup, + advisory_est_duplicate_tokens_chars_div_4=_est_tokens(total_chars_dup), + by_block_type=by_block_type, + top_duplicate_blocks=dup_blocks[:top_n], + notes=notes, + ) + + +def _iter_blocks(content: str, min_block_chars: int) -> Iterable[str]: + """Yield the distinct fingerprintable lines of one item (deduped in-item).""" + seen: set[str] = set() + for line in content.splitlines(): + block = line.strip() + if len(block) < min_block_chars: + continue + if block in seen: + continue + seen.add(block) + yield block + + +def analyze_llm_bound_blocks( + contents: Iterable[_LLMContent], + *, + salt: str, + min_block_chars: int, + min_repeat: int, + top_n: int, +) -> tuple[list[BlockTypeStat], list[CrossTypeBlockGroup]]: + """Fingerprint LLM-bound blocks and report redundancy. + + Returns (per-type stats, cross-type repeated block groups). All output is + salted hashes / counters / block-type enums -- no raw text. + """ + # block_hash -> {char_length, types: {block_type: occ}} + agg: dict[str, dict] = {} + # block_type -> source item count + item_counts: dict[str, int] = {} + + for item in contents: + bt = item.block_type + item_counts[bt] = item_counts.get(bt, 0) + 1 + for block in _iter_blocks(item.content, min_block_chars): + h = _salted_hash(block, salt) + entry = agg.get(h) + if entry is None: + agg[h] = {"char_length": len(block), "types": {bt: 1}} + else: + entry["types"][bt] = entry["types"].get(bt, 0) + 1 + + # --- per block-type aggregate redundancy ------------------------------ + per_type: dict[str, dict] = {} + for entry in agg.values(): + est = _est_tokens(entry["char_length"]) + for bt, occ in entry["types"].items(): + t = per_type.setdefault( + bt, + { + "block_count": 0, + "unique": 0, + "repeated": 0, + "redundant_tokens": 0, + }, + ) + t["block_count"] += occ + t["unique"] += 1 + if occ >= min_repeat: + t["repeated"] += 1 + t["redundant_tokens"] += est * (occ - 1) + + block_type_stats: list[BlockTypeStat] = [] + for bt in sorted(set(per_type) | set(item_counts)): + t = per_type.get( + bt, {"block_count": 0, "unique": 0, "repeated": 0, "redundant_tokens": 0} + ) + block_type_stats.append( + BlockTypeStat( + block_type=bt, + item_count=item_counts.get(bt, 0), + block_count=t["block_count"], + unique_block_count=t["unique"], + repeated_block_count=t["repeated"], + est_redundant_tokens=t["redundant_tokens"], + ) + ) + + # --- cross-type repeated blocks --------------------------------------- + cross: list[CrossTypeBlockGroup] = [] + for h, entry in agg.items(): + types = entry["types"] + if len(types) < 2: + continue + total_occ = sum(types.values()) + est = _est_tokens(entry["char_length"]) + cross.append( + CrossTypeBlockGroup( + block_hash=h, + block_types=sorted(types.keys()), + type_occurrences=[ + TypeCount(block_type=bt, count=occ) + for bt, occ in sorted(types.items()) + ], + occurrences=total_occ, + char_length=entry["char_length"], + est_tokens=est, + est_wasted_tokens=est * (total_occ - 1), + ) + ) + cross.sort(key=lambda g: g.est_wasted_tokens, reverse=True) + return block_type_stats, cross[:top_n] diff --git a/contextpilot/hermes_opportunities/models.py b/contextpilot/hermes_opportunities/models.py new file mode 100644 index 0000000..2bfe1d4 --- /dev/null +++ b/contextpilot/hermes_opportunities/models.py @@ -0,0 +1,483 @@ +"""Privacy-safe data structures and shared tunables for the analyzer. + +Every dataclass here is privacy-safe by construction: it carries only salted +hashes, numeric counters, and low-cardinality enums -- never raw message/tool/ +system/skill text. The in-memory carriers ``_ToolMessage`` and ``_LLMContent`` +DO hold content, but exclusively for in-process hashing; their ``content`` must +never be emitted to a report. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + +# Tunables (overridable via CLI). +DEFAULT_MIN_BLOCK_CHARS = 40 # ignore trivial lines when fingerprinting +DEFAULT_MIN_BLOCK_REPEAT = 3 # a block must recur this often to be a "repeat" +DEFAULT_LARGE_OUTPUT_CHARS = 8000 # tool outputs at/above this are "large" +DEFAULT_TOP_N = 20 +EST_CHARS_PER_TOKEN = 4 # legacy advisory only; runtime token accounting uses tokenizer fields + + +def _est_tokens(chars: int) -> int: + return chars // EST_CHARS_PER_TOKEN + + +# Recognized LLM-bound block types. These are low-cardinality enums, safe to +# emit verbatim (they describe the *origin* of a block, never its text). +BLOCK_TYPES = ( + "system_prompt", + "skill_prompt", + "user_prompt", + "assistant_context", + "tool_result", + "unknown", +) + + +# --------------------------------------------------------------------------- +# Tool-output redundancy structures (all privacy-safe: hashes + counters only) +# --------------------------------------------------------------------------- + + +@dataclass +class DuplicateToolOutput: + content_hash: str + tool_name: str | None + occurrences: int + char_length: int + est_tokens: int + est_wasted_tokens: int # tokens spent re-sending identical output: (n-1) * est_tokens + + +@dataclass +class RepeatedBlock: + block_hash: str + occurrences: int + char_length: int + est_tokens: int + est_wasted_tokens: int # (n-1) * est_tokens + + +@dataclass +class TypeCount: + block_type: str + count: int + + +@dataclass +class BlockTypeStat: + """Aggregate redundancy within a single LLM-bound block type.""" + + block_type: str + item_count: int # source items (prompts/messages) of this type + block_count: int # total fingerprintable block instances + unique_block_count: int # distinct fingerprints + repeated_block_count: int # fingerprints recurring >= min_repeat within type + est_redundant_tokens: int # sum over repeats of (occ-1) * est_tokens + + +@dataclass +class CrossTypeBlockGroup: + """A single block fingerprint observed in 2+ distinct block types. + + This is the headline signal: the same chunk of text is being shipped to the + LLM from, e.g., a skill/system prompt *and* a tool result, so it is paying + for the same tokens twice from different sources. + """ + + block_hash: str + block_types: list[str] # sorted distinct types this block spans + type_occurrences: list[TypeCount] # per-type occurrence counts + occurrences: int # total occurrences across all types + char_length: int + est_tokens: int + est_wasted_tokens: int # (occurrences - 1) * est_tokens + + +@dataclass +class ToolSizeStat: + tool_name: str + output_count: int + total_chars: int + max_chars: int + avg_chars: int + total_est_tokens: int + large_output_count: int # outputs >= large_output_chars threshold + + +@dataclass +class HeavySession: + session_hash: str + source: str | None + input_tokens: int + output_tokens: int + message_count: int + tool_call_count: int + api_call_count: int + + +# Source label used when a Hermes session carries no recorded provenance source. +# Keeps the provenance profile a low-cardinality enum view rather than leaking +# a null/raw value. +UNKNOWN_SOURCE = "unknown" + + +@dataclass +class ProvenanceSourceStat: + """Token-monitor rollup for one provenance source. + + Privacy-safe by construction: a low-cardinality source label plus numeric + aggregates only -- never raw session ids/hashes, prompts, content, or + reasoning. This is the per-source ``by_source`` row of + :class:`ProvenanceProfile`. + """ + + source: str # low-cardinality provenance label (e.g. "discord") + session_count: int + input_tokens: int + output_tokens: int + message_count: int + tool_call_count: int + api_call_count: int + total_tokens: int # input_tokens + output_tokens + + +@dataclass +class ProvenanceProfile: + """Privacy-safe per-source token-usage profile (the token-monitor view). + + Aggregates :class:`HeavySession` rows by their provenance ``source`` into + numeric counters. Emits only low-cardinality source enums and integer + aggregates -- no session hashes/ids, prompts, content, or reasoning ever + appear in this structure. + """ + + source_count: int + session_count: int + input_tokens: int + output_tokens: int + total_tokens: int + by_source: list[ProvenanceSourceStat] + + +@dataclass +class TelemetryCoverage: + events: int + chars_saved: int + tokens_saved: int # tokenizer-measured saved tokens only + avg_tokens_saved_per_event: float # derived from tokenizer-measured tokens + coverage_ratio_pct: float # derived ratio using tokenizer-measured tokens + malformed_records_skipped: int + + +# --------------------------------------------------------------------------- +# Prompt duplicate — SHADOW MODE structures (system/skill prompts only) +# --------------------------------------------------------------------------- + +# Block types this advisory section is allowed to scan. Static prompt text only; +# never user/assistant/tool message bodies. +PROMPT_DUPLICATE_BLOCK_TYPES = ("system_prompt", "skill_prompt") + + +@dataclass +class PromptDuplicateBlock: + """One exact block fingerprint seen 2+ times in system/skill prompt text. + + Salted hash + counters only -- never the block text. Char figures are ACTUAL + duplicated characters; the token figure is an ADVISORY chars/4 estimate. + """ + + block_hash: str + block_types: list[str] # which prompt types this block spans + occurrences: int + char_length: int + chars_duplicated: int # ACTUAL: (occurrences - 1) * char_length + advisory_est_duplicate_tokens_chars_div_4: int # ADVISORY only, NOT actual tokens + + +@dataclass +class PromptDuplicateTypeCount: + """Per-prompt-type occurrence rollup for duplicate blocks.""" + + block_type: str # system_prompt | skill_prompt + duplicate_block_count: int # distinct duplicate fingerprints touching this type + occurrence_count: int # total occurrences within this type + chars_duplicated: int # ACTUAL duplicated chars attributable within this type + + +@dataclass +class PromptDuplicateShadow: + """Advisory report of exact duplicate blocks in system/skill prompts. + + SHADOW/ADVISORY ONLY: this measures static prompt duplication; it never + rewrites, dedups, or otherwise mutates prompts, and its char/token figures + must never be reported as realized savings. + """ + + enabled: bool + item_count: int # system/skill prompt items scanned + scanned_block_types: list[str] + duplicate_group_count: int + total_duplicate_occurrences: int + total_chars_duplicated: int # ACTUAL duplicated chars (advisory, not realized) + advisory_est_duplicate_tokens_chars_div_4: int # ADVISORY chars/4, NOT actual tokens + by_block_type: list[PromptDuplicateTypeCount] + top_duplicate_blocks: list[PromptDuplicateBlock] + notes: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Prompt Dedup A/B — OFFLINE SIMULATION structures (system/skill prompts only) +# --------------------------------------------------------------------------- + +# The reference placeholder a *simulated* replacement would leave in place of a +# later duplicate occurrence. Used for accounting only -- ContextPilot never +# emits this string into a real payload. ```` / ```` are filled with +# the canonical prompt type and the salted block fingerprint. +PROMPT_DEDUP_AB_REFERENCE_TEMPLATE = ( + "[Prompt duplicate omitted in simulation; canonical=:]" +) + +# Safe candidate classes, simulated separately. The skill-only class is the +# lowest-risk first canary candidate; the others are reported but higher risk. +PROMPT_DEDUP_AB_CLASSES = ( + "same_type_skill_prompt_only", + "same_type_system_prompt_only", + "cross_type_system_skill", +) + + +@dataclass +class PromptDedupABClass: + """Simulated A/B accounting for one candidate class. + + All figures are OFFLINE SIMULATION over static system/skill prompt text and + are NOT realized savings -- ContextPilot performs no replacement at runtime. + ``chars_delta_simulated`` is signed: positive means the simulated reference + replacement would shrink the payload, negative means it would grow it (a + short duplicate replaced by a longer placeholder). + + Actual-token fields are populated ONLY when an exact tokenizer backend is + configured; otherwise they are ``None`` and ``tokenizer_status`` is + ``"unavailable"`` -- never a fabricated chars/4 figure. + """ + + candidate_class: str + risk_label: str # "low" (canary candidate) | "high" + candidate_group_count: int # distinct exact-duplicate block groups + replacement_occurrence_count: int # occurrences beyond the first, summed + chars_before: int # chars of all candidate occurrences + chars_after_simulated: int # first kept full, later -> reference str + chars_delta_simulated: int # chars_before - chars_after_simulated + tokenizer_status: str # "available" | "unavailable" + actual_tokens_before: int | None # only when tokenizer available + actual_tokens_after: int | None # only when tokenizer available + actual_tokens_delta: int | None # only when tokenizer available + note: str + + +@dataclass +class PromptDedupABSimulation: + """Offline A/B simulation harness for prompt dedup (system/skill prompts). + + OFFLINE SIMULATION + MEASUREMENT ONLY. This is the evidence gate to evaluate + *before* any canary replacement: it scans only ``system_prompt`` / + ``skill_prompt`` LLM-bound blocks, keeps the first occurrence of every exact + duplicate and replaces only later occurrences in a *simulated* accounting. It + never mutates the DB, runtime, or emitted prompts, and its char/token deltas + are NOT realized savings. + """ + + enabled: bool + item_count: int # system/skill prompt items scanned + scanned_block_types: list[str] + tokenizer_status: str # "available" | "unavailable" + tokenizer_backend: str | None # backend name when available, else None + reference_string_template: str + classes: list[PromptDedupABClass] + notes: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Worker Context Routing — SHADOW MODE structures (P0 data collection only) +# --------------------------------------------------------------------------- + + +@dataclass +class RouterLabelCount: + """Aggregate over all blocks assigned one router label.""" + + route_label: str + block_count: int # distinct fingerprints with this label + occurrence_count: int # total occurrences across the window + total_est_tokens: int # est tokens these blocks occupy (occ * est) + est_candidate_tokens: int # ADVISORY routable tokens (0 unless routable) + + +@dataclass +class RouterReasonCount: + """Aggregate keyed by (block_type, route_label, reason_code).""" + + block_type: str + route_label: str + reason_code: str + block_count: int + occurrence_count: int + total_est_tokens: int + est_candidate_tokens: int + + +@dataclass +class RouterCandidateBlock: + """A single routable-candidate fingerprint (salted hash + counters only).""" + + block_hash: str + block_type: str + route_label: str + reason_code: str + occurrences: int + char_length: int + est_tokens: int + est_candidate_tokens: int # ADVISORY upper bound only + + +@dataclass +class WorkerRoutingShadow: + """Shadow-mode worker-context routing report (P0: data collection only).""" + + enabled: bool + item_count: int # LLM-bound items classified + classified_block_count: int # distinct fingerprints classified + total_occurrences: int + must_keep_block_count: int + must_keep_occurrence_count: int + est_must_keep_tokens: int + est_candidate_tokens_total: int # ADVISORY routable ceiling + est_drop_candidate_tokens: int # ADVISORY + est_summarizable_candidate_tokens: int # ADVISORY + label_counts: list[RouterLabelCount] + reason_counts: list[RouterReasonCount] + top_candidate_blocks: list[RouterCandidateBlock] + notes: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Parent Aggregation Artifacts — SHADOW MODE structures (P0 telemetry only) +# --------------------------------------------------------------------------- + + +@dataclass +class ArtifactSourceCount: + """Provenance counter: occurrences of one artifact body from one source.""" + + source_type: str + count: int + + +@dataclass +class ParentAggregationGroup: + """One EXACT artifact body observed 2+ times across parent/worker contexts. + + Salted hash + counters only -- never the body text. + """ + + content_hash: str + artifact_kind: str + canonical_source_type: str # dominant origin, chosen deterministically + occurrences: int + char_length: int + est_tokens: int + est_duplicate_tokens: int # ADVISORY: (occurrences - 1) * est_tokens + source_type_counts: list[ArtifactSourceCount] # provenance: tool_result xN, ... + + +@dataclass +class ArtifactKindStat: + """Aggregate over all candidate artifact bodies of one kind.""" + + artifact_kind: str + group_count: int # distinct bodies of this kind + occurrence_count: int # total occurrences of those bodies + duplicate_group_count: int # bodies seen >= 2 times + est_tokens: int # sum of est tokens for distinct bodies + est_duplicate_tokens: int # ADVISORY duplicate tokens for this kind + + +@dataclass +class ParentAggregationArtifacts: + """Shadow-mode parent-aggregation artifact report (P0: telemetry only).""" + + enabled: bool + item_count: int # candidate artifact items considered + artifact_body_count: int # distinct bodies (groups) + total_occurrences: int + duplicate_group_count: int + est_total_tokens: int # est tokens for distinct bodies + est_duplicate_tokens: int # ADVISORY duplicate-artifact tokens + by_kind: list[ArtifactKindStat] + source_type_counts: list[ArtifactSourceCount] # provenance across candidates + top_duplicate_groups: list[ParentAggregationGroup] + notes: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Top-level report +# --------------------------------------------------------------------------- + + +@dataclass +class OpportunityReport: + date: str + since_hours: int + all_sessions: bool + salt_fingerprint: str + tool_message_count: int + total_tool_output_chars: int + total_tool_output_est_tokens: int + exact_duplicate_groups: list[DuplicateToolOutput] + duplicate_tool_output_groups: int + duplicate_tool_output_wasted_tokens: int + repeated_block_count: int + repeated_block_wasted_tokens: int + repeated_blocks: list[RepeatedBlock] + large_tool_outputs_by_tool: list[ToolSizeStat] + heavy_sessions: list[HeavySession] + # Token-monitor/provenance-profiler rollup (source enums + numeric aggregates only). + provenance_profile: ProvenanceProfile + telemetry: TelemetryCoverage + # LLM-bound block analysis (system/skill prompts, prompts, tool results). + llm_bound_item_count: int + llm_block_types: list[BlockTypeStat] + cross_type_block_groups: list[CrossTypeBlockGroup] + cross_type_wasted_tokens: int + # Prompt duplicate shadow (system/skill prompts only; advisory, never realized). + prompt_duplicates: PromptDuplicateShadow + # Prompt dedup A/B simulation (system/skill prompts only; offline, never realized). + prompt_dedup_ab: PromptDedupABSimulation + # Worker Context Routing shadow mode (P0 data collection; never prunes). + worker_routing: WorkerRoutingShadow + # Parent Aggregation Artifacts shadow mode (P0 telemetry; never dedups). + parent_aggregation: ParentAggregationArtifacts + notes: list[str] = field(default_factory=list) + + +# --------------------------------------------------------------------------- +# In-memory content carriers (hold raw text for hashing ONLY; never emitted) +# --------------------------------------------------------------------------- + + +@dataclass +class _ToolMessage: + tool_name: str | None + content: str + + +@dataclass +class _LLMContent: + """A chunk of content that Hermes would actually send to the LLM. + + Held in-memory only for hashing; ``content`` must never be emitted. + """ + + block_type: str + content: str diff --git a/contextpilot/hermes_opportunities/privacy.py b/contextpilot/hermes_opportunities/privacy.py new file mode 100644 index 0000000..488c27c --- /dev/null +++ b/contextpilot/hermes_opportunities/privacy.py @@ -0,0 +1,47 @@ +"""Privacy primitives: salted hashing and the forbidden-output guard. + +These functions are the privacy backbone of the analyzer. Message/tool/system +text may be read into memory for hashing, but it must never reach an output +file; ``_assert_no_forbidden_keys`` is the defensive backstop that enforces it. +""" +from __future__ import annotations + +import hashlib + +# Columns we are explicitly forbidden from EMITTING in any report. We may read +# message content in-memory for hashing, but it must never reach an output file. +FORBIDDEN_OUTPUT_KEYS = { + "content", + "system_prompt", + "reasoning", + "reasoning_content", + "reasoning_details", + "tool_calls", + "codex_reasoning_items", + "codex_message_items", +} + + +def _salted_hash(text: str, salt: str, *, length: int = 16) -> str: + return hashlib.sha256(f"{salt}:{text}".encode("utf-8", "replace")).hexdigest()[:length] + + +def _salt_fingerprint(salt: str) -> str: + # Confirms a salt was applied without revealing it. + return hashlib.sha256(f"fingerprint:{salt}".encode()).hexdigest()[:12] + + +def _assert_no_forbidden_keys(data: dict) -> None: + """Defensive guard: ensure no forbidden raw-content key reached the output.""" + + def walk(obj): + if isinstance(obj, dict): + for k, v in obj.items(): + if k in FORBIDDEN_OUTPUT_KEYS: + raise RuntimeError(f"refusing to emit forbidden key: {k}") + walk(v) + elif isinstance(obj, list): + for item in obj: + walk(item) + + walk(data) diff --git a/contextpilot/hermes_opportunities/prompt_dedup_canary.py b/contextpilot/hermes_opportunities/prompt_dedup_canary.py new file mode 100644 index 0000000..cee6afa --- /dev/null +++ b/contextpilot/hermes_opportunities/prompt_dedup_canary.py @@ -0,0 +1,340 @@ +"""Default-OFF prompt-dedup canary (the only runtime prompt mutation path). + +Everything else in this package is measurement/shadow/simulation only. This +module is the single, narrowly-scoped place where ContextPilot may *actually* +replace prompt text bound for the LLM -- and only when an operator has opted in +via an environment variable, and only for the lowest-risk duplicate class. + +Risk gate (all conditions must hold before a single character is changed): + +* Mode must be ``canary``. The mode is read from + ``CONTEXTPILOT_PROMPT_DEDUP_MODE`` (``off`` | ``shadow`` | ``canary``) and + defaults to ``off``. ``off`` and ``shadow`` never mutate the payload. +* The escape-hatch env ``CONTEXTPILOT_PROMPT_DEDUP_DISABLE`` (any truthy value) + forces ``off`` regardless of the mode var -- an immediate kill switch. +* Only the ``same_type_skill_prompt_only`` class is eligible: an EXACT duplicate + block whose every occurrence is inside ``skill_prompt`` content. Duplicates + confined to ``system_prompt`` and cross-type ``system_prompt``/``skill_prompt`` + blocks are NEVER replaced. +* Only ``skill_prompt`` items are ever rewritten. ``system_prompt``, + ``user_prompt``, ``assistant_context`` and ``tool_result`` content is never + touched. +* The first occurrence of each duplicate is kept verbatim; only later exact + occurrences are replaced, and only when the deterministic reference string is + strictly shorter than the line it replaces (never grows the payload). +* Any block matching the safety denylist (instruction / safety / security / + tool / auth / secret / must / never / always / required / ...) is left + unchanged even in canary mode. + +The reference string carries only a low-cardinality prompt-type enum and a +salted block hash -- never raw prompt content. Telemetry is metadata-only: +mode/class enums and integer counters; no prompt text and no realized-savings +claim unless an actual mutation occurred. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from typing import Iterable + +from .models import PROMPT_DUPLICATE_BLOCK_TYPES, _LLMContent +from .privacy import _assert_no_forbidden_keys, _salted_hash + +# Environment controls. ``off`` is the default and the safe state. +PROMPT_DEDUP_MODE_ENV = "CONTEXTPILOT_PROMPT_DEDUP_MODE" +PROMPT_DEDUP_DISABLE_ENV = "CONTEXTPILOT_PROMPT_DEDUP_DISABLE" +PROMPT_DEDUP_MODES = ("off", "shadow", "canary") +DEFAULT_PROMPT_DEDUP_MODE = "off" + +# The only duplicate class this canary will ever act on. +CANARY_DEDUP_CLASS = "same_type_skill_prompt_only" + +# Deterministic placeholder left in place of a later duplicate occurrence. Unlike +# the A/B simulation template ("... omitted in simulation ..."), this string is +# really emitted into the payload, so it is labelled as a real ContextPilot +# replacement. ```` / ```` are low-cardinality only. +PROMPT_DEDUP_CANARY_REFERENCE_TEMPLATE = ( + "[ContextPilot dedup: duplicate skill_prompt block omitted; ref=:]" +) + +# Safety denylist. If any of these case-insensitive substrings appears in a +# duplicate block, the block is left untouched even in canary mode. The list is +# deliberately broad: it is always safe to skip a replacement, never safe to +# silently rewrite a hard instruction / safety / security / auth / secret line. +SAFETY_DENYLIST = ( + "instruction", + "instructions", + "safety", + "security", + "secure", + "tool", + "auth", + "authenticate", + "authentication", + "authorization", + "secret", + "credential", + "password", + "api key", + "token", + "must", + "never", + "always", + "required", + "require", + "do not", + "don't", + "important", + "critical", + "mandatory", + "forbidden", + "permission", + "sensitive", + "confidential", + "policy", + "verify", +) + + +@dataclass +class PromptDedupCanaryResult: + """Metadata-only outcome of a canary pass. No raw prompt text, ever. + + ``chars_saved`` / ``blocks_replaced`` are REALIZED figures and are non-zero + only when ``mode == 'canary'`` and an actual replacement occurred. The + ``candidate_*`` fields are advisory (what a canary *would* replace) and are + populated in ``shadow`` mode for visibility without mutating anything. + """ + + mode: str # off | shadow | canary + prompt_dedup_class: str # always CANARY_DEDUP_CLASS + mutated: bool # True only if a real replacement happened + item_count: int # system/skill prompt items scanned + skill_item_count: int # skill_prompt items among them + candidate_block_count: int # eligible skill-only duplicate groups + candidate_chars: int # advisory chars later occurrences occupy + blocks_replaced: int # REALIZED replacements (canary only) + chars_saved: int # REALIZED chars saved (canary only) + denylisted_block_count: int # skill-only duplicate groups skipped by denylist + notes: list[str] = field(default_factory=list) + + +def _truthy(value: str | None) -> bool: + return bool(value) and value.strip().lower() not in ("", "0", "false", "no", "off") + + +def resolve_prompt_dedup_mode(env: dict | None = None) -> str: + """Resolve the active prompt-dedup mode, defaulting to the safe ``off``. + + Unknown values fall back to ``off``. The escape-hatch disable variable, when + truthy, forces ``off`` regardless of the mode variable. + """ + source = os.environ if env is None else env + if _truthy(source.get(PROMPT_DEDUP_DISABLE_ENV)): + return "off" + raw = (source.get(PROMPT_DEDUP_MODE_ENV) or DEFAULT_PROMPT_DEDUP_MODE).strip().lower() + return raw if raw in PROMPT_DEDUP_MODES else DEFAULT_PROMPT_DEDUP_MODE + + +def _is_denied(block: str) -> bool: + """Conservative safety gate: any denylist substring blocks replacement.""" + low = block.lower() + return any(keyword in low for keyword in SAFETY_DENYLIST) + + +def _reference_string(canonical_type: str, block_hash: str) -> str: + return PROMPT_DEDUP_CANARY_REFERENCE_TEMPLATE.replace( + "", canonical_type + ).replace("", block_hash) + + +def _build_eligibility( + contents: list[_LLMContent], *, salt: str, min_block_chars: int +) -> tuple[dict[str, str], int, int, int]: + """Fingerprint system/skill blocks and return the canary-eligible hashes. + + Returns ``(eligible, candidate_chars, denylisted, item_count)`` where + ``eligible`` maps a block hash to its reference string. A hash is eligible + only when it is an EXACT duplicate (occurs 2+ times), every occurrence is in + ``skill_prompt`` content (same_type_skill_prompt_only), and the block does + not match the safety denylist. + """ + # hash -> {char_length, types: {block_type: occ}, denied} + agg: dict[str, dict] = {} + item_count = 0 + for item in contents: + bt = item.block_type + if bt not in PROMPT_DUPLICATE_BLOCK_TYPES: + continue + item_count += 1 + for raw_line in item.content.split("\n"): + block = raw_line.strip() + if len(block) < min_block_chars: + continue + h = _salted_hash(block, salt) + entry = agg.get(h) + if entry is None: + agg[h] = { + "char_length": len(block), + "types": {bt: 1}, + "denied": _is_denied(block), + } + else: + entry["types"][bt] = entry["types"].get(bt, 0) + 1 + + eligible: dict[str, str] = {} + candidate_chars = 0 + denylisted = 0 + for h, entry in agg.items(): + types = entry["types"] + occ = sum(types.values()) + if occ < 2: + continue # not a duplicate -> nothing to replace + # same_type_skill_prompt_only: every occurrence is a skill prompt block. + if set(types) != {"skill_prompt"}: + continue + if entry["denied"]: + denylisted += 1 + continue + eligible[h] = _reference_string("skill_prompt", h) + # Advisory: chars the later (replaceable) occurrences currently occupy. + candidate_chars += (occ - 1) * entry["char_length"] + return eligible, candidate_chars, denylisted, item_count + + +def apply_prompt_dedup_canary( + contents: Iterable[_LLMContent], + *, + salt: str, + min_block_chars: int, + mode: str | None = None, + env: dict | None = None, +) -> PromptDedupCanaryResult: + """Run the prompt-dedup canary over LLM-bound content. + + ``contents`` are the in-memory ``_LLMContent`` items bound for the LLM. In + ``canary`` mode this MUTATES the ``content`` of eligible ``skill_prompt`` + items in place (keeping the first occurrence, replacing later exact + duplicates with a deterministic reference string). In ``off`` and ``shadow`` + modes nothing is mutated. + + ``mode`` overrides the resolved environment mode (used by tests); otherwise + the mode comes from :func:`resolve_prompt_dedup_mode`. + """ + items = list(contents) + resolved = mode if mode is not None else resolve_prompt_dedup_mode(env) + if resolved not in PROMPT_DEDUP_MODES: + resolved = DEFAULT_PROMPT_DEDUP_MODE + + skill_item_count = sum(1 for it in items if it.block_type == "skill_prompt") + + if resolved == "off": + # Safe default: no scan, no candidates, no savings. + return PromptDedupCanaryResult( + mode="off", + prompt_dedup_class=CANARY_DEDUP_CLASS, + mutated=False, + item_count=0, + skill_item_count=skill_item_count, + candidate_block_count=0, + candidate_chars=0, + blocks_replaced=0, + chars_saved=0, + denylisted_block_count=0, + notes=["prompt-dedup canary off (default): payload unchanged"], + ) + + eligible, candidate_chars, denylisted, item_count = _build_eligibility( + items, salt=salt, min_block_chars=min_block_chars + ) + + if resolved == "shadow": + # Measure what a canary would replace, but never touch the payload. + return PromptDedupCanaryResult( + mode="shadow", + prompt_dedup_class=CANARY_DEDUP_CLASS, + mutated=False, + item_count=item_count, + skill_item_count=skill_item_count, + candidate_block_count=len(eligible), + candidate_chars=candidate_chars, + blocks_replaced=0, + chars_saved=0, + denylisted_block_count=denylisted, + notes=["prompt-dedup canary shadow: candidates measured, payload unchanged"], + ) + + # --- canary: the ONLY branch that mutates LLM-bound payload --------------- + blocks_replaced = 0 + chars_saved = 0 + consumed: set[str] = set() # hashes whose first (kept) occurrence was seen + for item in items: + if item.block_type != "skill_prompt": + continue # never touch system/user/assistant/tool content + if not eligible: + break + new_lines: list[str] = [] + changed = False + for raw_line in item.content.split("\n"): + block = raw_line.strip() + if len(block) < min_block_chars: + new_lines.append(raw_line) + continue + h = _salted_hash(block, salt) + ref = eligible.get(h) + if ref is None: + new_lines.append(raw_line) + continue + if h not in consumed: + consumed.add(h) # keep the first occurrence verbatim + new_lines.append(raw_line) + continue + # Later exact duplicate: replace only when it actually shrinks the line. + if len(ref) < len(raw_line): + new_lines.append(ref) + blocks_replaced += 1 + chars_saved += len(raw_line) - len(ref) + changed = True + else: + new_lines.append(raw_line) + if changed: + item.content = "\n".join(new_lines) + + notes = ["prompt-dedup canary active: same_type_skill_prompt_only duplicates only"] + if denylisted: + notes.append(f"{denylisted} skill-only duplicate group(s) skipped by safety denylist") + return PromptDedupCanaryResult( + mode="canary", + prompt_dedup_class=CANARY_DEDUP_CLASS, + mutated=blocks_replaced > 0, + item_count=item_count, + skill_item_count=skill_item_count, + candidate_block_count=len(eligible), + candidate_chars=candidate_chars, + blocks_replaced=blocks_replaced, + chars_saved=chars_saved, + denylisted_block_count=denylisted, + notes=notes, + ) + + +def build_canary_telemetry_record(result: PromptDedupCanaryResult) -> dict: + """Build a metadata-only telemetry record for a canary pass. + + The aggregate ``chars_saved`` counter gains the prompt-dedup contribution + ONLY when a real mutation occurred (canary). ``off``/``shadow`` contribute 0 + to the total while still reporting the separated ``prompt_dedup_*`` fields. + Contains only mode/class enums and integer counters -- never prompt text. + """ + realized = result.chars_saved if result.mutated else 0 + record = { + "prompt_dedup_mode": result.mode, + "prompt_dedup_class": result.prompt_dedup_class, + "prompt_dedup_blocks_replaced": result.blocks_replaced if result.mutated else 0, + # Separated field: always present, mirrors the realized prompt-dedup save. + "prompt_dedup_chars_saved": realized, + # Aggregate total: includes prompt dedup only when a mutation occurred. + "chars_saved": realized, + } + _assert_no_forbidden_keys(record) + return record diff --git a/contextpilot/hermes_opportunities/provenance.py b/contextpilot/hermes_opportunities/provenance.py new file mode 100644 index 0000000..4cf5dad --- /dev/null +++ b/contextpilot/hermes_opportunities/provenance.py @@ -0,0 +1,65 @@ +"""Privacy-safe provenance profiling -- the standalone token-monitor view. + +``build_provenance_profile`` rolls a list of :class:`HeavySession` rows up by +their provenance ``source`` into numeric token/usage aggregates. It is the +lightweight "where are my tokens going?" report: no SciPy/RAG machinery, no +content, prompts, reasoning, or raw session ids/hashes -- only low-cardinality +source labels and integer counters. +""" +from __future__ import annotations + +from collections.abc import Iterable + +from .models import ( + UNKNOWN_SOURCE, + HeavySession, + ProvenanceProfile, + ProvenanceSourceStat, +) + + +def build_provenance_profile( + heavy_sessions: Iterable[HeavySession], +) -> ProvenanceProfile: + """Aggregate heavy sessions into a per-source token-usage profile. + + Sessions with no recorded ``source`` are folded into the ``"unknown"`` + bucket so the output stays a low-cardinality enum view. ``by_source`` rows + are returned sorted by descending total tokens (then source name) for stable, + human-meaningful ordering. + """ + buckets: dict[str, ProvenanceSourceStat] = {} + for session in heavy_sessions: + source = session.source or UNKNOWN_SOURCE + stat = buckets.get(source) + if stat is None: + stat = ProvenanceSourceStat( + source=source, + session_count=0, + input_tokens=0, + output_tokens=0, + message_count=0, + tool_call_count=0, + api_call_count=0, + total_tokens=0, + ) + buckets[source] = stat + stat.session_count += 1 + stat.input_tokens += session.input_tokens + stat.output_tokens += session.output_tokens + stat.message_count += session.message_count + stat.tool_call_count += session.tool_call_count + stat.api_call_count += session.api_call_count + stat.total_tokens += session.input_tokens + session.output_tokens + + by_source = sorted( + buckets.values(), key=lambda s: (-s.total_tokens, s.source) + ) + return ProvenanceProfile( + source_count=len(by_source), + session_count=sum(s.session_count for s in by_source), + input_tokens=sum(s.input_tokens for s in by_source), + output_tokens=sum(s.output_tokens for s in by_source), + total_tokens=sum(s.total_tokens for s in by_source), + by_source=by_source, + ) diff --git a/contextpilot/hermes_opportunities/report.py b/contextpilot/hermes_opportunities/report.py new file mode 100644 index 0000000..f393a75 --- /dev/null +++ b/contextpilot/hermes_opportunities/report.py @@ -0,0 +1,464 @@ +"""Report assembly and serialization (privacy-safe output). + +``build_report`` runs every detector over the in-memory carriers and assembles +a privacy-safe ``OpportunityReport`` (hashes + counters + enums only). +``write_report`` serializes it to JSON + Markdown, guarded by +``_assert_no_forbidden_keys`` so no raw content can ever reach disk. +""" +from __future__ import annotations + +import json +from dataclasses import asdict +from pathlib import Path + +from .aggregation import ( + DEFAULT_MIN_ARTIFACT_CHARS, + analyze_parent_aggregation_artifacts, +) +from .dedup_ab import simulate_prompt_dedup_ab +from .detection import ( + analyze_llm_bound_blocks, + detect_exact_duplicate_tool_outputs, + detect_prompt_duplicate_blocks, + detect_repeated_blocks, + summarize_tool_sizes, +) +from .models import ( + DEFAULT_LARGE_OUTPUT_CHARS, + DEFAULT_MIN_BLOCK_CHARS, + DEFAULT_MIN_BLOCK_REPEAT, + DEFAULT_TOP_N, + HeavySession, + OpportunityReport, + TelemetryCoverage, + _est_tokens, + _LLMContent, + _ToolMessage, +) +from .privacy import _assert_no_forbidden_keys, _salt_fingerprint +from .provenance import build_provenance_profile +from .routing import analyze_worker_routing_shadow +from .tokenizer import TokenizerBackend + + +def _dedup_ab_summary(ab) -> str: + """One-line rollup of the prompt-dedup A/B simulation for the summary block.""" + if not ab.enabled: + return "disabled" + groups = sum(c.candidate_group_count for c in ab.classes) + chars_delta = sum(c.chars_delta_simulated for c in ab.classes) + return ( + f"{groups} candidate groups, {chars_delta} simulated chars delta, " + f"tokenizer {ab.tokenizer_status}" + ) + + +def build_report( + *, + date: str, + since_hours: int, + salt: str, + tool_messages: list[_ToolMessage], + heavy_sessions: list[HeavySession], + telemetry: TelemetryCoverage, + llm_contents: list[_LLMContent] | None = None, + all_sessions: bool = False, + min_block_chars: int = DEFAULT_MIN_BLOCK_CHARS, + min_block_repeat: int = DEFAULT_MIN_BLOCK_REPEAT, + large_output_chars: int = DEFAULT_LARGE_OUTPUT_CHARS, + top_n: int = DEFAULT_TOP_N, + worker_routing_shadow: bool = True, + parent_aggregation_shadow: bool = True, + prompt_duplicate_shadow: bool = True, + prompt_dedup_ab: bool = True, + prompt_dedup_ab_tokenizer: TokenizerBackend | None = None, + min_artifact_chars: int = DEFAULT_MIN_ARTIFACT_CHARS, +) -> OpportunityReport: + dups = detect_exact_duplicate_tool_outputs(tool_messages, salt=salt, top_n=top_n) + blocks = detect_repeated_blocks( + tool_messages, + salt=salt, + min_block_chars=min_block_chars, + min_repeat=min_block_repeat, + top_n=top_n, + ) + sizes = summarize_tool_sizes( + tool_messages, large_output_chars=large_output_chars, top_n=top_n + ) + + llm_contents = llm_contents or [] + block_type_stats, cross_groups = analyze_llm_bound_blocks( + llm_contents, + salt=salt, + min_block_chars=min_block_chars, + min_repeat=min_block_repeat, + top_n=top_n, + ) + + prompt_duplicates = detect_prompt_duplicate_blocks( + llm_contents, + salt=salt, + min_block_chars=min_block_chars, + top_n=top_n, + enabled=prompt_duplicate_shadow, + ) + + prompt_dedup_ab_sim = simulate_prompt_dedup_ab( + llm_contents, + salt=salt, + min_block_chars=min_block_chars, + tokenizer=prompt_dedup_ab_tokenizer, + enabled=prompt_dedup_ab, + ) + + worker_routing = analyze_worker_routing_shadow( + llm_contents, + salt=salt, + large_output_chars=large_output_chars, + min_repeat=min_block_repeat, + top_n=top_n, + enabled=worker_routing_shadow, + ) + + parent_aggregation = analyze_parent_aggregation_artifacts( + llm_contents, + salt=salt, + min_artifact_chars=min_artifact_chars, + top_n=top_n, + enabled=parent_aggregation_shadow, + ) + provenance_profile = build_provenance_profile(heavy_sessions) + + total_chars = sum(len(m.content) for m in tool_messages) + dup_wasted = sum(d.est_wasted_tokens for d in dups) + block_wasted = sum(b.est_wasted_tokens for b in blocks) + cross_wasted = sum(g.est_wasted_tokens for g in cross_groups) + + notes = [ + "content-aware analysis: message/tool text was hashed in-memory only and never written to reports", + "all identifiers are salted SHA-256 fingerprints; counters are aggregates", + "token figures use tokenizer-measured telemetry only; no chars/4 fallback is used", + "session 'source', 'tool_name', and block_type are emitted verbatim as low-cardinality enums, not raw text", + "llm-bound scan covers only content sent to the LLM: system/skill prompts, active user/assistant/tool messages", + "worker-routing section is SHADOW MODE P0: it labels blocks for a future router but never drops/summarizes context", + "parent-aggregation section is SHADOW MODE P0 telemetry: it groups exact artifact bodies but never dedups/replaces context", + "prompt-duplicate section is ADVISORY ONLY (system/skill prompts): it counts exact duplicate prompt blocks but never rewrites/dedups prompts; its chars/tokens are NOT realized savings", + "prompt-dedup A/B section is OFFLINE SIMULATION ONLY (system/skill prompts): it simulates keeping the first occurrence and replacing later duplicate occurrences to measure candidate savings; it performs NO runtime replacement/canonicalization and its deltas are NOT realized savings; it is the evidence gate before any canary replace", + ] + if all_sessions: + notes.append("all-sessions mode: time window ignored; scanned all non-archived sessions/active messages") + if not tool_messages: + notes.append("no tool-output messages observed in the selected window") + if not llm_contents: + notes.append("no llm-bound content observed in the selected window") + + return OpportunityReport( + date=date, + since_hours=since_hours, + all_sessions=all_sessions, + salt_fingerprint=_salt_fingerprint(salt), + tool_message_count=len(tool_messages), + total_tool_output_chars=total_chars, + total_tool_output_est_tokens=_est_tokens(total_chars), + exact_duplicate_groups=dups, + duplicate_tool_output_groups=len(dups), + duplicate_tool_output_wasted_tokens=dup_wasted, + repeated_block_count=len(blocks), + repeated_block_wasted_tokens=block_wasted, + repeated_blocks=blocks, + large_tool_outputs_by_tool=sizes, + heavy_sessions=heavy_sessions, + provenance_profile=provenance_profile, + telemetry=telemetry, + llm_bound_item_count=len(llm_contents), + llm_block_types=block_type_stats, + cross_type_block_groups=cross_groups, + cross_type_wasted_tokens=cross_wasted, + prompt_duplicates=prompt_duplicates, + prompt_dedup_ab=prompt_dedup_ab_sim, + worker_routing=worker_routing, + parent_aggregation=parent_aggregation, + notes=notes, + ) + + +def write_report(report: OpportunityReport, out_dir: Path) -> tuple[Path, Path]: + out_dir.mkdir(parents=True, exist_ok=True) + data = asdict(report) + _assert_no_forbidden_keys(data) + + json_path = out_dir / f"opportunities_{report.date}.json" + md_path = out_dir / f"opportunities_{report.date}.md" + json_path.write_text( + json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + + t = report.telemetry + window = "all sessions (no time window)" if report.all_sessions else f"last {report.since_hours}h" + md = [ + f"# ContextPilot Hermes opportunity scan — {report.date}", + "", + f"Window: {window}", + f"Salt fingerprint: `{report.salt_fingerprint}`", + "", + "## Summary", + f"- Tool-output messages: {report.tool_message_count}", + f"- Total tool-output tokens (est): {report.total_tool_output_est_tokens}", + f"- Exact duplicate groups: {report.duplicate_tool_output_groups} " + f"(~{report.duplicate_tool_output_wasted_tokens} wasted tokens)", + f"- Repeated blocks: {report.repeated_block_count} " + f"(~{report.repeated_block_wasted_tokens} wasted tokens)", + f"- LLM-bound items scanned: {report.llm_bound_item_count}", + f"- Cross-type repeated blocks: {len(report.cross_type_block_groups)} " + f"(~{report.cross_type_wasted_tokens} wasted tokens)", + f"- Prompt duplicates (system/skill, advisory): " + f"{report.prompt_duplicates.duplicate_group_count} groups, " + f"{report.prompt_duplicates.total_chars_duplicated} chars duplicated " + f"(~{report.prompt_duplicates.advisory_est_duplicate_tokens_chars_div_4} " + f"advisory chars/4 tokens) — NOT realized savings", + f"- Prompt dedup A/B (simulation): " + f"{_dedup_ab_summary(report.prompt_dedup_ab)} — NOT realized savings", + f"- Telemetry: {t.events} events, {t.chars_saved} chars saved by processing; " + f"tokenizer tokens saved={t.tokens_saved}, ratio={t.coverage_ratio_pct}%", + f"- Worker routing (shadow): {report.worker_routing.classified_block_count} blocks " + f"classified, {report.worker_routing.must_keep_block_count} must-keep, " + f"~{report.worker_routing.est_candidate_tokens_total} advisory candidate tokens", + f"- Parent aggregation (shadow): {report.parent_aggregation.duplicate_group_count} " + f"duplicate artifact groups, " + f"~{report.parent_aggregation.est_duplicate_tokens} advisory duplicate tokens", + f"- Provenance profile: {report.provenance_profile.session_count} sessions across " + f"{report.provenance_profile.source_count} sources, " + f"{report.provenance_profile.total_tokens} actual input+output tokens", + "", + "## Token profile by source", + ] + for src in report.provenance_profile.by_source: + md.append( + f"- {src.source}: sessions={src.session_count} input={src.input_tokens} " + f"output={src.output_tokens} total={src.total_tokens} " + f"messages={src.message_count} tools={src.tool_call_count} api_calls={src.api_call_count}" + ) + md.append("") + md.append("## LLM-bound redundancy by block type") + for bt in report.llm_block_types: + md.append( + f"- {bt.block_type}: items={bt.item_count} blocks={bt.block_count} " + f"unique={bt.unique_block_count} repeated={bt.repeated_block_count} " + f"~redundant={bt.est_redundant_tokens} tokens" + ) + md.append("") + md.append("## Cross-type repeated blocks (same block, multiple sources)") + for g in report.cross_type_block_groups: + spread = ", ".join(f"{tc.block_type}x{tc.count}" for tc in g.type_occurrences) + md.append( + f"- `{g.block_hash}` types=[{', '.join(g.block_types)}] ({spread}) " + f"chars={g.char_length} ~wasted={g.est_wasted_tokens} tokens" + ) + md.append("") + pd = report.prompt_duplicates + md.append("## Prompt duplicate blocks — system/skill (advisory only)") + if not pd.enabled: + md.append("- disabled") + else: + md.append( + f"- Scanned prompt types: {', '.join(pd.scanned_block_types)} " + f"(items: {pd.item_count})" + ) + md.append( + f"- Duplicate groups: {pd.duplicate_group_count} " + f"(occurrences: {pd.total_duplicate_occurrences})" + ) + md.append( + f"- Chars duplicated (actual): {pd.total_chars_duplicated} " + f"(~{pd.advisory_est_duplicate_tokens_chars_div_4} advisory chars/4 tokens, " + f"NOT actual tokens, NOT a realized saving)" + ) + md.append("") + md.append("### Occurrences by prompt type") + for tc in pd.by_block_type: + md.append( + f"- {tc.block_type}: dup_blocks={tc.duplicate_block_count} " + f"occ={tc.occurrence_count} chars_duplicated={tc.chars_duplicated}" + ) + md.append("") + md.append("### Top duplicate prompt blocks (hashed)") + for b in pd.top_duplicate_blocks: + md.append( + f"- `{b.block_hash}` types=[{', '.join(b.block_types)}] " + f"x{b.occurrences} chars={b.char_length} " + f"chars_duplicated={b.chars_duplicated} " + f"(~{b.advisory_est_duplicate_tokens_chars_div_4} advisory chars/4 tokens)" + ) + md.append("") + ab = report.prompt_dedup_ab + md.append("## Prompt dedup A/B simulation — system/skill (offline only)") + if not ab.enabled: + md.append("- disabled") + else: + md.append( + f"- Scanned prompt types: {', '.join(ab.scanned_block_types)} " + f"(items: {ab.item_count})" + ) + backend = ab.tokenizer_backend or "none" + md.append( + f"- Tokenizer: status={ab.tokenizer_status} backend={backend} " + f"(actual tokens shown only when an exact backend is configured)" + ) + md.append(f"- Simulated reference string: `{ab.reference_string_template}`") + md.append( + "- OFFLINE SIMULATION ONLY — no runtime replacement/canonicalization; " + "deltas below are candidate figures, NOT realized savings" + ) + md.append("") + md.append("### Candidate classes (simulated separately)") + for c in ab.classes: + line = ( + f"- {c.candidate_class} [risk={c.risk_label}]: " + f"groups={c.candidate_group_count} " + f"replacements={c.replacement_occurrence_count} " + f"chars_before={c.chars_before} " + f"chars_after_simulated={c.chars_after_simulated} " + f"chars_delta_simulated={c.chars_delta_simulated}" + ) + if c.tokenizer_status == "available": + line += ( + f" actual_tokens_before={c.actual_tokens_before} " + f"actual_tokens_after={c.actual_tokens_after} " + f"actual_tokens_delta={c.actual_tokens_delta}" + ) + else: + line += " actual_tokens=unavailable" + md.append(line) + md.append(f" - {c.note}") + md.append("") + md.append("## Top exact-duplicate tool outputs") + for d in report.exact_duplicate_groups: + md.append( + f"- `{d.content_hash}` tool={d.tool_name} x{d.occurrences} " + f"chars={d.char_length} ~wasted={d.est_wasted_tokens} tokens" + ) + md.append("") + md.append("## Top repeated blocks") + for b in report.repeated_blocks: + md.append( + f"- `{b.block_hash}` x{b.occurrences} chars={b.char_length} " + f"~wasted={b.est_wasted_tokens} tokens" + ) + md.append("") + md.append("## Large tool outputs by tool") + for s in report.large_tool_outputs_by_tool: + md.append( + f"- {s.tool_name}: count={s.output_count} total_chars={s.total_chars} " + f"max={s.max_chars} avg={s.avg_chars} large(>=thresh)={s.large_output_count}" + ) + md.append("") + md.append("## Heavy sessions (hashed)") + for h in report.heavy_sessions: + md.append( + f"- `{h.session_hash}` source={h.source} input={h.input_tokens} " + f"output={h.output_tokens} msgs={h.message_count} tools={h.tool_call_count} " + f"apis={h.api_call_count}" + ) + md.append("") + md.append("## Telemetry from processed payload") + md.extend( + [ + f"- Events: {t.events}", + f"- Chars saved by ContextPilot processing: {t.chars_saved}", + f"- Tokenizer tokens saved: {t.tokens_saved}", + f"- Avg tokenizer tokens / event: {t.avg_tokens_saved_per_event}", + f"- Tokenizer-token ratio: {t.coverage_ratio_pct}%", + f"- Malformed records skipped: {t.malformed_records_skipped}", + ] + ) + md.append("") + wr = report.worker_routing + md.append("## Worker Context Routing — shadow mode (P0, advisory only)") + if not wr.enabled: + md.append("- disabled") + else: + md.append( + f"- Items classified: {wr.item_count} " + f"(distinct fingerprints: {wr.classified_block_count}, " + f"occurrences: {wr.total_occurrences})" + ) + md.append( + f"- Must-keep: {wr.must_keep_block_count} blocks / " + f"{wr.must_keep_occurrence_count} occurrences " + f"(~{wr.est_must_keep_tokens} tokens, never routable)" + ) + md.append( + f"- Advisory candidate tokens: ~{wr.est_candidate_tokens_total} " + f"(drop ~{wr.est_drop_candidate_tokens}, " + f"summarize ~{wr.est_summarizable_candidate_tokens}) — NOT a realized saving" + ) + md.append("") + md.append("### Router labels") + for lc in wr.label_counts: + md.append( + f"- {lc.route_label}: blocks={lc.block_count} " + f"occ={lc.occurrence_count} tokens={lc.total_est_tokens} " + f"~candidate={lc.est_candidate_tokens}" + ) + md.append("") + md.append("### Reason codes (block_type / label / reason)") + for rc in wr.reason_counts: + md.append( + f"- {rc.block_type} / {rc.route_label} / {rc.reason_code}: " + f"blocks={rc.block_count} occ={rc.occurrence_count} " + f"tokens={rc.total_est_tokens} ~candidate={rc.est_candidate_tokens}" + ) + md.append("") + md.append("### Top routable-candidate blocks (hashed)") + for cb in wr.top_candidate_blocks: + md.append( + f"- `{cb.block_hash}` type={cb.block_type} " + f"label={cb.route_label} reason={cb.reason_code} " + f"x{cb.occurrences} chars={cb.char_length} ~candidate={cb.est_candidate_tokens}" + ) + md.append("") + pa = report.parent_aggregation + md.append("## Parent Aggregation Artifacts — shadow mode") + if not pa.enabled: + md.append("- disabled") + else: + md.append( + f"- Candidate artifact items: {pa.item_count} " + f"(distinct bodies: {pa.artifact_body_count}, " + f"occurrences: {pa.total_occurrences})" + ) + md.append( + f"- Duplicate artifact groups: {pa.duplicate_group_count} " + f"(~{pa.est_duplicate_tokens} advisory duplicate tokens of " + f"~{pa.est_total_tokens} distinct-body tokens) — NOT a realized saving, " + f"payloads are unchanged" + ) + md.append("") + md.append("### By artifact kind") + for ks in pa.by_kind: + md.append( + f"- {ks.artifact_kind}: bodies={ks.group_count} " + f"occ={ks.occurrence_count} dup_groups={ks.duplicate_group_count} " + f"tokens={ks.est_tokens} ~dup={ks.est_duplicate_tokens}" + ) + md.append("") + md.append("### Provenance (artifact source types)") + for sc in pa.source_type_counts: + md.append(f"- {sc.source_type}: {sc.count}") + md.append("") + md.append("### Top duplicate artifact groups (hashed)") + for g in pa.top_duplicate_groups: + spread = ", ".join( + f"{sc.source_type}x{sc.count}" for sc in g.source_type_counts + ) + md.append( + f"- `{g.content_hash}` kind={g.artifact_kind} " + f"canonical={g.canonical_source_type} x{g.occurrences} " + f"({spread}) chars={g.char_length} ~dup={g.est_duplicate_tokens} tokens" + ) + md.append("") + md.append("## Notes") + for note in report.notes: + md.append(f"- {note}") + md_path.write_text("\n".join(md) + "\n", encoding="utf-8") + return json_path, md_path diff --git a/contextpilot/hermes_opportunities/routing.py b/contextpilot/hermes_opportunities/routing.py new file mode 100644 index 0000000..9f8d510 --- /dev/null +++ b/contextpilot/hermes_opportunities/routing.py @@ -0,0 +1,325 @@ +"""Worker Context Routing — SHADOW MODE (P0 data collection only). + +Classifies LLM-bound blocks with a conservative, deterministic heuristic and +emits aggregate counters + salted hashes so the labels can be evaluated offline. +Nothing here ever drops, summarizes, replaces, or mutates context: it is pure +reporting/measurement. ``est_candidate_tokens`` is an ADVISORY upper bound on +what a *future* router might route away -- never a realized saving. +""" +from __future__ import annotations + +from typing import Iterable + +from .models import ( + RouterCandidateBlock, + RouterLabelCount, + RouterReasonCount, + WorkerRoutingShadow, + _est_tokens, + _LLMContent, +) +from .privacy import _salted_hash + +# Low-cardinality router labels. These are the *training/eval* labels a future +# small worker-context router would predict. P0 is data-collection only: nothing +# here ever drops, summarizes, or mutates context -- it only classifies blocks +# and emits aggregate counters + salted hashes so the labels can be evaluated +# offline before any online pruning is built. +ROUTER_LABELS = ( + "policy_must_keep", # never droppable (user/system/skill/safety constraints) + "direct_task_hint", # short actionable task signal -- keep + "likely_relevant", # default keep; not obviously prunable + "summarizable_candidate", # large single block that *might* be summarized later + "likely_drop_candidate", # large/repeated tool-like block, candidate to route away +) + +# Labels whose blocks a future router might safely route away. Used only to +# tally *advisory* candidate tokens; P0 never acts on them. +_ROUTABLE_LABELS = ("summarizable_candidate", "likely_drop_candidate") + +# Block-type priority when one fingerprint spans multiple origins: the most +# "must-keep" origin wins, so cross-origin blocks are classified conservatively. +_TYPE_KEEP_PRIORITY = { + "user_prompt": 5, + "system_prompt": 4, + "skill_prompt": 4, + "assistant_context": 2, + "tool_result": 1, + "unknown": 0, +} + +# Cues marking content that must NEVER be dropped even from a tool/assistant +# block: explicit safety / acceptance / hard-constraint language. Matching here +# is intentionally generous -- over-keeping is the safe direction for P0. +_SAFETY_CONSTRAINT_CUES = ( + "must not", + "must never", + "never drop", + "do not delete", + "do not remove", + "do not modify", + "acceptance criteria", + "acceptance test", + "safety", + "must keep", + "you must", + "required:", + "constraint", + "forbidden", + "policy", +) + +# Cues marking a short, actionable task hint worth keeping verbatim. +_TASK_HINT_CUES = ( + "todo", + "next step", + "error:", + "traceback", + "failed", + "fixme", + "task:", + "goal:", + "implement", + "reproduce", +) + + +def classify_router_label( + block_type: str, + content: str, + *, + occurrences: int, + large_output_chars: int, + min_repeat: int, +) -> tuple[str, str]: + """Heuristically assign a worker-routing label + reason code to a block. + + Pure P0 heuristic: no ML, no network, no mutation. Operates on in-memory + text only and returns two low-cardinality enums (``route_label``, + ``reason_code``) -- never the text. The bias is deliberately conservative: + when in doubt, keep. Anything that is a user prompt, a system/skill prompt, + or carries explicit safety/acceptance-constraint language is pinned to + ``policy_must_keep`` and can never become a routable candidate. + """ + low = content.lower() + + # 1. Never-drop by origin: prompts the user/system/skills authored. + if block_type == "user_prompt": + return "policy_must_keep", "user_prompt_never_drop" + if block_type in ("system_prompt", "skill_prompt"): + return "policy_must_keep", "system_or_skill_constraint_never_drop" + + # 2. Never-drop by content: explicit safety / acceptance / hard constraints, + # even inside an assistant or tool block. + if any(cue in low for cue in _SAFETY_CONSTRAINT_CUES): + return "policy_must_keep", "safety_or_acceptance_constraint" + + char_len = len(content) + has_task_hint = any(cue in low for cue in _TASK_HINT_CUES) + + # 3. Short actionable task hints -> keep verbatim. Very large diagnostic + # logs often contain "error:"/"failed"/"traceback"; keep collecting + # them as summarization candidates instead of pinning the whole log. + if has_task_hint and char_len < large_output_chars: + return "direct_task_hint", "actionable_task_signal" + + # 4. Bulky / repeated tool-like material -> routable candidates (advisory). + if block_type in ("tool_result", "assistant_context", "unknown"): + if has_task_hint and char_len >= large_output_chars: + return "summarizable_candidate", "large_actionable_tool_block" + is_large = char_len >= large_output_chars + is_repeated = occurrences >= min_repeat + if is_large and is_repeated: + return "likely_drop_candidate", "large_repeated_tool_block" + if is_repeated: + return "likely_drop_candidate", "repeated_tool_block" + if is_large: + return "summarizable_candidate", "large_single_tool_block" + + # 5. Everything else: keep by default. + return "likely_relevant", "default_keep" + + +def analyze_worker_routing_shadow( + contents: Iterable[_LLMContent], + *, + salt: str, + large_output_chars: int, + min_repeat: int, + top_n: int, + enabled: bool = True, +) -> WorkerRoutingShadow: + """Shadow-mode worker-context routing classifier (P0: data collection only). + + Fingerprints each LLM-bound item, assigns a conservative router label, and + returns aggregate counters + salted hashes for routable candidates. Emits + NO raw text and never mutates/drops context. ``est_candidate_tokens`` is an + advisory upper bound on what a *future* router might route away -- not a + realized saving. + """ + if not enabled: + return WorkerRoutingShadow( + enabled=False, + item_count=0, + classified_block_count=0, + total_occurrences=0, + must_keep_block_count=0, + must_keep_occurrence_count=0, + est_must_keep_tokens=0, + est_candidate_tokens_total=0, + est_drop_candidate_tokens=0, + est_summarizable_candidate_tokens=0, + label_counts=[], + reason_counts=[], + top_candidate_blocks=[], + notes=["worker-routing shadow analysis disabled via flag"], + ) + + # Aggregate occurrences per fingerprint, picking the most must-keep origin + # when one block spans several block types. + agg: dict[str, dict] = {} + item_count = 0 + for item in contents: + content = item.content + if not content: + continue + item_count += 1 + h = _salted_hash(content, salt) + bt = item.block_type + entry = agg.get(h) + if entry is None: + agg[h] = { + "block_type": bt, + "char_length": len(content), + "occurrences": 1, + "content": content, + } + else: + entry["occurrences"] += 1 + cur = entry["block_type"] + bt_pri = _TYPE_KEEP_PRIORITY.get(bt, 0) + cur_pri = _TYPE_KEEP_PRIORITY.get(cur, 0) + if bt_pri > cur_pri or (bt_pri == cur_pri and bt < cur): + entry["block_type"] = bt + + # Classify each unique fingerprint and roll up counters. + label_agg: dict[str, dict] = {} + reason_agg: dict[tuple[str, str, str], dict] = {} + candidates: list[RouterCandidateBlock] = [] + must_keep_blocks = 0 + must_keep_occ = 0 + est_must_keep_tokens = 0 + drop_tokens = 0 + summ_tokens = 0 + + for h, entry in agg.items(): + bt = entry["block_type"] + occ = entry["occurrences"] + char_len = entry["char_length"] + est = _est_tokens(char_len) + total_est = est * occ + label, reason = classify_router_label( + bt, + entry["content"], + occurrences=occ, + large_output_chars=large_output_chars, + min_repeat=min_repeat, + ) + candidate_tokens = total_est if label in _ROUTABLE_LABELS else 0 + + la = label_agg.setdefault( + label, + {"block_count": 0, "occ": 0, "total_est": 0, "candidate": 0}, + ) + la["block_count"] += 1 + la["occ"] += occ + la["total_est"] += total_est + la["candidate"] += candidate_tokens + + ra = reason_agg.setdefault( + (bt, label, reason), + {"block_count": 0, "occ": 0, "total_est": 0, "candidate": 0}, + ) + ra["block_count"] += 1 + ra["occ"] += occ + ra["total_est"] += total_est + ra["candidate"] += candidate_tokens + + if label == "policy_must_keep": + must_keep_blocks += 1 + must_keep_occ += occ + est_must_keep_tokens += total_est + if label == "likely_drop_candidate": + drop_tokens += candidate_tokens + elif label == "summarizable_candidate": + summ_tokens += candidate_tokens + + if candidate_tokens > 0: + candidates.append( + RouterCandidateBlock( + block_hash=h, + block_type=bt, + route_label=label, + reason_code=reason, + occurrences=occ, + char_length=char_len, + est_tokens=est, + est_candidate_tokens=candidate_tokens, + ) + ) + + # Deterministic ordering: label_counts follow the canonical label order; + # reason_counts and candidates sort by a stable key. + label_counts = [ + RouterLabelCount( + route_label=lbl, + block_count=label_agg[lbl]["block_count"], + occurrence_count=label_agg[lbl]["occ"], + total_est_tokens=label_agg[lbl]["total_est"], + est_candidate_tokens=label_agg[lbl]["candidate"], + ) + for lbl in ROUTER_LABELS + if lbl in label_agg + ] + reason_counts = [ + RouterReasonCount( + block_type=bt, + route_label=lbl, + reason_code=reason, + block_count=v["block_count"], + occurrence_count=v["occ"], + total_est_tokens=v["total_est"], + est_candidate_tokens=v["candidate"], + ) + for (bt, lbl, reason), v in sorted(reason_agg.items()) + ] + candidates.sort( + key=lambda c: (c.est_candidate_tokens, c.occurrences, c.block_hash), + reverse=True, + ) + + total_occ = sum(e["occurrences"] for e in agg.values()) + notes = [ + "SHADOW MODE P0: classification only -- no context was dropped, summarized, or mutated", + "route_label/reason_code/block_type are low-cardinality enums; block_hash is a salted SHA-256 fingerprint", + "est_candidate_tokens is ADVISORY (an upper bound for a FUTURE router), not a realized saving", + "user/system/skill prompts and safety/acceptance constraints are pinned to policy_must_keep and never routable", + "classification is conservative: when uncertain, blocks are kept (likely_relevant)", + ] + + return WorkerRoutingShadow( + enabled=True, + item_count=item_count, + classified_block_count=len(agg), + total_occurrences=total_occ, + must_keep_block_count=must_keep_blocks, + must_keep_occurrence_count=must_keep_occ, + est_must_keep_tokens=est_must_keep_tokens, + est_candidate_tokens_total=drop_tokens + summ_tokens, + est_drop_candidate_tokens=drop_tokens, + est_summarizable_candidate_tokens=summ_tokens, + label_counts=label_counts, + reason_counts=reason_counts, + top_candidate_blocks=candidates[:top_n], + notes=notes, + ) diff --git a/contextpilot/hermes_opportunities/telemetry.py b/contextpilot/hermes_opportunities/telemetry.py new file mode 100644 index 0000000..b06f9f9 --- /dev/null +++ b/contextpilot/hermes_opportunities/telemetry.py @@ -0,0 +1,72 @@ +"""ContextPilot telemetry parsing (metadata-only). + +Aggregates the metadata-only ContextPilot telemetry file into a privacy-safe +``TelemetryCoverage``. Never reads message content; tolerates malformed lines +by skipping and counting them. +""" +from __future__ import annotations + +import json +from pathlib import Path + +from .db import _window_cutoff +from .models import TelemetryCoverage + + +def parse_telemetry( + telemetry_path: Path, + *, + since_hours: int, + total_input_tokens: int, + all_sessions: bool = False, +) -> TelemetryCoverage: + """Aggregate the metadata-only ContextPilot telemetry file. + + Tolerates malformed lines (non-JSON, non-dict, missing counters) by + skipping and counting them. Never reads message content. With + ``all_sessions=True`` the time window is ignored. + """ + events = 0 + chars = 0 + tokens = 0 + malformed = 0 + if telemetry_path and telemetry_path.exists(): + cutoff = _window_cutoff(since_hours, all_sessions) + with telemetry_path.open("r", encoding="utf-8", errors="replace") as f: + for raw in f: + line = raw.strip() + if not line: + continue + try: + record = json.loads(line) + except (ValueError, TypeError): + malformed += 1 + continue + if not isinstance(record, dict): + malformed += 1 + continue + ts = record.get("ts") + if cutoff is not None and isinstance(ts, (int, float)) and ts < cutoff: + continue + cs = record.get("chars_saved") + if not isinstance(cs, (int, float)): + malformed += 1 + continue + events += 1 + chars += int(cs) + if record.get("actual_token_status") == "available": + saved = record.get("actual_tokens_saved") + if isinstance(saved, (int, float)): + tokens += int(saved) + + denom = tokens + total_input_tokens + coverage = (tokens / denom * 100.0) if denom else 0.0 + avg = (tokens / events) if events else 0.0 + return TelemetryCoverage( + events=events, + chars_saved=chars, + tokens_saved=tokens, + avg_tokens_saved_per_event=round(avg, 2), + coverage_ratio_pct=round(coverage, 2), + malformed_records_skipped=malformed, + ) diff --git a/contextpilot/hermes_opportunities/tokenizer.py b/contextpilot/hermes_opportunities/tokenizer.py new file mode 100644 index 0000000..e3e226d --- /dev/null +++ b/contextpilot/hermes_opportunities/tokenizer.py @@ -0,0 +1,74 @@ +"""Optional, opt-in exact tokenizer helper for offline simulation only. + +Mirrors the philosophy of the actual-token prompt shadow (#53): exact token +counts are surfaced ONLY when an explicitly configured tokenizer backend is +available. By default this module resolves to ``None`` (status ``unavailable``), +and callers must never fabricate token figures in that case. + +This helper runs in-memory over block text purely to produce integer counts; it +never emits or persists any text. It is used by the prompt-dedup A/B *simulation* +harness, which measures candidate savings offline and never mutates runtime +payloads. +""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Callable + + +@dataclass +class TokenizerBackend: + """A resolved exact tokenizer. + + ``name`` is a low-cardinality backend identifier safe to emit (e.g. + ``"tiktoken:cl100k_base"``); ``count`` maps a string to its exact token + count. Counting happens in-memory only -- the text is never emitted. + """ + + name: str + count: Callable[[str], int] + + +def resolve_tokenizer(spec: object | None) -> TokenizerBackend | None: + """Resolve an explicitly-configured exact tokenizer backend, or ``None``. + + Off by default: ``spec=None`` (the default everywhere) returns ``None`` so + the A/B harness reports ``tokenizer_status=unavailable`` and emits no actual + token fields. ``spec`` may be: + + * ``None`` -> not configured; returns ``None``. + * a :class:`TokenizerBackend` -> used directly (test/dependency injection). + * a string ``"tiktoken:"`` -> best-effort load of a tiktoken + encoding. If tiktoken (or the encoding) is unavailable, returns ``None`` + rather than guessing; the caller then reports ``unavailable``. + + Any backend that cannot be resolved exactly yields ``None`` -- we never + substitute a chars/4 estimate behind an "actual tokens" label. + """ + if spec is None: + return None + if isinstance(spec, TokenizerBackend): + return spec + if isinstance(spec, str): + return _resolve_named(spec) + return None + + +def _resolve_named(spec: str) -> TokenizerBackend | None: + spec = spec.strip() + if not spec: + return None + if spec.startswith("tiktoken:"): + encoding = spec.split(":", 1)[1].strip() or "cl100k_base" + try: + import tiktoken # type: ignore + + enc = tiktoken.get_encoding(encoding) + except Exception: # noqa: BLE001 - missing dep/encoding -> unavailable, never fake + return None + return TokenizerBackend( + name=f"tiktoken:{encoding}", + count=lambda text: len(enc.encode(text)), + ) + # Unknown backend spec: stay unavailable rather than fabricate counts. + return None diff --git a/contextpilot/provenance_linking.py b/contextpilot/provenance_linking.py new file mode 100644 index 0000000..f012270 --- /dev/null +++ b/contextpilot/provenance_linking.py @@ -0,0 +1,293 @@ +"""General claim→evidence provenance-linking dataset and shadow baseline. + +This module is deliberately separate from the artifact-dedup canary. It is the +training/eval substrate for the more general design: build examples containing +raw source blocks plus assistant claims, run a shadow evidence linker, and report +metrics without changing the online LLM payload. + +Privacy contract: committed fixtures should be synthetic. Real trace exports may +contain raw content and must live under a local/gitignored path (for example +``~/contextpilot/provenance_datasets``). +""" +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Iterable + +PROVENANCE_LINKING_SCHEMA_VERSION = 1 +MUTABLE_EVIDENCE_TYPES = {"tool_result", "assistant_context", "worker_output", "file_snippet", "web_result"} +CLAIM_BLOCK_TYPES = {"assistant_context", "assistant", "parent_summary"} +SOURCE_BLOCK_TYPES = {"tool_result", "worker_output", "file_snippet", "web_result", "assistant_context"} +ALLOWED_RELATIONS = { + "copied", + "supports", + "supports_candidate", + "extracted", + "extracted_support", + "summarized_support", + "aggregated_support", + "contradicts", + "insufficient", +} + +_TOKEN_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_./:-]{2,}|\d+(?:\.\d+)?|[\u4e00-\u9fff]{2,}") +_SENTENCE_SPLIT_RE = re.compile(r"(?<=[。！？.!?])\s+|\n+") + + +@dataclass(frozen=True) +class ProvenanceBlock: + block_id: str + block_type: str + text: str + metadata: dict[str, str | int | float | bool] = field(default_factory=dict) + + +@dataclass(frozen=True) +class ProvenanceClaim: + claim_id: str + block_id: str + text: str + start: int + end: int + claim_type: str = "factual" + + +@dataclass(frozen=True) +class ProvenanceLink: + claim_id: str + evidence_block_id: str + start: int + end: int + relation: str + confidence: float + method: str + + +@dataclass(frozen=True) +class ProvenanceExample: + example_id: str + domain: str + blocks: list[ProvenanceBlock] + claims: list[ProvenanceClaim] + gold_links: list[ProvenanceLink] = field(default_factory=list) + + +def _stable_id(prefix: str, *parts: str) -> str: + h = hashlib.sha256("\x1f".join(parts).encode("utf-8")).hexdigest()[:16] + return f"{prefix}_{h}" + + +def _tokens(text: str) -> set[str]: + return {m.group(0).lower() for m in _TOKEN_RE.finditer(text)} + + +def extract_claims(block: ProvenanceBlock, *, max_claims: int = 12, min_chars: int = 12) -> list[ProvenanceClaim]: + """Extract candidate factual claims from an assistant/summary block. + + This is intentionally cheap and recall-oriented; it is a shadow data builder, + not a production semantic claim extractor. A trained linker can replace this + later while keeping the dataset schema stable. + """ + if block.block_type not in CLAIM_BLOCK_TYPES: + return [] + claims: list[ProvenanceClaim] = [] + pos = 0 + for part in _SENTENCE_SPLIT_RE.split(block.text): + text = part.strip() + if not text or len(text) < min_chars: + pos += len(part) + 1 + continue + start = block.text.find(text, pos) + if start < 0: + start = block.text.find(text) + end = start + len(text) + # Prefer factual-looking claims: numbers, paths, result words, or Chinese + # technical verbs. Skip pure planning/hedging lines when possible. + lower = text.lower() + factual_cues = ( + any(ch.isdigit() for ch in text) + or any(tok in lower for tok in ["passed", "failed", "error", "warning", "commit", "pr", "test", "source", "evidence"]) + or any(tok in text for tok in ["通过", "失败", "错误", "证据", "来自", "支持", "省", "测试", "结论"]) + ) + if factual_cues: + cid = _stable_id("claim", block.block_id, str(start), text) + claims.append(ProvenanceClaim(cid, block.block_id, text, start, end)) + if len(claims) >= max_claims: + break + pos = max(end, pos + len(part)) + return claims + + +def shadow_link_claims( + example: ProvenanceExample, + *, + top_k: int = 1, + min_overlap: int = 2, +) -> list[ProvenanceLink]: + """Cheap shadow provenance linker: claim→top-k earlier evidence snippets. + + This baseline never folds payloads. It creates training/eval candidates using + exact substring, numeric/path overlap, and token Jaccard signals. A future + trained model should replace the scoring function, not the safety contract. + """ + by_id = {b.block_id: b for b in example.blocks} + order = {b.block_id: i for i, b in enumerate(example.blocks)} + links: list[ProvenanceLink] = [] + for claim in example.claims: + claim_tokens = _tokens(claim.text) + if not claim_tokens: + continue + scored: list[tuple[float, ProvenanceBlock, int, int, str]] = [] + claim_block_order = order.get(claim.block_id, 10**9) + for block in example.blocks: + if block.block_id == claim.block_id or block.block_type not in SOURCE_BLOCK_TYPES: + continue + if order.get(block.block_id, 10**9) >= claim_block_order: + continue + start = block.text.lower().find(claim.text.lower()) + if start >= 0: + scored.append((1.0, block, start, start + len(claim.text), "copied")) + continue + block_tokens = _tokens(block.text) + overlap = claim_tokens & block_tokens + if len(overlap) < min_overlap: + continue + score = len(overlap) / max(len(claim_tokens), 1) + # Pick a compact evidence window around the first overlapping token. + first_positions = [block.text.lower().find(tok) for tok in overlap if block.text.lower().find(tok) >= 0] + if first_positions: + center = min(first_positions) + start = max(0, center - 220) + end = min(len(block.text), center + 420) + else: + start, end = 0, min(len(block.text), 640) + relation = "extracted" if score >= 0.55 else "supports_candidate" + scored.append((score, block, start, end, relation)) + scored.sort(key=lambda row: (-row[0], order[row[1].block_id], row[1].block_id)) + for score, block, start, end, relation in scored[:top_k]: + links.append( + ProvenanceLink( + claim_id=claim.claim_id, + evidence_block_id=block.block_id, + start=start, + end=end, + relation=relation, + confidence=round(float(score), 4), + method="shadow_lexical_v1", + ) + ) + return links + + +def example_from_trace_case(case: dict, *, domain: str = "hermes_trace") -> ProvenanceExample: + """Convert a trace-validation JSONL case into a provenance-linking example.""" + blocks: list[ProvenanceBlock] = [] + claims: list[ProvenanceClaim] = [] + for idx, msg in enumerate(case.get("messages", [])): + block_type = str(msg.get("block_type") or "unknown") + text = str(msg.get("content") or "") + bid = f"b{idx:04d}_{block_type}" + block = ProvenanceBlock( + block_id=bid, + block_type=block_type, + text=text, + metadata={"role": msg.get("role") or "", "index": idx}, + ) + blocks.append(block) + claims.extend(extract_claims(block)) + return ProvenanceExample( + example_id=str(case.get("case_id") or _stable_id("ex", json.dumps(case, sort_keys=True)[:1000])), + domain=domain, + blocks=blocks, + claims=claims, + ) + + +def to_jsonable(example: ProvenanceExample, *, shadow_links: list[ProvenanceLink] | None = None) -> dict: + out = asdict(example) + out["schema_version"] = PROVENANCE_LINKING_SCHEMA_VERSION + if shadow_links is not None: + out["shadow_links"] = [asdict(link) for link in shadow_links] + return out + + +def read_jsonl_examples(path: Path) -> list[ProvenanceExample]: + examples: list[ProvenanceExample] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + raw = json.loads(line) + links = [ProvenanceLink(**l) for l in raw.get("gold_links", [])] + bad_relations = sorted({link.relation for link in links if link.relation not in ALLOWED_RELATIONS}) + if bad_relations: + raise ValueError(f"unsupported provenance relation(s): {bad_relations}") + examples.append( + ProvenanceExample( + example_id=raw["example_id"], + domain=raw.get("domain", "unknown"), + blocks=[ProvenanceBlock(**b) for b in raw.get("blocks", [])], + claims=[ProvenanceClaim(**c) for c in raw.get("claims", [])], + gold_links=links, + ) + ) + return examples + + +def write_jsonl(path: Path, rows: Iterable[dict]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + + +def evaluate_shadow(examples: Iterable[ProvenanceExample]) -> dict: + """Evaluate shadow links against examples that include gold links. + + Matching is intentionally evidence-block level for generality: a model can + learn better span boundaries later, while the early metric asks whether it + found the right source evidence for each claim. + """ + tp = fp = fn = 0 + case_rows = [] + predicted_total = gold_total = 0 + for ex in examples: + pred = shadow_link_claims(ex) + pred_pairs = {(l.claim_id, l.evidence_block_id) for l in pred} + gold_pairs = {(l.claim_id, l.evidence_block_id) for l in ex.gold_links} + case_tp = len(pred_pairs & gold_pairs) + case_fp = len(pred_pairs - gold_pairs) + case_fn = len(gold_pairs - pred_pairs) + tp += case_tp + fp += case_fp + fn += case_fn + predicted_total += len(pred_pairs) + gold_total += len(gold_pairs) + case_rows.append( + { + "example_id": ex.example_id, + "claim_count": len(ex.claims), + "gold_links": len(gold_pairs), + "predicted_links": len(pred_pairs), + "tp": case_tp, + "fp": case_fp, + "fn": case_fn, + } + ) + return { + "schema_version": PROVENANCE_LINKING_SCHEMA_VERSION, + "corpus": "provenance_linking_shadow_eval", + "claim_scope": "shadow claim→evidence linking; does not mutate online context", + "example_count": len(case_rows), + "gold_links": gold_total, + "predicted_links": predicted_total, + "shadow_link_tp": tp, + "shadow_link_fp": fp, + "shadow_link_fn": fn, + "shadow_link_precision": tp / (tp + fp) if tp + fp else 1.0, + "shadow_link_recall": tp / (tp + fn) if tp + fn else 1.0, + "cases": case_rows, + } diff --git a/contextpilot/server/http_server.py b/contextpilot/server/http_server.py index cd3681f..a99e09d 100644 --- a/contextpilot/server/http_server.py +++ b/contextpilot/server/http_server.py @@ -1733,11 +1733,10 @@ async def _intercept_and_forward(request: Request, api_format: str): if total_reordered > 0 or total_deduped > 0 or total_slimmed > 0: saved = chars_before_slim - chars_after_slim - saved_tokens = saved // 4 if saved > 0 else 0 logger.info( f"Intercept ({api_format}): reordered {total_reordered}, " f"deduped {total_deduped}, slimmed {total_slimmed} " - f"(saved {saved:,} chars ≈ {saved_tokens:,} tokens)" + f"(saved {saved:,} chars; tokenizer accounting required for tokens)" ) _dedup_result = DedupResult() diff --git a/contextpilot/trace_validation/__init__.py b/contextpilot/trace_validation/__init__.py new file mode 100644 index 0000000..b15790d --- /dev/null +++ b/contextpilot/trace_validation/__init__.py @@ -0,0 +1,82 @@ +"""Trace-derived validation-set framework for ContextPilot. + +A fixed, replayable corpus + a gate runner so any future accuracy-affecting or +runtime-payload-changing change can be checked against stable Hermes traces +instead of an ad-hoc "run once and see". + +Two pieces, with a deliberate privacy split: + +* :mod:`.builder` reads a local Hermes SQLite DB (read-only) and exports a fixed + JSONL corpus under a local, gitignored directory. Raw content lives ONLY in + that local artifact; the sidecar manifest is privacy-safe. +* :mod:`.runner` replays the corpus through ContextPilot's optimization in a + baseline (``off``) mode vs a configured candidate mode and checks + accuracy-preservation invariants, emitting a privacy-safe pass/fail report. + +Both reuse the analyzer's read-only DB loaders, salted hashing, and forbidden-key +guard so the privacy primitives stay defined in one place. +""" +from __future__ import annotations + +from .builder import ( + DEFAULT_OUT_DIR, + DEFAULT_SALT, + DEFAULT_STATE_DB, + build_manifest, + case_to_json, + load_trace_cases, + write_validation_set, +) +from .builder import main as build_main +from .models import ( + DEFAULT_CASE_LIMIT, + DEFAULT_MIN_INPUT_TOKENS, + MUTABLE_BLOCK_TYPE, + VALIDATION_SET_SCHEMA_VERSION, + TraceCase, + TraceMessage, + ValidationCaseResult, + ValidationReport, +) +from .runner import ( + INVARIANT_NAMES, + assert_report_privacy_safe, + check_invariants, + load_cases, + optimize_case, + render_markdown, + report_to_dict, + run_validation, +) +from .runner import main as run_main + +__all__ = [ + # models / constants + "VALIDATION_SET_SCHEMA_VERSION", + "DEFAULT_CASE_LIMIT", + "DEFAULT_MIN_INPUT_TOKENS", + "MUTABLE_BLOCK_TYPE", + "INVARIANT_NAMES", + "TraceMessage", + "TraceCase", + "ValidationCaseResult", + "ValidationReport", + # builder + "DEFAULT_STATE_DB", + "DEFAULT_OUT_DIR", + "DEFAULT_SALT", + "load_trace_cases", + "case_to_json", + "build_manifest", + "write_validation_set", + "build_main", + # runner + "load_cases", + "optimize_case", + "check_invariants", + "run_validation", + "report_to_dict", + "render_markdown", + "assert_report_privacy_safe", + "run_main", +] diff --git a/contextpilot/trace_validation/builder.py b/contextpilot/trace_validation/builder.py new file mode 100644 index 0000000..1249e38 --- /dev/null +++ b/contextpilot/trace_validation/builder.py @@ -0,0 +1,384 @@ +"""Trace-derived validation-set builder (read-only, local-only output). + +Reads a Hermes local SQLite state DB in read-only mode and exports a *fixed* +JSONL corpus of replayable cases under a local, gitignored directory. The corpus +captures the exact LLM-bound payload (system/skill prompts + ordered messages) +so future accuracy-affecting or runtime-payload-changing changes can be validated +against a stable set instead of an ad-hoc "run once and see". + +Privacy contract: + +* The DB is opened ``mode=ro`` and never written. +* Raw ``content`` is written ONLY into the local JSONL artifact (default under + the user's home, and the in-repo convenience dir is gitignored). It is never + committed and never placed in the privacy-safe manifest. +* Case ids are salted hashes of the session id -- the raw session id is never + emitted. The manifest carries only a salt *fingerprint*, counters, and enums. +* Sampling is conservative by default (small ``--limit``, a time window) so the + artifact stays a representative sample, not a full export. + +This module deliberately reuses the analyzer's read-only DB helpers and salted +hashing rather than re-implementing them, so the privacy primitives stay in one +place. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import json +from dataclasses import asdict +from pathlib import Path + +from contextpilot.hermes_opportunities.db import ( + _classify_system_prompt, + _connect_readonly, + _message_block_type, + _window_cutoff, +) +from contextpilot.hermes_opportunities.privacy import ( + _assert_no_forbidden_keys, + _salt_fingerprint, + _salted_hash, +) + +from .models import ( + DEFAULT_CASE_LIMIT, + DEFAULT_MIN_INPUT_TOKENS, + DEFAULT_MIN_MESSAGES, + DEFAULT_SINCE_HOURS, + VALIDATION_SET_SCHEMA_VERSION, + TraceCase, + TraceMessage, +) + + +# Manifest dictionaries are passed through the shared privacy guard, which +# rejects forbidden key names such as "system_prompt" or "user_prompt". Keep the +# human meaning but avoid those exact raw-content-shaped key names in reports. +_BLOCK_TYPE_REPORT_KEYS = { + "system_prompt": "system_ctx", + "skill_prompt": "skill_ctx", + "user_prompt": "user_ctx", + "assistant_context": "assistant_ctx", + "tool_result": "tool_result", +} + + +def _report_block_type_key(block_type: str) -> str: + return _BLOCK_TYPE_REPORT_KEYS.get(block_type, block_type.replace("prompt", "ctx")) + +DEFAULT_STATE_DB = Path("/root/.hermes/state.db") +# Default output lives OUTSIDE the repo, under the user's home, so a generated +# corpus can never be committed by accident. The in-repo ``.contextpilot_validation/`` +# convenience dir is also gitignored for users who prefer to keep it local-to-repo. +DEFAULT_OUT_DIR = Path.home() / "contextpilot" / "validation_sets" +DEFAULT_SALT = "contextpilot-trace-validation-v1" + + +def _order_clause(mcols: set[str]) -> str: + """Pick a deterministic message ordering column, preferring explicit ids.""" + if "id" in mcols: + return "messages.id ASC" + if "timestamp" in mcols: + return "messages.timestamp ASC, messages.rowid ASC" + return "messages.rowid ASC" + + +def load_trace_cases( + db_path: Path, + *, + since_hours: int, + salt: str, + limit: int, + all_sessions: bool = False, + min_input_tokens: int = DEFAULT_MIN_INPUT_TOKENS, + min_messages: int = DEFAULT_MIN_MESSAGES, + include_system_prompt: bool = True, +) -> list[TraceCase]: + """Load up to ``limit`` replayable cases from the Hermes state DB. + + Sessions are filtered by the time window (unless ``all_sessions``), archival + flag, and ``min_input_tokens``, then ordered by ``input_tokens`` descending so + the heaviest (most worth validating) sessions are sampled first. Per session + the optional classified system prompt is emitted first, followed by active + messages in deterministic order. Content is read in-memory only. + """ + cutoff = _window_cutoff(since_hours, all_sessions) + conn = _connect_readonly(db_path) + try: + scols = {row[1] for row in conn.execute("PRAGMA table_info(sessions)")} + mcols = {row[1] for row in conn.execute("PRAGMA table_info(messages)")} + if "id" not in scols: + return [] + + wanted = ["id", "source", "input_tokens"] + select_cols = [c if c in scols else f"NULL AS {c}" for c in wanted] + has_sys = include_system_prompt and "system_prompt" in scols + select_cols.append("system_prompt" if has_sys else "NULL AS system_prompt") + + where: list[str] = [] + params: list[object] = [] + if cutoff is not None and "started_at" in scols: + where.append("started_at >= ?") + params.append(cutoff) + if "archived" in scols: + where.append("archived = 0") + if min_input_tokens > 0 and "input_tokens" in scols: + where.append("input_tokens >= ?") + params.append(min_input_tokens) + sql = f"SELECT {', '.join(select_cols)} FROM sessions" + if where: + sql += " WHERE " + " AND ".join(where) + if "input_tokens" in scols: + sql += " ORDER BY input_tokens DESC" + session_rows = conn.execute(sql, params).fetchall() + + # Pre-resolve the per-session message query shape once. + has_content = "content" in mcols + has_role = "role" in mcols + has_tool = "tool_name" in mcols + has_session_fk = "session_id" in mcols + msg_select = ", ".join( + [ + "messages.role" if has_role else "NULL AS role", + "messages.content", + "messages.tool_name" if has_tool else "NULL AS tool_name", + ] + ) + order_by = _order_clause(mcols) + + cases: list[TraceCase] = [] + for sid, source, input_tokens, system_prompt in session_rows: + if len(cases) >= limit: + break + messages: list[TraceMessage] = [] + + if has_sys and system_prompt is not None: + text = str(system_prompt) + messages.append( + TraceMessage( + role="system", + block_type=_classify_system_prompt(text), + content=text, + ) + ) + + if has_content and has_session_fk: + mwhere = ["messages.content IS NOT NULL", "messages.session_id = ?"] + mparams: list[object] = [sid] + if has_role: + mwhere.append( + "messages.role IN ('system', 'user', 'assistant', 'tool')" + ) + if "active" in mcols: + mwhere.append("messages.active = 1") + msql = ( + f"SELECT {msg_select} FROM messages " + f"WHERE {' AND '.join(mwhere)} ORDER BY {order_by}" + ) + for role, content, tool_name in conn.execute(msql, mparams): + if content is None: + continue + messages.append( + TraceMessage( + role=role, + block_type=_message_block_type(role, tool_name), + content=str(content), + ) + ) + + if len(messages) < min_messages: + continue + cases.append( + TraceCase( + case_id=_salted_hash(str(sid), salt), + source=source, + input_tokens=int(input_tokens or 0), + message_count=len(messages), + messages=messages, + ) + ) + finally: + conn.close() + return cases + + +def case_to_json(case: TraceCase) -> dict: + """Serialize a case for the LOCAL JSONL artifact (includes raw content).""" + return { + "schema_version": VALIDATION_SET_SCHEMA_VERSION, + "case_id": case.case_id, + "source": case.source, + "input_tokens": case.input_tokens, + "message_count": case.message_count, + "messages": [asdict(m) for m in case.messages], + } + + +def build_manifest( + cases: list[TraceCase], + *, + date: str, + salt: str, + since_hours: int, + all_sessions: bool, + min_input_tokens: int, + include_system_prompt: bool, + corpus_filename: str, +) -> dict: + """Build the PRIVACY-SAFE manifest: counters, enums, and a salt fingerprint. + + Never contains raw content; passed through the forbidden-key guard before it + is returned so a future regression cannot smuggle content in via this path. + """ + by_source: dict[str, int] = {} + by_block_type: dict[str, int] = {} + total_messages = 0 + for case in cases: + key = case.source or "unknown" + by_source[key] = by_source.get(key, 0) + 1 + total_messages += case.message_count + for m in case.messages: + report_key = _report_block_type_key(m.block_type) + by_block_type[report_key] = by_block_type.get(report_key, 0) + 1 + + manifest = { + "schema_version": VALIDATION_SET_SCHEMA_VERSION, + "generated_date": date, + "salt_fingerprint": _salt_fingerprint(salt), + "window": "all_sessions" if all_sessions else f"last_{since_hours}h", + "since_hours": since_hours, + "all_sessions": all_sessions, + "min_input_tokens": min_input_tokens, + "include_system_prompt": include_system_prompt, + "corpus_file": corpus_filename, + "case_count": len(cases), + "total_messages": total_messages, + "cases_by_source": by_source, + "messages_by_block_type": by_block_type, + "privacy_note": ( + "manifest is metadata-only (salted ids, counters, enums); the corpus " + "JSONL holds raw content and is local-only / gitignored, never committed" + ), + } + _assert_no_forbidden_keys(manifest) + return manifest + + +def write_validation_set( + cases: list[TraceCase], manifest: dict, out_dir: Path, corpus_filename: str +) -> tuple[Path, Path]: + """Write the local JSONL corpus and its privacy-safe manifest sidecar.""" + out_dir.mkdir(parents=True, exist_ok=True) + corpus_path = out_dir / corpus_filename + manifest_path = out_dir / (corpus_filename + ".manifest.json") + with corpus_path.open("w", encoding="utf-8") as f: + for case in cases: + f.write(json.dumps(case_to_json(case), ensure_ascii=False) + "\n") + manifest_path.write_text( + json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" + ) + return corpus_path, manifest_path + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Build a trace-derived ContextPilot validation set from a local " + "Hermes state DB. Raw content is written ONLY to a local/gitignored " + "JSONL corpus; the manifest is privacy-safe." + ) + ) + parser.add_argument("--state-db", type=Path, default=DEFAULT_STATE_DB) + parser.add_argument("--out", type=Path, default=None, help="output directory") + parser.add_argument("--since-hours", type=int, default=DEFAULT_SINCE_HOURS) + parser.add_argument( + "--all-sessions", + action="store_true", + help="ignore --since-hours; scan all non-archived sessions", + ) + parser.add_argument( + "--limit", + type=int, + default=DEFAULT_CASE_LIMIT, + help=f"max number of cases to export (default {DEFAULT_CASE_LIMIT})", + ) + parser.add_argument( + "--min-input-tokens", + type=int, + default=DEFAULT_MIN_INPUT_TOKENS, + help="only include sessions with at least this many input tokens", + ) + parser.add_argument( + "--min-messages", + type=int, + default=DEFAULT_MIN_MESSAGES, + help="drop cases with fewer than this many LLM-bound messages", + ) + parser.add_argument( + "--include-system-prompt", + dest="include_system_prompt", + action="store_true", + default=True, + help="include the session system/skill prompt as the first message (default)", + ) + parser.add_argument( + "--no-system-prompt", + dest="include_system_prompt", + action="store_false", + help="exclude session system/skill prompts from the corpus", + ) + parser.add_argument("--salt", default=DEFAULT_SALT) + parser.add_argument("--date", default=dt.date.today().isoformat()) + args = parser.parse_args(argv) + + if not args.state_db.exists(): + raise SystemExit(f"Hermes state DB not found: {args.state_db}") + + out_dir = args.out if args.out is not None else DEFAULT_OUT_DIR + corpus_filename = f"validation_set_{args.date}.jsonl" + + # Cron-safe: never dump a traceback (which could echo the DB path or SQL); + # emit only the exception class name and a non-zero exit code. + try: + cases = load_trace_cases( + args.state_db, + since_hours=args.since_hours, + salt=args.salt, + limit=args.limit, + all_sessions=args.all_sessions, + min_input_tokens=args.min_input_tokens, + min_messages=args.min_messages, + include_system_prompt=args.include_system_prompt, + ) + manifest = build_manifest( + cases, + date=args.date, + salt=args.salt, + since_hours=args.since_hours, + all_sessions=args.all_sessions, + min_input_tokens=args.min_input_tokens, + include_system_prompt=args.include_system_prompt, + corpus_filename=corpus_filename, + ) + corpus_path, manifest_path = write_validation_set( + cases, manifest, out_dir, corpus_filename + ) + except Exception as exc: # noqa: BLE001 - cron-safe: class name only, no payload + print(json.dumps({"ok": False, "error": type(exc).__name__})) + return 1 + + # Stdout is privacy-safe: paths + counters only (raw content stays in the file). + print( + json.dumps( + { + "ok": True, + "corpus": str(corpus_path), + "manifest": str(manifest_path), + "case_count": manifest["case_count"], + "total_messages": manifest["total_messages"], + }, + ensure_ascii=False, + ) + ) + return 0 diff --git a/contextpilot/trace_validation/models.py b/contextpilot/trace_validation/models.py new file mode 100644 index 0000000..dcaf204 --- /dev/null +++ b/contextpilot/trace_validation/models.py @@ -0,0 +1,112 @@ +"""Data structures for the trace-derived validation-set framework. + +Two layers, with a deliberate privacy split: + +* :class:`TraceCase` / :class:`TraceMessage` are the *local* corpus carriers. + They DO hold raw LLM-bound text (``content``) because the validation set must + be able to replay the exact payload ContextPilot would process. They are only + ever serialized into the gitignored local artifact produced by the builder -- + never into a committed fixture or a runner report. +* :class:`ValidationCaseResult` / :class:`ValidationReport` are the *report* + carriers. They are privacy-safe by construction: salted case ids, integer + counters, low-cardinality enums and pass/fail booleans only -- never raw + prompt/message/tool text. + +The runner's report is additionally passed through the analyzer's +``_assert_no_forbidden_keys`` guard and a raw-substring scan before it is +emitted, so a regression that accidentally threads content into a report fails +loudly instead of leaking. +""" +from __future__ import annotations + +from dataclasses import dataclass, field + +# Bumped when the on-disk JSONL case schema changes in a non-additive way. +VALIDATION_SET_SCHEMA_VERSION = 1 + +# Conservative sampling defaults: the builder is meant to capture a small, +# representative corpus, not exfiltrate a whole history. +DEFAULT_CASE_LIMIT = 25 +DEFAULT_SINCE_HOURS = 24 +DEFAULT_MIN_INPUT_TOKENS = 0 +DEFAULT_MIN_MESSAGES = 1 + +# The only block type the prompt-dedup canary is ever allowed to mutate. Every +# other block type is "protected" and must survive optimization byte-identical. +MUTABLE_BLOCK_TYPE = "skill_prompt" + + +@dataclass +class TraceMessage: + """One ordered LLM-bound message in a replayed case. + + ``content`` is RAW text and is local-only: it appears in the gitignored + corpus artifact, never in a committed fixture or a runner report. + """ + + role: str | None + block_type: str + content: str + + +@dataclass +class TraceCase: + """A single replayable case derived from one Hermes session. + + ``case_id`` is a salted hash of the session id (never the raw id). The + counters are privacy-safe; ``messages`` carries raw content and is local-only. + """ + + case_id: str + source: str | None + input_tokens: int + message_count: int + messages: list[TraceMessage] + + +@dataclass +class ValidationCaseResult: + """Privacy-safe per-case outcome of a validation run. + + Salted id + counters + enums + invariant booleans only. ``chars_saved`` and + ``blocks_replaced`` are REALIZED processed-payload figures (actual before/ + after character delta), not opportunity counts. Token figures are populated + only when an exact tokenizer backend was configured. + """ + + case_id: str + source: str | None + message_count: int + skill_item_count: int + mutated: bool + blocks_replaced: int + chars_saved: int # REALIZED before-after char delta + invariants: dict[str, bool] + passed: bool + failed_invariants: list[str] = field(default_factory=list) + actual_tokens_before: int | None = None + actual_tokens_after: int | None = None + actual_tokens_saved: int | None = None + + +@dataclass +class ValidationReport: + """Privacy-safe summary of a whole validation run (gate + accounting).""" + + schema_version: int + generated_date: str + salt_fingerprint: str + baseline_mode: str + candidate_mode: str + case_count: int + passed: bool # overall gate + passed_cases: int + failed_cases: int + total_blocks_replaced: int + total_chars_saved: int # REALIZED before-after char delta + tokenizer_status: str # "available" | "unavailable" + tokenizer_backend: str | None + total_actual_tokens_saved: int | None + invariant_names: list[str] + cases: list[ValidationCaseResult] + notes: list[str] = field(default_factory=list) diff --git a/contextpilot/trace_validation/runner.py b/contextpilot/trace_validation/runner.py new file mode 100644 index 0000000..c70b677 --- /dev/null +++ b/contextpilot/trace_validation/runner.py @@ -0,0 +1,766 @@ +"""Trace validation runner: gate accuracy-preservation against fixed cases. + +Loads the local JSONL corpus produced by :mod:`.builder` and, for every case, +runs ContextPilot's optimization in two controlled modes: + +* a **baseline** (``off``) pass, which must leave the payload byte-identical, and +* a **candidate** pass, whose mode is taken from the configured environment + (``CONTEXTPILOT_PROMPT_DEDUP_MODE``) or overridden on the command line. + +It then checks accuracy-preservation invariants between the two payloads -- +message count, order, and roles preserved; protected (non-skill) content +byte-identical; mutation confined to the allowed scope and never growing the +payload; and realized-savings accounting consistent. The realized ``chars_saved`` +is the ACTUAL processed-payload before/after character delta, never an +opportunity count. Exact-token figures appear only when a tokenizer backend is +configured; otherwise the status is ``unavailable`` and no token fields are set. + +The emitted report is privacy-safe: salted ids, counters, enums, and pass/fail +only. It is passed through the analyzer's forbidden-key guard and a raw-content +substring scan before it is printed, so a regression cannot leak raw text. The +process exits non-zero on any gate failure. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import json +from pathlib import Path +from typing import Callable + +from contextpilot.hermes_opportunities.models import DEFAULT_MIN_BLOCK_CHARS, _LLMContent +from contextpilot.hermes_opportunities.privacy import ( + _assert_no_forbidden_keys, + _salt_fingerprint, +) +from contextpilot.hermes_opportunities.prompt_dedup_canary import ( + PROMPT_DEDUP_CANARY_REFERENCE_TEMPLATE, + PromptDedupCanaryResult, + apply_prompt_dedup_canary, + resolve_prompt_dedup_mode, +) +from contextpilot.hermes_opportunities.artifact_dedup_canary import ( + MUTABLE_ARTIFACT_BLOCK_TYPES, + ArtifactDedupCanaryResult, + ArtifactSpanLink, + _parse_artifact_reference, + _segment_fenced_blocks, + _line_aligned, + apply_artifact_dedup_canary, + dangling_artifact_references, + resolve_artifact_dedup_mode, +) +from contextpilot.hermes_opportunities.tokenizer import resolve_tokenizer + +from .builder import DEFAULT_SALT +from .models import ( + MUTABLE_BLOCK_TYPE, + VALIDATION_SET_SCHEMA_VERSION, + ValidationCaseResult, + ValidationReport, +) + +# Stable invariant identifiers, also used as the report's gate vocabulary. +INVARIANT_NAMES = [ + "message_count_preserved", + "order_and_roles_preserved", + "protected_content_preserved", + "mutation_scope_allowed", + "savings_accounting_consistent", +] + +# The fixed prefix/needle of the canary's reference string. A mutated skill line +# must equal a string of this shape -- anything else means the optimizer emitted +# unexpected (possibly raw) text, which is a gate failure. +_REF_PREFIX = PROMPT_DEDUP_CANARY_REFERENCE_TEMPLATE.split("", 1)[0] # "[...ref=" +_REF_NEEDLE = f"ref={MUTABLE_BLOCK_TYPE}:" + + +def load_cases(corpus_path: Path) -> list[dict]: + """Load the JSONL corpus into a list of case dicts (raw content in-memory). + + Tolerates blank lines; raises on malformed JSON so a corrupt corpus fails + loudly rather than silently validating a partial set. + """ + cases: list[dict] = [] + with corpus_path.open("r", encoding="utf-8") as f: + for raw in f: + line = raw.strip() + if not line: + continue + cases.append(json.loads(line)) + return cases + + +def _messages(case: dict) -> list[dict]: + return [ + { + "role": m.get("role"), + "block_type": m.get("block_type", "unknown"), + "content": m.get("content", ""), + } + for m in case.get("messages", []) + ] + + +def optimize_case( + messages: list[dict], *, mode: str, salt: str, min_block_chars: int +) -> tuple[list[dict], PromptDedupCanaryResult]: + """Run the prompt-dedup canary over a case's messages in the given mode. + + Returns ``(out_messages, result)``. The canary mutates only ``skill_prompt`` + content in place; ``out_messages`` mirrors the input role/block_type/order + with the (possibly) rewritten content so the caller can diff payloads. + """ + contents = [_LLMContent(m["block_type"], m["content"]) for m in messages] + result = apply_prompt_dedup_canary( + contents, salt=salt, min_block_chars=min_block_chars, mode=mode + ) + out = [ + {"role": m["role"], "block_type": m["block_type"], "content": c.content} + for m, c in zip(messages, contents) + ] + return out, result + + +def _is_reference_line(line: str) -> bool: + """True if a line is a canary reference placeholder (no raw content).""" + return line.startswith(_REF_PREFIX) and _REF_NEEDLE in line and line.endswith("]") + + +def _mutation_scope_ok(base: dict, cand: dict) -> bool: + """A single message changed only within the allowed (skill-only) scope.""" + if base["content"] == cand["content"]: + return True + # Only skill_prompt content may ever change. + if base["block_type"] != MUTABLE_BLOCK_TYPE: + return False + # Never grow the payload. + if len(cand["content"]) > len(base["content"]): + return False + base_lines = base["content"].split("\n") + cand_lines = cand["content"].split("\n") + # The canary replaces lines 1:1; a differing line count is out of scope. + if len(base_lines) != len(cand_lines): + return False + for b, c in zip(base_lines, cand_lines): + if b == c: + continue + # A changed line must be a reference placeholder strictly shorter than + # what it replaced -- never new free text and never a growth. + if not (_is_reference_line(c) and len(c) < len(b)): + return False + return True + + +def check_invariants( + baseline: list[dict], candidate: list[dict], result: PromptDedupCanaryResult +) -> tuple[dict[str, bool], int]: + """Check accuracy-preservation invariants between two payloads. + + Returns ``(invariant -> passed, realized_chars_saved)`` where + ``realized_chars_saved`` is the ACTUAL summed before/after character delta of + the processed payload (not an opportunity count). + """ + inv: dict[str, bool] = {} + + inv["message_count_preserved"] = len(baseline) == len(candidate) + + if inv["message_count_preserved"]: + inv["order_and_roles_preserved"] = all( + b["role"] == c["role"] and b["block_type"] == c["block_type"] + for b, c in zip(baseline, candidate) + ) + inv["protected_content_preserved"] = all( + b["content"] == c["content"] + for b, c in zip(baseline, candidate) + if b["block_type"] != MUTABLE_BLOCK_TYPE + ) + inv["mutation_scope_allowed"] = all( + _mutation_scope_ok(b, c) for b, c in zip(baseline, candidate) + ) + realized = sum( + len(b["content"]) - len(c["content"]) + for b, c in zip(baseline, candidate) + ) + else: + # Count mismatch makes positional comparison meaningless; fail the rest. + inv["order_and_roles_preserved"] = False + inv["protected_content_preserved"] = False + inv["mutation_scope_allowed"] = False + realized = 0 + + # Realized savings must equal the optimizer's own realized figure, must be + # non-negative, and a non-zero saving must coincide with a reported mutation. + inv["savings_accounting_consistent"] = ( + realized >= 0 + and realized == result.chars_saved + and (realized > 0) == bool(result.mutated) + and (result.blocks_replaced > 0) == bool(result.mutated) + ) + return inv, realized + + +def _raw_content_strings(cases: list[dict], *, min_len: int = 12) -> list[str]: + """Collect non-trivial raw content lines for the privacy substring scan.""" + out: list[str] = [] + for case in cases: + for m in case.get("messages", []): + text = (m.get("content") or "") + for line in text.split("\n"): + line = line.strip() + if len(line) >= min_len: + out.append(line) + return out + + +def assert_report_privacy_safe(report_dict: dict, raw_texts: list[str]) -> None: + """Guard the report before emission: no forbidden keys, no raw content.""" + _assert_no_forbidden_keys(report_dict) + blob = json.dumps(report_dict, ensure_ascii=False) + for text in raw_texts: + if text and text in blob: + raise RuntimeError("refusing to emit report containing raw case content") + + +def run_validation( + cases: list[dict], + *, + baseline_mode: str = "off", + candidate_mode: str, + salt: str, + min_block_chars: int = DEFAULT_MIN_BLOCK_CHARS, + date: str, + tokenizer_spec: object | None = None, + optimize_fn: Callable[..., tuple[list[dict], PromptDedupCanaryResult]] | None = None, +) -> ValidationReport: + """Validate every case under baseline vs candidate and build the gate report.""" + # Resolve at call time (not as a default arg) so the module-level + # ``optimize_case`` stays monkeypatchable from tests and callers. + optimize_fn = optimize_fn or optimize_case + tokenizer = resolve_tokenizer(tokenizer_spec) + tok_status = "available" if tokenizer is not None else "unavailable" + + case_results: list[ValidationCaseResult] = [] + total_blocks = 0 + total_chars = 0 + total_actual_saved = 0 if tokenizer is not None else None + + for case in cases: + msgs = _messages(case) + baseline_msgs, _ = optimize_fn( + list(msgs), mode=baseline_mode, salt=salt, min_block_chars=min_block_chars + ) + candidate_msgs, result = optimize_fn( + list(msgs), mode=candidate_mode, salt=salt, min_block_chars=min_block_chars + ) + + inv, realized = check_invariants(baseline_msgs, candidate_msgs, result) + failed = [name for name, ok in inv.items() if not ok] + + at_before = at_after = at_saved = None + if tokenizer is not None: + at_before = sum(tokenizer.count(m["content"]) for m in baseline_msgs) + at_after = sum(tokenizer.count(m["content"]) for m in candidate_msgs) + at_saved = at_before - at_after + total_actual_saved += at_saved + + skill_items = sum(1 for m in msgs if m["block_type"] == MUTABLE_BLOCK_TYPE) + total_blocks += result.blocks_replaced if result.mutated else 0 + total_chars += realized + + case_results.append( + ValidationCaseResult( + case_id=str(case.get("case_id", "")), + source=case.get("source"), + message_count=len(msgs), + skill_item_count=skill_items, + mutated=bool(result.mutated), + blocks_replaced=result.blocks_replaced if result.mutated else 0, + chars_saved=realized, + invariants=inv, + passed=not failed, + failed_invariants=failed, + actual_tokens_before=at_before, + actual_tokens_after=at_after, + actual_tokens_saved=at_saved, + ) + ) + + passed_cases = sum(1 for c in case_results if c.passed) + failed_cases = len(case_results) - passed_cases + notes = [ + "baseline runs the optimizer in 'off' mode and must leave the payload " + "byte-identical; the candidate mode is the change under test", + "chars_saved is the REALIZED processed-payload before/after char delta, " + "not an opportunity count", + ] + if tokenizer is None: + notes.append( + "actual-token savings unavailable (no exact tokenizer backend configured); " + "no actual-token fields are reported" + ) + + return ValidationReport( + schema_version=VALIDATION_SET_SCHEMA_VERSION, + generated_date=date, + salt_fingerprint=_salt_fingerprint(salt), + baseline_mode=baseline_mode, + candidate_mode=candidate_mode, + case_count=len(case_results), + passed=failed_cases == 0, + passed_cases=passed_cases, + failed_cases=failed_cases, + total_blocks_replaced=total_blocks, + total_chars_saved=total_chars, + tokenizer_status=tok_status, + tokenizer_backend=tokenizer.name if tokenizer is not None else None, + total_actual_tokens_saved=total_actual_saved, + invariant_names=list(INVARIANT_NAMES), + cases=case_results, + notes=notes, + ) + + +def report_to_dict(report: ValidationReport) -> dict: + from dataclasses import asdict + + return asdict(report) + + +def render_markdown(report: ValidationReport) -> str: + gate = "PASS ✅" if report.passed else "FAIL ❌" + lines = [ + f"# ContextPilot trace validation — {report.generated_date}", + "", + f"Gate: **{gate}**", + f"Salt fingerprint: `{report.salt_fingerprint}`", + f"Baseline mode: `{report.baseline_mode}` | Candidate mode: `{report.candidate_mode}`", + "", + "## Summary", + f"- Cases: {report.case_count} ({report.passed_cases} passed, " + f"{report.failed_cases} failed)", + f"- Blocks replaced (realized): {report.total_blocks_replaced}", + f"- Chars saved (realized before/after delta): {report.total_chars_saved}", + ] + if report.tokenizer_status == "available": + lines.append( + f"- Actual tokens saved ({report.tokenizer_backend}): " + f"{report.total_actual_tokens_saved}" + ) + else: + lines.append("- Actual tokens saved: unavailable (no tokenizer backend)") + lines.append("") + lines.append("## Invariants checked") + for name in report.invariant_names: + lines.append(f"- {name}") + if report.failed_cases: + lines.append("") + lines.append("## Failures") + for c in report.cases: + if not c.passed: + lines.append( + f"- `{c.case_id}` (source={c.source}): " + f"{', '.join(c.failed_invariants)}" + ) + return "\n".join(lines) + "\n" + + +# --------------------------------------------------------------------------- +# Artifact-dedup canary validation (provenance-aware tool-artifact reuse) +# --------------------------------------------------------------------------- + +# Stable invariant identifiers for the artifact-dedup gate. Mirrors the prompt +# gate but swaps in artifact-scope and reference-resolvability checks. +ARTIFACT_INVARIANT_NAMES = [ + "message_count_preserved", + "order_and_roles_preserved", + "protected_content_preserved", + "artifact_mutation_scope_allowed", + "artifact_reference_resolvable", + "savings_accounting_consistent", +] + + +def _span_links(case: dict) -> list[ArtifactSpanLink]: + links = [] + for raw in case.get("span_links") or []: + try: + links.append( + ArtifactSpanLink( + source_index=int(raw["source_index"]), + source_start=int(raw["source_start"]), + source_end=int(raw["source_end"]), + target_index=int(raw["target_index"]), + target_start=int(raw["target_start"]), + target_end=int(raw["target_end"]), + ) + ) + except (KeyError, TypeError, ValueError): + continue + return links + + +def optimize_artifact_case( + messages: list[dict], *, mode: str, salt: str, min_block_chars: int, span_links: list[ArtifactSpanLink] | None = None +) -> tuple[list[dict], ArtifactDedupCanaryResult]: + """Run the artifact-dedup canary over a case's messages in the given mode. + + Returns ``(out_messages, result)``. The canary mutates only mutable artifact + bodies in place; ``out_messages`` mirrors the input role/block_type/order + with the (possibly) rewritten content so the caller can diff payloads. + """ + contents = [_LLMContent(m["block_type"], m["content"]) for m in messages] + result = apply_artifact_dedup_canary( + contents, salt=salt, min_block_chars=min_block_chars, mode=mode, span_links=span_links + ) + out = [ + {"role": m["role"], "block_type": m["block_type"], "content": c.content} + for m, c in zip(messages, contents) + ] + return out, result + + +def _artifact_mutation_scope_ok( + idx: int, + base: dict, + cand: dict, + *, + span_links: list[ArtifactSpanLink] | None = None, +) -> bool: + """A single message changed only within the allowed (artifact-only) scope.""" + if base["content"] == cand["content"]: + return True + # Only mutable artifact bodies may ever change. + if base["block_type"] not in MUTABLE_ARTIFACT_BLOCK_TYPES: + return False + if len(cand["content"]) >= len(base["content"]): + return False + + # Whole-body replacement remains valid. + if _parse_artifact_reference(cand["content"]) is not None: + return True + + # Declared source-span replacement: validate against the declared target + # offsets instead of maximal prefix/suffix inference. Prefix/suffix inference + # can accidentally consume a trailing ']' from the replacement reference when + # the original copied span also ends with ']', causing a false gate failure. + for link in span_links or []: + if link.target_index != idx: + continue + if not _line_aligned(base["content"], link.target_start, link.target_end): + continue + prefix_text = base["content"][: link.target_start] + suffix_text = base["content"][link.target_end :] + if not (cand["content"].startswith(prefix_text) and cand["content"].endswith(suffix_text)): + continue + new_mid = cand["content"][len(prefix_text) : len(cand["content"]) - len(suffix_text)] + old_mid = base["content"][link.target_start : link.target_end] + if ( + old_mid + and _parse_artifact_reference(new_mid) is not None + and len(new_mid) < len(old_mid) + ): + return True + + # Declared source-span replacement: a byte-identical line-aligned span may be + # swapped for one standalone strictly shorter reference while surrounding + # prose remains byte-identical. + base_text = base["content"] + cand_text = cand["content"] + prefix = 0 + while prefix < len(base_text) and prefix < len(cand_text) and base_text[prefix] == cand_text[prefix]: + prefix += 1 + suffix = 0 + while ( + suffix < len(base_text) - prefix + and suffix < len(cand_text) - prefix + and base_text[len(base_text) - 1 - suffix] == cand_text[len(cand_text) - 1 - suffix] + ): + suffix += 1 + base_end = len(base_text) - suffix + cand_end = len(cand_text) - suffix + old_mid = base_text[prefix:base_end] + new_mid = cand_text[prefix:cand_end] + if ( + old_mid + and _parse_artifact_reference(new_mid.strip()) is not None + and len(new_mid.strip()) < len(old_mid) + and (prefix == 0 or base_text[prefix - 1] == "\n") + and (base_end == len(base_text) or base_text[base_end] == "\n") + ): + return True + + # Fenced sub-artifact replacement: prose must be byte-identical and only a + # whole fenced segment may be swapped for one strictly shorter reference line. + pos = 0 + changed = False + for kind, text in _segment_fenced_blocks(base["content"]): + if kind != "fence": + if not cand["content"].startswith(text, pos): + return False + pos += len(text) + continue + if cand["content"].startswith(text, pos): + pos += len(text) + continue + newline = cand["content"].find("\n", pos) + end = len(cand["content"]) if newline == -1 else newline + ref = cand["content"][pos:end] + if _parse_artifact_reference(ref) is None or len(ref) >= len(text): + return False + pos = end + changed = True + return changed and pos == len(cand["content"]) + + +def check_artifact_invariants( + baseline: list[dict], + candidate: list[dict], + result: ArtifactDedupCanaryResult, + *, + salt: str, + span_links: list[ArtifactSpanLink] | None = None, +) -> tuple[dict[str, bool], int]: + """Check accuracy-preservation invariants for an artifact-dedup pass. + + Returns ``(invariant -> passed, realized_chars_saved)`` where + ``realized_chars_saved`` is the ACTUAL summed before/after character delta of + the processed payload (not an opportunity count). + """ + inv: dict[str, bool] = {} + + inv["message_count_preserved"] = len(baseline) == len(candidate) + + if inv["message_count_preserved"]: + inv["order_and_roles_preserved"] = all( + b["role"] == c["role"] and b["block_type"] == c["block_type"] + for b, c in zip(baseline, candidate) + ) + inv["protected_content_preserved"] = all( + b["content"] == c["content"] + for b, c in zip(baseline, candidate) + if b["block_type"] not in MUTABLE_ARTIFACT_BLOCK_TYPES + ) + inv["artifact_mutation_scope_allowed"] = all( + _artifact_mutation_scope_ok(i, b, c, span_links=span_links) + for i, (b, c) in enumerate(zip(baseline, candidate)) + ) + cand_contents = [ + _LLMContent(c["block_type"], c["content"]) for c in candidate + ] + inv["artifact_reference_resolvable"] = ( + dangling_artifact_references(cand_contents, salt=salt, span_links=span_links) == [] + ) + realized = sum( + len(b["content"]) - len(c["content"]) + for b, c in zip(baseline, candidate) + ) + else: + # Count mismatch makes positional comparison meaningless; fail the rest. + inv["order_and_roles_preserved"] = False + inv["protected_content_preserved"] = False + inv["artifact_mutation_scope_allowed"] = False + inv["artifact_reference_resolvable"] = False + realized = 0 + + inv["savings_accounting_consistent"] = ( + realized >= 0 + and realized == result.chars_saved + and (realized > 0) == bool(result.mutated) + and (result.blocks_replaced > 0) == bool(result.mutated) + ) + return inv, realized + + +def run_artifact_validation( + cases: list[dict], + *, + baseline_mode: str = "off", + candidate_mode: str, + salt: str, + min_block_chars: int = DEFAULT_MIN_BLOCK_CHARS, + date: str, + tokenizer_spec: object | None = None, + optimize_fn: Callable[..., tuple[list[dict], ArtifactDedupCanaryResult]] | None = None, +) -> ValidationReport: + """Validate every case under baseline vs candidate for the artifact canary.""" + optimize_fn = optimize_fn or optimize_artifact_case + tokenizer = resolve_tokenizer(tokenizer_spec) + tok_status = "available" if tokenizer is not None else "unavailable" + + case_results: list[ValidationCaseResult] = [] + total_blocks = 0 + total_chars = 0 + total_actual_saved = 0 if tokenizer is not None else None + + for case in cases: + msgs = _messages(case) + span_links = _span_links(case) + if span_links: + baseline_msgs, _ = optimize_fn( + list(msgs), mode=baseline_mode, salt=salt, min_block_chars=min_block_chars, span_links=span_links + ) + candidate_msgs, result = optimize_fn( + list(msgs), mode=candidate_mode, salt=salt, min_block_chars=min_block_chars, span_links=span_links + ) + else: + baseline_msgs, _ = optimize_fn( + list(msgs), mode=baseline_mode, salt=salt, min_block_chars=min_block_chars + ) + candidate_msgs, result = optimize_fn( + list(msgs), mode=candidate_mode, salt=salt, min_block_chars=min_block_chars + ) + + inv, realized = check_artifact_invariants( + baseline_msgs, candidate_msgs, result, salt=salt, span_links=span_links + ) + failed = [name for name, ok in inv.items() if not ok] + + at_before = at_after = at_saved = None + if tokenizer is not None: + at_before = sum(tokenizer.count(m["content"]) for m in baseline_msgs) + at_after = sum(tokenizer.count(m["content"]) for m in candidate_msgs) + at_saved = at_before - at_after + total_actual_saved += at_saved + + artifact_items = sum( + 1 for m in msgs if m["block_type"] in MUTABLE_ARTIFACT_BLOCK_TYPES + ) + total_blocks += result.blocks_replaced if result.mutated else 0 + total_chars += realized + + case_results.append( + ValidationCaseResult( + case_id=str(case.get("case_id", "")), + source=case.get("source"), + message_count=len(msgs), + skill_item_count=artifact_items, + mutated=bool(result.mutated), + blocks_replaced=result.blocks_replaced if result.mutated else 0, + chars_saved=realized, + invariants=inv, + passed=not failed, + failed_invariants=failed, + actual_tokens_before=at_before, + actual_tokens_after=at_after, + actual_tokens_saved=at_saved, + ) + ) + + passed_cases = sum(1 for c in case_results if c.passed) + failed_cases = len(case_results) - passed_cases + notes = [ + "baseline runs the artifact canary in 'off' mode and must leave the " + "payload byte-identical; the candidate mode is the change under test", + "chars_saved is the REALIZED processed-payload before/after char delta, " + "not an opportunity count", + ] + if tokenizer is None: + notes.append( + "actual-token savings unavailable (no exact tokenizer backend configured); " + "no actual-token fields are reported" + ) + + return ValidationReport( + schema_version=VALIDATION_SET_SCHEMA_VERSION, + generated_date=date, + salt_fingerprint=_salt_fingerprint(salt), + baseline_mode=baseline_mode, + candidate_mode=candidate_mode, + case_count=len(case_results), + passed=failed_cases == 0, + passed_cases=passed_cases, + failed_cases=failed_cases, + total_blocks_replaced=total_blocks, + total_chars_saved=total_chars, + tokenizer_status=tok_status, + tokenizer_backend=tokenizer.name if tokenizer is not None else None, + total_actual_tokens_saved=total_actual_saved, + invariant_names=list(ARTIFACT_INVARIANT_NAMES), + cases=case_results, + notes=notes, + ) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + description=( + "Run ContextPilot trace validation: check accuracy-preservation " + "invariants of a candidate optimization against a fixed local corpus. " + "Exits non-zero on any gate failure." + ) + ) + parser.add_argument("corpus", type=Path, help="path to the JSONL validation corpus") + parser.add_argument( + "--gate", + choices=["prompt", "artifact"], + default="prompt", + help=( + "which validation gate to run: 'prompt' for skill-prompt dedup " + "or 'artifact' for provenance-aware tool/artifact reuse (default: prompt)" + ), + ) + parser.add_argument( + "--candidate-mode", + default=None, + help=( + "dedup mode to validate (off|shadow|canary). Defaults to the " + "resolved CONTEXTPILOT_*_DEDUP_MODE env for the selected gate." + ), + ) + parser.add_argument( + "--baseline-mode", + default="off", + help="reference mode the candidate is compared against (default: off)", + ) + parser.add_argument("--salt", default=DEFAULT_SALT) + parser.add_argument( + "--min-block-chars", type=int, default=DEFAULT_MIN_BLOCK_CHARS + ) + parser.add_argument( + "--tokenizer", + default=None, + help=( + "opt-in exact tokenizer backend for actual-token accounting, e.g. " + "'tiktoken:cl100k_base' (off by default -> tokens reported unavailable)" + ), + ) + parser.add_argument( + "--format", choices=["json", "markdown"], default="json" + ) + parser.add_argument("--date", default=dt.date.today().isoformat()) + args = parser.parse_args(argv) + + if not args.corpus.exists(): + raise SystemExit(f"validation corpus not found: {args.corpus}") + + if args.candidate_mode is not None: + candidate_mode = args.candidate_mode + elif args.gate == "artifact": + candidate_mode = resolve_artifact_dedup_mode() + else: + candidate_mode = resolve_prompt_dedup_mode() + + cases = load_cases(args.corpus) + run_fn = run_artifact_validation if args.gate == "artifact" else run_validation + report = run_fn( + cases, + baseline_mode=args.baseline_mode, + candidate_mode=candidate_mode, + salt=args.salt, + min_block_chars=args.min_block_chars, + date=args.date, + tokenizer_spec=args.tokenizer, + ) + + report_dict = report_to_dict(report) + # Hard privacy gate: never emit a report carrying forbidden keys or raw text. + assert_report_privacy_safe(report_dict, _raw_content_strings(cases)) + + if args.format == "markdown": + print(render_markdown(report)) + else: + print(json.dumps(report_dict, ensure_ascii=False, indent=2)) + + return 0 if report.passed else 1 diff --git a/datasets/provenance_linking/synthetic_v1.jsonl b/datasets/provenance_linking/synthetic_v1.jsonl new file mode 100644 index 0000000..f740e93 --- /dev/null +++ b/datasets/provenance_linking/synthetic_v1.jsonl @@ -0,0 +1,3 @@ +{"blocks": [{"block_id": "coding_tool_pytest", "block_type": "tool_result", "metadata": {"domain": "coding", "tool": "pytest"}, "text": "pytest output\n46 passed, 2 warnings in 0.33s\nfull suite: 592 passed, 37 skipped in 19.02s\n"}, {"block_id": "coding_worker_review", "block_type": "worker_output", "metadata": {"domain": "coding", "worker": "reviewer"}, "text": "Read-only review: PASS. No correctness blockers. Minor naming nit only.\n"}, {"block_id": "coding_parent_summary", "block_type": "assistant_context", "metadata": {"domain": "coding"}, "text": "Full suite passed: 592 passed, 37 skipped. Read-only review PASS, so PR can continue."}], "claims": [{"block_id": "coding_parent_summary", "claim_id": "claim_coding_tests_passed", "claim_type": "factual", "end": 42, "start": 0, "text": "Full suite passed: 592 passed, 37 skipped."}, {"block_id": "coding_parent_summary", "claim_id": "claim_coding_pr_continue", "claim_type": "factual", "end": 85, "start": 43, "text": "Read-only review PASS, so PR can continue."}], "domain": "coding", "example_id": "synthetic_coding_review", "gold_links": [{"claim_id": "claim_coding_tests_passed", "confidence": 1.0, "end": 79, "evidence_block_id": "coding_tool_pytest", "method": "gold", "relation": "supports", "start": 57}, {"claim_id": "claim_coding_pr_continue", "confidence": 1.0, "end": 47, "evidence_block_id": "coding_worker_review", "method": "gold", "relation": "supports", "start": 18}], "schema_version": 1, "shadow_links": [{"claim_id": "claim_coding_tests_passed", "confidence": 0.5714, "end": 90, "evidence_block_id": "coding_tool_pytest", "method": "shadow_lexical_v1", "relation": "extracted", "start": 0}]} +{"blocks": [{"block_id": "research_web_source", "block_type": "web_result", "metadata": {"domain": "research"}, "text": "Paper abstract: the system uses retrieval plus citation verification to reduce unsupported claims."}, {"block_id": "research_worker_summary", "block_type": "worker_output", "metadata": {"domain": "research"}, "text": "Worker finding: citation verification catches unsupported claims before final answer."}, {"block_id": "research_parent_summary", "block_type": "assistant_context", "metadata": {"domain": "research"}, "text": "The method reduces unsupported claims by using retrieval plus citation verification."}], "claims": [{"block_id": "research_parent_summary", "claim_id": "claim_research_attribution", "claim_type": "factual", "end": 84, "start": 0, "text": "The method reduces unsupported claims by using retrieval plus citation verification."}], "domain": "research", "example_id": "synthetic_research_claim", "gold_links": [{"claim_id": "claim_research_attribution", "confidence": 1.0, "end": 68, "evidence_block_id": "research_web_source", "method": "gold", "relation": "summarized_support", "start": 32}], "schema_version": 1, "shadow_links": [{"claim_id": "claim_research_attribution", "confidence": 0.5, "end": 98, "evidence_block_id": "research_web_source", "method": "shadow_lexical_v1", "relation": "supports_candidate", "start": 0}]} +{"blocks": [{"block_id": "ops_metric_source", "block_type": "tool_result", "metadata": {"domain": "ops"}, "text": "monitor: p95 latency = 183ms, error_rate = 0.02%, queue_depth = 4"}, {"block_id": "ops_summary", "block_type": "assistant_context", "metadata": {"domain": "ops"}, "text": "Latency is healthy: p95 latency = 183ms and error_rate = 0.02%."}], "claims": [{"block_id": "ops_summary", "claim_id": "claim_ops_latency_healthy", "claim_type": "factual", "end": 63, "start": 0, "text": "Latency is healthy: p95 latency = 183ms and error_rate = 0.02%."}], "domain": "ops", "example_id": "synthetic_ops_metrics", "gold_links": [{"claim_id": "claim_ops_latency_healthy", "confidence": 1.0, "end": 28, "evidence_block_id": "ops_metric_source", "method": "gold", "relation": "extracted_support", "start": 9}], "schema_version": 1, "shadow_links": [{"claim_id": "claim_ops_latency_healthy", "confidence": 0.7143, "end": 65, "evidence_block_id": "ops_metric_source", "method": "shadow_lexical_v1", "relation": "extracted", "start": 0}]} diff --git a/datasets/provenance_linking/synthetic_v1.jsonl.manifest.json b/datasets/provenance_linking/synthetic_v1.jsonl.manifest.json new file mode 100644 index 0000000..2f61f47 --- /dev/null +++ b/datasets/provenance_linking/synthetic_v1.jsonl.manifest.json @@ -0,0 +1,12 @@ +{ + "schema_version": 1, + "dataset_kind": "provenance_linking", + "mode": "synthetic", + "output": "datasets/provenance_linking/synthetic_v1.jsonl", + "example_count": 3, + "block_count": 8, + "claim_count": 4, + "gold_link_count": 4, + "shadow_link_count": 3, + "privacy_note": "synthetic is commit-safe; trace mode may contain raw local content and must stay local/gitignored" +} diff --git a/docs/guides/hermes-monitor.md b/docs/guides/hermes-monitor.md index 5500b2c..9afffee 100644 --- a/docs/guides/hermes-monitor.md +++ b/docs/guides/hermes-monitor.md @@ -55,10 +55,14 @@ Then read the generated Markdown report for today and send a short Chinese summa ## Quick savings summary (lightweight) -If you just want to answer "how many tokens did ContextPilot save?", use the -lightweight `scripts/contextpilot_savings.py` command instead of this monitor or -the analyzer below. It reads **only** the metadata-only telemetry file, imports -no Hermes internals, and prints a one-screen summary: +If you just want a lightweight realized-savings summary, use the +`scripts/contextpilot_savings.py` command instead of this monitor or the analyzer +below. It reads **only** the metadata-only telemetry file, imports no Hermes +internals, and prints a one-screen summary. Character savings are measured from +ContextPilot's actual before/after processed payload; exact tokenizer tokens are +shown only when telemetry recorded an explicitly configured exact tokenizer +backend. The legacy chars/4 counter is labelled as derived; tokenizer measurement +is off by default to avoid provider/tokenizer mismatches. ```bash python scripts/contextpilot_savings.py # last 24h @@ -68,10 +72,10 @@ python scripts/contextpilot_savings.py --format json python ~/.hermes/plugins/ContextPilot/scripts/contextpilot_savings.py ``` -It reports events, chars saved, estimated tokens saved, the window, and average -tokens per event. This is the right tool for ordinary users; the monitor in this -guide (which also reads `state.db` metadata) and the content-aware analyzer below -are for deeper investigation. +It reports events, processed-payload chars saved, exact tokenizer tokens when +available, and the legacy derived chars/4 counter. This is the right tool for +ordinary users; the monitor in this guide (which also reads `state.db` metadata) +and the content-aware analyzer below are for deeper investigation. ### Ask Hermes for savings @@ -100,7 +104,9 @@ It surfaces concrete token-reduction opportunities: - repeated line/block fingerprints (shared boilerplate across outputs), - large tool outputs grouped by `tool_name`, - heavy sessions by input-token / tool-call / message counts (hashed ids), -- ContextPilot telemetry coverage and savings ratios, +- **Prompt duplicate shadow telemetry** for exact system/skill prompt template + repeats (advisory only; no prompt rewriting), +- ContextPilot telemetry coverage and processed-payload savings counters, - **Worker Context Routing shadow labels** for future router training/eval, - **Parent Aggregation Artifact telemetry** (exact duplicate worker/parent artifacts grouped by hash) for future parent-aggregation dedup eval. @@ -129,6 +135,104 @@ aggregated. The report then shows: prompt *and* a tool result *and* a user prompt). Reported only as a hash plus per-type counters — never the raw text. +### Prompt duplicate shadow mode + +The analyzer includes a dedicated **Prompt duplicate blocks — system/skill** +section for the static-template opportunity found in Hermes workloads. It scans +only `system_prompt` and `skill_prompt` blocks, groups **EXACT** duplicate block +fingerprints, and reports: + +- duplicate group count and duplicate occurrence count, +- actual duplicated characters observed in prompt assembly, +- a derived chars/4 advisory token counter labelled as advisory, +- per-type counters and top salted hashes. + +This section is **advisory only**. It never rewrites, summarizes, deduplicates, +or replaces prompt text, and its counters are not realized savings. Use it to +prioritize a future prompt-assembly A/B where before/after payloads are measured +with an exact tokenizer/API usage comparison. + +### Prompt dedup A/B simulation + +The analyzer also includes a **Prompt dedup A/B simulation — system/skill** +section. This is the evidence gate before any canary replacement. It still does +not mutate runtime payloads: it keeps prompt text in memory, groups exact +duplicate `system_prompt` / `skill_prompt` blocks, and simulates the accounting +for keeping the first occurrence while replacing only later occurrences with a +deterministic reference placeholder. + +The simulation reports candidate classes separately: + +- `same_type_skill_prompt_only` — lowest-risk first canary candidate, +- `same_type_system_prompt_only` — higher risk, +- `cross_type_system_skill` — higher risk because it crosses prompt hierarchy. + +For each class the report includes group counts, replacement occurrence counts, +`chars_before`, `chars_after_simulated`, and signed `chars_delta_simulated`. +When you pass an explicitly configured tokenizer backend, for example +`--prompt-dedup-tokenizer tiktoken:cl100k_base`, it also reports actual tokenizer +before/after/delta fields for the simulation. Without that opt-in backend, +`tokenizer_status=unavailable` and no fake actual-token numbers are emitted. + +Use `--disable-prompt-dedup-ab` to omit this section. Even when enabled, all +figures are **simulation-only**, **not realized savings**, and no prompt text is +rewritten, summarized, deduplicated, or emitted. + +### Prompt dedup canary (runtime; default OFF) + +> **Use only after the A/B simulation above shows a clear, positive +> `same_type_skill_prompt_only` delta and you have a golden eval in place.** +> This is the one ContextPilot path that *actually rewrites prompt text*; treat +> it as gray/canary, not default behavior. + +Everything else in the analyzer is measurement/shadow/simulation only. The +canary (`contextpilot.hermes_opportunities.prompt_dedup_canary`) is the single +runtime replacement path and it is **off by default**. It is controlled entirely +by environment variables — no config file is required: + +```sh +# off (default): no scan, no mutation, no prompt-dedup savings recorded +CONTEXTPILOT_PROMPT_DEDUP_MODE=off + +# shadow: measure what a canary *would* replace; payload still unchanged +CONTEXTPILOT_PROMPT_DEDUP_MODE=shadow + +# canary: actually replace later exact duplicate skill-prompt blocks +CONTEXTPILOT_PROMPT_DEDUP_MODE=canary +``` + +**Rollback / kill switch.** Set the mode back to `off` (or unset the variable) +to disable immediately. The escape-hatch variable forces `off` regardless of the +mode variable, for an instant kill without editing the mode: + +```sh +CONTEXTPILOT_PROMPT_DEDUP_DISABLE=1 # forces off even if MODE=canary +``` + +What the canary will and will not do, even when `MODE=canary`: + +- It acts **only** on the `same_type_skill_prompt_only` class — an EXACT + duplicate block whose every occurrence is inside `skill_prompt` content. +- It **never** replaces `system_prompt`-only duplicates, **never** replaces + cross-type `system_prompt`/`skill_prompt` duplicates, and **never** touches + user, assistant, tool, or ordinary system-prompt content. +- The **first** occurrence is always kept verbatim; only later exact duplicates + are replaced, and only with a deterministic reference string containing a + low-cardinality prompt-type enum plus a salted hash — never raw prompt text. +- A replacement happens only when the reference string is **strictly shorter** + than the line it replaces, so the payload is never grown. +- A broad **safety denylist** (instruction / safety / security / tool / auth / + secret / must / never / always / required / ...) leaves any matching block + unchanged even in canary mode. Skill-prompt detection is conservative: if a + block is not clearly a skill-prompt duplicate, it is left as-is. + +Telemetry is metadata-only: `prompt_dedup_mode`, `prompt_dedup_class`, +`prompt_dedup_blocks_replaced`, and `prompt_dedup_chars_saved` (mode/class enums +and integer counters only — no prompt text). The realized `prompt_dedup_chars_saved` +and its contribution to the aggregate `chars_saved` total are non-zero **only +when a real canary mutation occurred**; `off` and `shadow` record no prompt-dedup +savings. + ### Worker Context Routing shadow mode The analyzer now includes a **Worker Context Routing — shadow mode** section by @@ -217,9 +321,31 @@ gate below before changing ContextPilot config or code. A defensive guard in `write_report` refuses to emit any forbidden raw-content key, so the reports are safe to ship from an unattended cron job. -## Accuracy gate +## Default-off canaries + +Prompt and artifact reuse are opt-in canaries. Ordinary installs keep both off +unless an operator explicitly enables them and restarts Hermes. + +```bash +# Skill-prompt exact duplicate canary (lowest-risk prompt class) +export CONTEXTPILOT_PROMPT_DEDUP_MODE=canary + +# Provenance-aware tool/artifact exact duplicate canary +export CONTEXTPILOT_ARTIFACT_DEDUP_MODE=canary + +# Emergency kill switches +export CONTEXTPILOT_PROMPT_DEDUP_DISABLE=1 +export CONTEXTPILOT_ARTIFACT_DEDUP_DISABLE=1 +``` + +The artifact canary only keeps the first full `tool_result`/`assistant_context` +artifact body and replaces later exact duplicates with a shorter ContextPilot +reference. It does not summarize, semantically compress, or drop user/system +content. + +## Safety gates -This monitor only measures token/cost savings and operational signals. Before shipping ContextPilot changes, run a fixed golden eval set and require: +This monitor reports processed-payload savings, exact tokenizer token deltas when recorded, and operational signals. Before shipping ContextPilot changes, run a fixed golden eval set and require: - no task-success regression, - no drop in context recall beyond the chosen threshold, diff --git a/docs/guides/hermes.md b/docs/guides/hermes.md index 03a7a56..e55d8ad 100644 --- a/docs/guides/hermes.md +++ b/docs/guides/hermes.md @@ -64,10 +64,18 @@ print(engine.get_status()) # {'engine': 'contextpilot', 'contextpilot_chars_saved': 18420, ...} ``` -## See token savings - -Once ContextPilot has run for a bit, you can see how many tokens it saved with a -single command from the ContextPilot repo or plugin directory: +## See processed-payload savings + +Once ContextPilot has run for a bit, you can see realized savings from the +metadata-only telemetry with a single command from the ContextPilot repo or +plugin directory. Character savings are always measured from the actual +before/after LLM-bound payload after ContextPilot processing. Exact tokenizer +savings are shown only when telemetry recorded an exact tokenizer backend; the +legacy chars/4 counter is labelled as derived. To record tokenizer-based deltas, +configure an exact matching tokenizer explicitly, for example +`CONTEXTPILOT_EXACT_TOKENIZER=tiktoken` with +`CONTEXTPILOT_TIKTOKEN_ENCODING=`; it is off by default to avoid +provider/tokenizer mismatches. ```bash python scripts/contextpilot_savings.py @@ -76,12 +84,12 @@ python ~/.hermes/plugins/ContextPilot/scripts/contextpilot_savings.py ``` ``` -ContextPilot token savings (last 24h) - Events: 117 - Chars saved: 6,147,074 - Estimated tokens saved: ~1,536,728 - Avg tokens/event: ~13,134 - Telemetry file: /root/.hermes/contextpilot/telemetry.jsonl +ContextPilot savings (last 24h) + Events: 117 + Chars saved: 6,147,074 + Est. tokens saved (chars/4, derived): 1,536,728 + Actual tokens saved (tokenizer): unavailable (no exact tokenizer backend recorded) + Telemetry file: /root/.hermes/contextpilot/telemetry.jsonl ``` Useful options: @@ -142,6 +150,6 @@ ContextPilot runs *before* the threshold-based compressor, reducing how often th **Plugin not discovered after install.** Check `~/.hermes/plugins/ContextPilot/plugin.yaml` exists and contains `type: context_engine`. Run `hermes plugins list` to confirm. -**No token savings logged.** Dedup only fires when the LLM reads the same file content more than once in a session. On first reads, content is indexed but not deduplicated. +**No savings logged.** Dedup only fires when the LLM reads the same file content more than once in a session. On first reads, content is indexed but not deduplicated. **`ModuleNotFoundError: No module named 'numpy'`.** Reorder requires numpy. If unavailable, ContextPilot silently falls back to dedup-only mode. diff --git a/docs/guides/trace-validation.md b/docs/guides/trace-validation.md new file mode 100644 index 0000000..4699143 --- /dev/null +++ b/docs/guides/trace-validation.md @@ -0,0 +1,129 @@ +# Trace-derived validation sets for ContextPilot + +ContextPilot changes can reduce token usage only if they preserve task accuracy. +For any future change that mutates the LLM-bound payload, do **not** rely on a +single live run. First build or reuse a fixed validation set derived from local +Hermes traces, then run the validation gate against the candidate mode. + +## Privacy model + +- The builder reads the local Hermes SQLite state DB in read-only mode. +- The generated JSONL corpus contains raw replay content and is **local-only**. + Do not commit it, upload it, or paste it into reviews. +- The default output directory is outside the repo: + `~/contextpilot/validation_sets/`. +- The in-repo convenience directory `.contextpilot_validation/` is gitignored. +- Reports and manifests are metadata-only: salted case ids, counters, enums and + pass/fail flags. They must not contain raw conversation, tool output, system + prompt, reasoning, API keys, or session ids. + +## Build a validation set + +Conservative last-24h sample: + +```bash +python scripts/build_trace_validation_set.py +``` + +Heavier all-history sample for accuracy-sensitive changes: + +```bash +python scripts/build_trace_validation_set.py \ + --all-sessions \ + --min-input-tokens 20000 \ + --limit 50 \ + --out ~/contextpilot/validation_sets +``` + +Exclude system/skill prompts if the change does not touch prompt handling: + +```bash +python scripts/build_trace_validation_set.py --no-system-prompt +``` + +The command prints a privacy-safe JSON object with the corpus and manifest paths. +Only the corpus file contains raw replay content. + +## Run the validation gate + +Validate the current prompt-dedup canary candidate: + +```bash +python scripts/run_trace_validation.py \ + ~/contextpilot/validation_sets/validation_set_YYYY-MM-DD.jsonl \ + --gate prompt \ + --candidate-mode canary \ + --format markdown +``` + +Validate the provenance-aware artifact/tool-context reuse canary candidate: + +```bash +python scripts/run_trace_validation.py \ + ~/contextpilot/validation_sets/validation_set_YYYY-MM-DD.jsonl \ + --gate artifact \ + --candidate-mode canary \ + --format markdown +``` + +Use the environment-configured mode instead: + +```bash +CONTEXTPILOT_PROMPT_DEDUP_MODE=canary \ +python scripts/run_trace_validation.py \ + ~/contextpilot/validation_sets/validation_set_YYYY-MM-DD.jsonl +``` + +Optional exact-token accounting, only when an exact tokenizer backend is +available: + +```bash +python scripts/run_trace_validation.py \ + ~/contextpilot/validation_sets/validation_set_YYYY-MM-DD.jsonl \ + --candidate-mode canary \ + --tokenizer tiktoken:cl100k_base +``` + +If no tokenizer is configured, the report says actual-token savings are +`unavailable`. It does not substitute chars/4 as actual tokens. + +## Gate semantics + +The runner compares a baseline `off` pass with the candidate pass and exits +non-zero on any failed invariant: + +- message count preserved; +- message order and roles preserved; +- protected user/assistant/tool/system content preserved; +- mutation confined to the explicitly allowed scope; +- realized savings accounting matches the actual processed-payload before/after + character delta. + +For the current prompt canary, the only allowed mutation scope is +`same_type_skill_prompt_only`: later exact duplicate `skill_prompt` lines may be +replaced with a deterministic ContextPilot reference if and only if the reference +is shorter and the line is not safety-denylisted. + +For the artifact/tool-context reuse canary, the only allowed mutation scope is +`same_payload_exact_artifact_body`: later exact duplicate `tool_result` or +`assistant_context` artifact bodies may be replaced with a deterministic +ContextPilot artifact reference if and only if the first full canonical body +appears earlier in the same payload, the reference is shorter, and all +non-artifact content remains byte-identical. The artifact gate adds an explicit +reference-resolution invariant so dangling references fail the run. + +## When this is required + +Run this gate before merging or enabling any change that can affect accuracy, +including: + +- prompt/system/skill dedup or replacement; +- context routing, filtering, summarization or dropping; +- parent/child artifact aggregation rewrites; +- changes to runtime optimization order or telemetry accounting that affect the + LLM-bound payload. + +Passing this gate is necessary but not always sufficient. High-risk changes such +as system prompt replacement or context dropping still need shadow telemetry, +offline A/B evidence, golden evals and default-off canary rollout before default +enablement. diff --git a/docs/provenance_training_shadow_plan.md b/docs/provenance_training_shadow_plan.md new file mode 100644 index 0000000..8dacee1 --- /dev/null +++ b/docs/provenance_training_shadow_plan.md @@ -0,0 +1,58 @@ +# General provenance linker training/shadow plan + +## Goal + +Move beyond artifact exact dedup by learning a **claim → evidence provenance graph**. Coding is only one domain example; the schema is domain-general. + +## Data boundary + +- **Committed dataset:** `datasets/provenance_linking/synthetic_v1.jsonl` — synthetic, safe to version. +- **Local real-trace dataset:** `~/contextpilot/provenance_datasets/*.jsonl` — raw Hermes trace content, never commit. +- **Schema:** examples contain `blocks`, extracted `claims`, optional `gold_links`, and optional `shadow_links`. + +## Shadow first + +Online payload is never changed by this path. The shadow linker only emits candidate links: + +```text +claim_id -> evidence_block_id[start:end], relation, confidence, method +``` + +Initial baseline is deliberately cheap/high-precision lexical matching. It is expected to miss semantic support; that gap is the training target. + +## Training target + +Train or distill a provenance linker/verifier to replace `shadow_lexical_v1` scoring: + +```text +input: claim + top-k candidate evidence snippets + metadata +output: support relation + evidence span + confidence +``` + +Relations should include `copied`, `extracted_support`, `summarized_support`, `aggregated_support`, `contradicts`, and `insufficient`. + +## Action policy + +The trained model still does not delete context. Context actions require a deterministic gate: + +1. source exists and is earlier than claim; +2. pointer is recoverable; +3. protected user/system/developer/skill content is untouched; +4. source is not stale; +5. validator passes; +6. online path only consumes cached/shadow graph. + +## Metrics + +- shadow link precision/recall against gold examples; +- unsupported/contradicted claim rate; +- fold opportunity in chars/tokens; +- background latency only; zero user-facing e2e blocking. + +## Current baseline result + +`evals/provenance_shadow_synthetic_v1.json`: + +- 3 examples, 4 gold links; +- precision 1.00, recall 0.75; +- high precision but misses aggregated coding claim evidence, showing why a learned model is needed. diff --git a/evals/artifact_precision_synthetic_2026-06-19.json b/evals/artifact_precision_synthetic_2026-06-19.json new file mode 100644 index 0000000..73ec064 --- /dev/null +++ b/evals/artifact_precision_synthetic_2026-06-19.json @@ -0,0 +1,172 @@ +{ + "schema_version": 1, + "generated_at": "2026-06-19T01:12:09+0200", + "corpus": "synthetic_labeled_artifact_precision_v1", + "claim_scope": "synthetic exact/provenance gate self-consistency; not field/model/product precision", + "case_count": 16, + "synthetic_event_tp": 7, + "synthetic_event_fp": 0, + "synthetic_event_fn": 0, + "synthetic_negative_case_tn": 10, + "synthetic_negative_case_fpr": 0.0, + "synthetic_event_precision": 1.0, + "synthetic_event_recall": 1.0, + "synthetic_case_accuracy": 1.0, + "predicted_replacements": 7, + "expected_replacements": 7, + "synthetic_realized_chars_saved": 2158, + "mode_gate_checks": { + "off_no_mutation": true, + "shadow_no_mutation": true, + "disable_env_no_mutation": true + }, + "validation_gate_checks": { + "forged_reference_detected": true + }, + "rows": [ + { + "name": "whole_tool_exact_duplicate", + "expected_replacements": 1, + "actual_replacements": 1, + "pass": true, + "chars_saved": 293, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "whole_cross_type_exact_duplicate", + "expected_replacements": 1, + "actual_replacements": 1, + "pass": true, + "chars_saved": 293, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "whole_near_duplicate_not_mutated", + "expected_replacements": 0, + "actual_replacements": 0, + "pass": true, + "chars_saved": 0, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "fenced_internal_duplicate", + "expected_replacements": 1, + "actual_replacements": 1, + "pass": true, + "chars_saved": 358, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "two_fenced_duplicates", + "expected_replacements": 2, + "actual_replacements": 2, + "pass": true, + "chars_saved": 716, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "protected_user_system_duplicates", + "expected_replacements": 0, + "actual_replacements": 0, + "pass": true, + "chars_saved": 0, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "short_duplicate_never_grow", + "expected_replacements": 0, + "actual_replacements": 0, + "pass": true, + "chars_saved": 0, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "unterminated_fence_not_mutated", + "expected_replacements": 0, + "actual_replacements": 0, + "pass": true, + "chars_saved": 0, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "copied_plain_span_without_declared_link", + "expected_replacements": 0, + "actual_replacements": 0, + "pass": true, + "chars_saved": 0, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "protected_duplicate_tool_vs_user", + "expected_replacements": 0, + "actual_replacements": 0, + "pass": true, + "chars_saved": 0, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "protected_duplicate_tool_vs_system", + "expected_replacements": 0, + "actual_replacements": 0, + "pass": true, + "chars_saved": 0, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "declared_source_span_exact", + "expected_replacements": 1, + "actual_replacements": 1, + "pass": true, + "chars_saved": 249, + "span_replacements": 1, + "dangling": [] + }, + { + "name": "declared_span_content_differs", + "expected_replacements": 0, + "actual_replacements": 0, + "pass": true, + "chars_saved": 0, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "forward_span_link_rejected", + "expected_replacements": 0, + "actual_replacements": 0, + "pass": true, + "chars_saved": 0, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "oob_span_link_rejected", + "expected_replacements": 0, + "actual_replacements": 0, + "pass": true, + "chars_saved": 0, + "span_replacements": 0, + "dangling": [] + }, + { + "name": "duplicate_span_declaration_counts_once", + "expected_replacements": 1, + "actual_replacements": 1, + "pass": true, + "chars_saved": 249, + "span_replacements": 1, + "dangling": [] + } + ] +} diff --git a/evals/provenance_shadow_synthetic_v1.json b/evals/provenance_shadow_synthetic_v1.json new file mode 100644 index 0000000..cf9f0f0 --- /dev/null +++ b/evals/provenance_shadow_synthetic_v1.json @@ -0,0 +1,42 @@ +{ + "schema_version": 1, + "corpus": "provenance_linking_shadow_eval", + "claim_scope": "shadow claim→evidence linking; does not mutate online context", + "example_count": 3, + "gold_links": 4, + "predicted_links": 3, + "shadow_link_tp": 3, + "shadow_link_fp": 0, + "shadow_link_fn": 1, + "shadow_link_precision": 1.0, + "shadow_link_recall": 0.75, + "cases": [ + { + "example_id": "synthetic_coding_review", + "claim_count": 2, + "gold_links": 2, + "predicted_links": 1, + "tp": 1, + "fp": 0, + "fn": 1 + }, + { + "example_id": "synthetic_research_claim", + "claim_count": 1, + "gold_links": 1, + "predicted_links": 1, + "tp": 1, + "fp": 0, + "fn": 0 + }, + { + "example_id": "synthetic_ops_metrics", + "claim_count": 1, + "gold_links": 1, + "predicted_links": 1, + "tp": 1, + "fp": 0, + "fn": 0 + } + ] +} diff --git a/scripts/analyze_hermes_context_opportunities.py b/scripts/analyze_hermes_context_opportunities.py index b6cca65..ea9204b 100644 --- a/scripts/analyze_hermes_context_opportunities.py +++ b/scripts/analyze_hermes_context_opportunities.py @@ -1,1911 +1,46 @@ #!/usr/bin/env python3 """Privacy-safe Hermes context opportunity analyzer for ContextPilot. +This is a thin wrapper around the :mod:`contextpilot.hermes_opportunities` +package, kept at its historical script path so existing cron jobs and callers +keep working. All logic lives in the package; see its modules for details. + Unlike ``hermes_contextpilot_monitor.py`` (which never reads message bodies), this analyzer *does* inspect message content and tool outputs in order to find -concrete token-reduction opportunities: exact duplicate tool outputs, repeated -line/block fingerprints, oversized tool outputs per tool, heavy sessions, and -ContextPilot telemetry coverage. - -It reads content only in-memory to compute salted hashes and aggregate -counters. Reports never contain raw message/tool text, system prompts, or raw -session ids -- only salted SHA-256 fingerprints and numeric aggregates. This -makes it safe to run continuously from a cron job and ship the reports. +concrete token-reduction opportunities, but it reads content only in-memory to +compute salted hashes and aggregate counters. Reports never contain raw +message/tool text, system prompts, or raw session ids -- only salted SHA-256 +fingerprints and numeric aggregates. This makes it safe to run continuously +from a cron job and ship the reports. """ from __future__ import annotations -import argparse -import datetime as dt -import hashlib -import json -import sqlite3 -from dataclasses import asdict, dataclass, field +import sys from pathlib import Path -from typing import Iterable - -# Columns we are explicitly forbidden from EMITTING in any report. We may read -# message content in-memory for hashing, but it must never reach an output file. -FORBIDDEN_OUTPUT_KEYS = { - "content", - "system_prompt", - "reasoning", - "reasoning_content", - "reasoning_details", - "tool_calls", - "codex_reasoning_items", - "codex_message_items", -} - -# Tunables (overridable via CLI). -DEFAULT_MIN_BLOCK_CHARS = 40 # ignore trivial lines when fingerprinting -DEFAULT_MIN_BLOCK_REPEAT = 3 # a block must recur this often to be a "repeat" -DEFAULT_LARGE_OUTPUT_CHARS = 8000 # tool outputs at/above this are "large" -DEFAULT_TOP_N = 20 -EST_CHARS_PER_TOKEN = 4 - - -def _est_tokens(chars: int) -> int: - return chars // EST_CHARS_PER_TOKEN - - -def _salted_hash(text: str, salt: str, *, length: int = 16) -> str: - return hashlib.sha256(f"{salt}:{text}".encode("utf-8", "replace")).hexdigest()[:length] - - -def _salt_fingerprint(salt: str) -> str: - # Confirms a salt was applied without revealing it. - return hashlib.sha256(f"fingerprint:{salt}".encode()).hexdigest()[:12] - - -def _connect_readonly(path: Path) -> sqlite3.Connection: - uri = f"file:{path}?mode=ro" - return sqlite3.connect(uri, uri=True) - -# --------------------------------------------------------------------------- -# Data structures (all privacy-safe: hashes + counters only) -# --------------------------------------------------------------------------- - - -@dataclass -class DuplicateToolOutput: - content_hash: str - tool_name: str | None - occurrences: int - char_length: int - est_tokens: int - est_wasted_tokens: int # tokens spent re-sending identical output: (n-1) * est_tokens - - -@dataclass -class RepeatedBlock: - block_hash: str - occurrences: int - char_length: int - est_tokens: int - est_wasted_tokens: int # (n-1) * est_tokens - - -# Recognized LLM-bound block types. These are low-cardinality enums, safe to -# emit verbatim (they describe the *origin* of a block, never its text). -BLOCK_TYPES = ( - "system_prompt", - "skill_prompt", - "user_prompt", - "assistant_context", - "tool_result", - "unknown", +# Allow running as a standalone script (``python scripts/analyze_...py``) by +# making the repo root importable, so ``contextpilot`` resolves without an +# editable install. +_REPO_ROOT = Path(__file__).resolve().parents[1] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +# Re-export the full public API at the historical module path. Tests and other +# callers that load this script by file path keep accessing every name (incl. +# the in-memory carriers and shadow-mode enums) via this module. +from contextpilot.hermes_opportunities import * # noqa: F401,F403,E402 +from contextpilot.hermes_opportunities import ( # noqa: F401,E402 + EST_CHARS_PER_TOKEN, + FORBIDDEN_OUTPUT_KEYS, + _assert_no_forbidden_keys, + _est_tokens, + _LLMContent, + _ROUTABLE_LABELS, + _salt_fingerprint, + _salted_hash, + _ToolMessage, + main, ) - -@dataclass -class TypeCount: - block_type: str - count: int - - -@dataclass -class BlockTypeStat: - """Aggregate redundancy within a single LLM-bound block type.""" - - block_type: str - item_count: int # source items (prompts/messages) of this type - block_count: int # total fingerprintable block instances - unique_block_count: int # distinct fingerprints - repeated_block_count: int # fingerprints recurring >= min_repeat within type - est_redundant_tokens: int # sum over repeats of (occ-1) * est_tokens - - -@dataclass -class CrossTypeBlockGroup: - """A single block fingerprint observed in 2+ distinct block types. - - This is the headline signal: the same chunk of text is being shipped to the - LLM from, e.g., a skill/system prompt *and* a tool result, so it is paying - for the same tokens twice from different sources. - """ - - block_hash: str - block_types: list[str] # sorted distinct types this block spans - type_occurrences: list[TypeCount] # per-type occurrence counts - occurrences: int # total occurrences across all types - char_length: int - est_tokens: int - est_wasted_tokens: int # (occurrences - 1) * est_tokens - - -# --------------------------------------------------------------------------- -# Worker Context Routing — SHADOW MODE (P0 data collection only) -# --------------------------------------------------------------------------- -# Low-cardinality router labels. These are the *training/eval* labels a future -# small worker-context router would predict. P0 is data-collection only: nothing -# here ever drops, summarizes, or mutates context — it only classifies blocks -# and emits aggregate counters + salted hashes so the labels can be evaluated -# offline before any online pruning is built. -ROUTER_LABELS = ( - "policy_must_keep", # never droppable (user/system/skill/safety constraints) - "direct_task_hint", # short actionable task signal — keep - "likely_relevant", # default keep; not obviously prunable - "summarizable_candidate", # large single block that *might* be summarized later - "likely_drop_candidate", # large/repeated tool-like block, candidate to route away -) - -# Labels whose blocks a future router might safely route away. Used only to -# tally *advisory* candidate tokens; P0 never acts on them. -_ROUTABLE_LABELS = ("summarizable_candidate", "likely_drop_candidate") - -# Block-type priority when one fingerprint spans multiple origins: the most -# "must-keep" origin wins, so cross-origin blocks are classified conservatively. -_TYPE_KEEP_PRIORITY = { - "user_prompt": 5, - "system_prompt": 4, - "skill_prompt": 4, - "assistant_context": 2, - "tool_result": 1, - "unknown": 0, -} - -# Cues marking content that must NEVER be dropped even from a tool/assistant -# block: explicit safety / acceptance / hard-constraint language. Matching here -# is intentionally generous — over-keeping is the safe direction for P0. -_SAFETY_CONSTRAINT_CUES = ( - "must not", - "must never", - "never drop", - "do not delete", - "do not remove", - "do not modify", - "acceptance criteria", - "acceptance test", - "safety", - "must keep", - "you must", - "required:", - "constraint", - "forbidden", - "policy", -) - -# Cues marking a short, actionable task hint worth keeping verbatim. -_TASK_HINT_CUES = ( - "todo", - "next step", - "error:", - "traceback", - "failed", - "fixme", - "task:", - "goal:", - "implement", - "reproduce", -) - - -@dataclass -class RouterLabelCount: - """Aggregate over all blocks assigned one router label.""" - - route_label: str - block_count: int # distinct fingerprints with this label - occurrence_count: int # total occurrences across the window - total_est_tokens: int # est tokens these blocks occupy (occ * est) - est_candidate_tokens: int # ADVISORY routable tokens (0 unless routable) - - -@dataclass -class RouterReasonCount: - """Aggregate keyed by (block_type, route_label, reason_code).""" - - block_type: str - route_label: str - reason_code: str - block_count: int - occurrence_count: int - total_est_tokens: int - est_candidate_tokens: int - - -@dataclass -class RouterCandidateBlock: - """A single routable-candidate fingerprint (salted hash + counters only).""" - - block_hash: str - block_type: str - route_label: str - reason_code: str - occurrences: int - char_length: int - est_tokens: int - est_candidate_tokens: int # ADVISORY upper bound only - - -@dataclass -class WorkerRoutingShadow: - """Shadow-mode worker-context routing report (P0: data collection only).""" - - enabled: bool - item_count: int # LLM-bound items classified - classified_block_count: int # distinct fingerprints classified - total_occurrences: int - must_keep_block_count: int - must_keep_occurrence_count: int - est_must_keep_tokens: int - est_candidate_tokens_total: int # ADVISORY routable ceiling - est_drop_candidate_tokens: int # ADVISORY - est_summarizable_candidate_tokens: int # ADVISORY - label_counts: list[RouterLabelCount] - reason_counts: list[RouterReasonCount] - top_candidate_blocks: list[RouterCandidateBlock] - notes: list[str] = field(default_factory=list) - - -@dataclass -class ToolSizeStat: - tool_name: str - output_count: int - total_chars: int - max_chars: int - avg_chars: int - total_est_tokens: int - large_output_count: int # outputs >= large_output_chars threshold - - -@dataclass -class HeavySession: - session_hash: str - source: str | None - input_tokens: int - output_tokens: int - message_count: int - tool_call_count: int - api_call_count: int - - -@dataclass -class TelemetryCoverage: - events: int - chars_saved: int - tokens_saved: int - avg_tokens_saved_per_event: float - coverage_ratio_pct: float # tokens_saved / (tokens_saved + total_input_tokens) - malformed_records_skipped: int - - -@dataclass -class OpportunityReport: - date: str - since_hours: int - all_sessions: bool - salt_fingerprint: str - tool_message_count: int - total_tool_output_chars: int - total_tool_output_est_tokens: int - exact_duplicate_groups: list[DuplicateToolOutput] - duplicate_tool_output_groups: int - duplicate_tool_output_wasted_tokens: int - repeated_block_count: int - repeated_block_wasted_tokens: int - repeated_blocks: list[RepeatedBlock] - large_tool_outputs_by_tool: list[ToolSizeStat] - heavy_sessions: list[HeavySession] - telemetry: TelemetryCoverage - # LLM-bound block analysis (system/skill prompts, prompts, tool results). - llm_bound_item_count: int - llm_block_types: list[BlockTypeStat] - cross_type_block_groups: list[CrossTypeBlockGroup] - cross_type_wasted_tokens: int - # Worker Context Routing shadow mode (P0 data collection; never prunes). - worker_routing: WorkerRoutingShadow - # Parent Aggregation Artifacts shadow mode (P0 telemetry; never dedups). - parent_aggregation: ParentAggregationArtifacts - notes: list[str] = field(default_factory=list) - - -# --------------------------------------------------------------------------- -# Loading -# --------------------------------------------------------------------------- - - -@dataclass -class _ToolMessage: - tool_name: str | None - content: str - - -@dataclass -class _LLMContent: - """A chunk of content that Hermes would actually send to the LLM. - - Held in-memory only for hashing; ``content`` must never be emitted. - """ - - block_type: str - content: str - - -def _window_cutoff(since_hours: int, all_sessions: bool) -> float | None: - """Return the epoch cutoff, or ``None`` to scan all history. - - ``all_sessions=True`` disables the time window so old sessions/messages are - included regardless of ``since_hours``. - """ - if all_sessions: - return None - return dt.datetime.now(dt.timezone.utc).timestamp() - since_hours * 3600 - - -def _classify_system_prompt(text: str) -> str: - """Heuristically label a system prompt as skill material or a plain prompt. - - Operates on in-memory text only; returns a low-cardinality enum, never the - text itself. - """ - low = text.lower() - stripped = low.lstrip() - # Skill-style frontmatter block (e.g. "---\nname: ...\ndescription: ..."). - if stripped.startswith("---") and "name:" in low[:300]: - return "skill_prompt" - cues = ( - "use this skill", - "available skills", - "when to use", - "invoke it via skill", - " str: - if role == "tool" or tool_name is not None: - return "tool_result" - if role == "user": - return "user_prompt" - if role == "assistant": - return "assistant_context" - if role == "system": - return "system_prompt" - return "unknown" - - -def load_tool_messages( - db_path: Path, *, since_hours: int, all_sessions: bool = False -) -> list[_ToolMessage]: - """Load tool-output messages within the window. - - Content is returned for in-memory hashing only; callers must not emit it. - A message is treated as tool output when ``role='tool'`` or ``tool_name`` - is set. With ``all_sessions=True`` the time window is ignored. - """ - cutoff = _window_cutoff(since_hours, all_sessions) - conn = _connect_readonly(db_path) - try: - cols = {row[1] for row in conn.execute("PRAGMA table_info(messages)")} - if "content" not in cols: - return [] - has_tool_name = "tool_name" in cols - has_ts = "timestamp" in cols - select_tool = "tool_name" if has_tool_name else "NULL AS tool_name" - where = [] - params: list[object] = [] - if has_ts and cutoff is not None: - where.append("timestamp >= ?") - params.append(cutoff) - if "active" in cols: - where.append("active = 1") - tool_pred = "role = 'tool'" - if has_tool_name: - tool_pred = "(role = 'tool' OR tool_name IS NOT NULL)" - where.append(tool_pred) - sql = ( - f"SELECT {select_tool}, content FROM messages " - f"WHERE {' AND '.join(where)}" - ) - rows = conn.execute(sql, params).fetchall() - finally: - conn.close() - - out: list[_ToolMessage] = [] - for tool_name, content in rows: - if content is None: - continue - out.append(_ToolMessage(tool_name=tool_name, content=str(content))) - return out - - -def load_llm_bound_content( - db_path: Path, *, since_hours: int, all_sessions: bool = False -) -> list[_LLMContent]: - """Load only content Hermes would actually send to an LLM. - - Sources, all read in-memory for hashing (never emitted): - * ``sessions.system_prompt`` -> ``system_prompt`` or ``skill_prompt``, - * ``messages.content`` for active messages with role in - ``system``/``user``/``assistant``/``tool`` -> per-role block type, - * tool-result messages (role=tool or ``tool_name`` set) -> ``tool_result``. - - Inactive messages are skipped when an ``active`` column exists; archived - sessions (and their messages) are skipped when an ``archived`` column - exists. With ``all_sessions=True`` the time window is ignored. - """ - cutoff = _window_cutoff(since_hours, all_sessions) - conn = _connect_readonly(db_path) - out: list[_LLMContent] = [] - try: - scols = {row[1] for row in conn.execute("PRAGMA table_info(sessions)")} - mcols = {row[1] for row in conn.execute("PRAGMA table_info(messages)")} - - # --- system / skill prompts from sessions ------------------------- - if "system_prompt" in scols: - where = ["system_prompt IS NOT NULL"] - params: list[object] = [] - if cutoff is not None and "started_at" in scols: - where.append("started_at >= ?") - params.append(cutoff) - if "archived" in scols: - where.append("archived = 0") - sql = f"SELECT system_prompt FROM sessions WHERE {' AND '.join(where)}" - for (sp,) in conn.execute(sql, params): - if sp is None: - continue - text = str(sp) - out.append( - _LLMContent(block_type=_classify_system_prompt(text), content=text) - ) - - # --- active messages bound for the LLM ---------------------------- - if "content" in mcols: - has_role = "role" in mcols - has_tool_name = "tool_name" in mcols - select = [ - "messages.role" if has_role else "NULL AS role", - "messages.content", - "messages.tool_name" if has_tool_name else "NULL AS tool_name", - ] - where = ["messages.content IS NOT NULL"] - params = [] - if has_role: - where.append( - "messages.role IN ('system', 'user', 'assistant', 'tool')" - ) - if cutoff is not None and "timestamp" in mcols: - where.append("messages.timestamp >= ?") - params.append(cutoff) - if "active" in mcols: - where.append("messages.active = 1") - join = "" - if "archived" in scols and "session_id" in mcols and "id" in scols: - join = " JOIN sessions ON sessions.id = messages.session_id" - where.append("sessions.archived = 0") - sql = ( - f"SELECT {', '.join(select)} FROM messages{join} " - f"WHERE {' AND '.join(where)}" - ) - for role, content, tool_name in conn.execute(sql, params): - if content is None: - continue - out.append( - _LLMContent( - block_type=_message_block_type(role, tool_name), - content=str(content), - ) - ) - finally: - conn.close() - return out - - -def load_heavy_sessions( - db_path: Path, *, since_hours: int, salt: str, top_n: int, all_sessions: bool = False -) -> list[HeavySession]: - cutoff = _window_cutoff(since_hours, all_sessions) - conn = _connect_readonly(db_path) - try: - cols = {row[1] for row in conn.execute("PRAGMA table_info(sessions)")} - if "id" not in cols: - return [] - wanted = [ - "id", - "source", - "input_tokens", - "output_tokens", - "message_count", - "tool_call_count", - "api_call_count", - ] - select_cols = [c if c in cols else f"NULL AS {c}" for c in wanted] - where = [] - params: list[object] = [] - if cutoff is not None and "started_at" in cols: - where.append("started_at >= ?") - params.append(cutoff) - if "archived" in cols: - where.append("archived = 0") - sql = f"SELECT {', '.join(select_cols)} FROM sessions" - if where: - sql += " WHERE " + " AND ".join(where) - sql += " ORDER BY input_tokens DESC" - rows = conn.execute(sql, params).fetchall() - finally: - conn.close() - - sessions: list[HeavySession] = [] - for sid, source, inp, out_tok, msgs, tools, apis in rows: - sessions.append( - HeavySession( - session_hash=_salted_hash(str(sid), salt), - source=source, - input_tokens=int(inp or 0), - output_tokens=int(out_tok or 0), - message_count=int(msgs or 0), - tool_call_count=int(tools or 0), - api_call_count=int(apis or 0), - ) - ) - sessions.sort(key=lambda s: (s.input_tokens, s.tool_call_count), reverse=True) - return sessions[:top_n] - - -def total_input_tokens( - db_path: Path, *, since_hours: int, all_sessions: bool = False -) -> int: - """Sum input tokens across ALL in-window sessions (not just the top-N).""" - cutoff = _window_cutoff(since_hours, all_sessions) - conn = _connect_readonly(db_path) - try: - cols = {row[1] for row in conn.execute("PRAGMA table_info(sessions)")} - if "input_tokens" not in cols: - return 0 - where = [] - params: list[object] = [] - if cutoff is not None and "started_at" in cols: - where.append("started_at >= ?") - params.append(cutoff) - if "archived" in cols: - where.append("archived = 0") - sql = "SELECT COALESCE(SUM(input_tokens), 0) FROM sessions" - if where: - sql += " WHERE " + " AND ".join(where) - (total,) = conn.execute(sql, params).fetchone() - finally: - conn.close() - return int(total or 0) - - -def parse_telemetry( - telemetry_path: Path, - *, - since_hours: int, - total_input_tokens: int, - all_sessions: bool = False, -) -> TelemetryCoverage: - """Aggregate the metadata-only ContextPilot telemetry file. - - Tolerates malformed lines (non-JSON, non-dict, missing counters) by - skipping and counting them. Never reads message content. With - ``all_sessions=True`` the time window is ignored. - """ - events = 0 - chars = 0 - tokens = 0 - malformed = 0 - if telemetry_path and telemetry_path.exists(): - cutoff = _window_cutoff(since_hours, all_sessions) - with telemetry_path.open("r", encoding="utf-8", errors="replace") as f: - for raw in f: - line = raw.strip() - if not line: - continue - try: - record = json.loads(line) - except (ValueError, TypeError): - malformed += 1 - continue - if not isinstance(record, dict): - malformed += 1 - continue - ts = record.get("ts") - if cutoff is not None and isinstance(ts, (int, float)) and ts < cutoff: - continue - cs = record.get("chars_saved") - if not isinstance(cs, (int, float)): - malformed += 1 - continue - saved_tokens = record.get("tokens_saved") - events += 1 - chars += int(cs) - tokens += ( - int(saved_tokens) - if isinstance(saved_tokens, (int, float)) - else int(cs) // EST_CHARS_PER_TOKEN - ) - - denom = tokens + total_input_tokens - coverage = (tokens / denom * 100.0) if denom else 0.0 - avg = (tokens / events) if events else 0.0 - return TelemetryCoverage( - events=events, - chars_saved=chars, - tokens_saved=tokens, - avg_tokens_saved_per_event=round(avg, 2), - coverage_ratio_pct=round(coverage, 2), - malformed_records_skipped=malformed, - ) - - -# --------------------------------------------------------------------------- -# Detection -# --------------------------------------------------------------------------- - - -def detect_exact_duplicate_tool_outputs( - messages: Iterable[_ToolMessage], *, salt: str, top_n: int -) -> list[DuplicateToolOutput]: - groups: dict[str, dict] = {} - for msg in messages: - content = msg.content - if not content: - continue - h = _salted_hash(content, salt) - g = groups.get(h) - if g is None: - groups[h] = { - "tool_name": msg.tool_name, - "occurrences": 1, - "char_length": len(content), - } - else: - g["occurrences"] += 1 - if g["tool_name"] != msg.tool_name: - g["tool_name"] = None # mixed tools produced identical output - - dups: list[DuplicateToolOutput] = [] - for h, g in groups.items(): - if g["occurrences"] < 2: - continue - est = _est_tokens(g["char_length"]) - dups.append( - DuplicateToolOutput( - content_hash=h, - tool_name=g["tool_name"], - occurrences=g["occurrences"], - char_length=g["char_length"], - est_tokens=est, - est_wasted_tokens=est * (g["occurrences"] - 1), - ) - ) - dups.sort(key=lambda d: d.est_wasted_tokens, reverse=True) - return dups[:top_n] - - -def detect_repeated_blocks( - messages: Iterable[_ToolMessage], - *, - salt: str, - min_block_chars: int, - min_repeat: int, - top_n: int, -) -> list[RepeatedBlock]: - counts: dict[str, dict] = {} - for msg in messages: - seen_in_msg: set[str] = set() - for line in msg.content.splitlines(): - block = line.strip() - if len(block) < min_block_chars: - continue - h = _salted_hash(block, salt) - # Count cross-message recurrence; collapse repeats within one - # message so a single noisy output cannot dominate. - if h in seen_in_msg: - continue - seen_in_msg.add(h) - c = counts.get(h) - if c is None: - counts[h] = {"occurrences": 1, "char_length": len(block)} - else: - c["occurrences"] += 1 - - blocks: list[RepeatedBlock] = [] - for h, c in counts.items(): - if c["occurrences"] < min_repeat: - continue - est = _est_tokens(c["char_length"]) - blocks.append( - RepeatedBlock( - block_hash=h, - occurrences=c["occurrences"], - char_length=c["char_length"], - est_tokens=est, - est_wasted_tokens=est * (c["occurrences"] - 1), - ) - ) - blocks.sort(key=lambda b: b.est_wasted_tokens, reverse=True) - return blocks[:top_n] - - -def summarize_tool_sizes( - messages: Iterable[_ToolMessage], *, large_output_chars: int, top_n: int -) -> list[ToolSizeStat]: - agg: dict[str, dict] = {} - for msg in messages: - name = msg.tool_name or "(unknown)" - length = len(msg.content) - a = agg.get(name) - if a is None: - agg[name] = { - "output_count": 1, - "total_chars": length, - "max_chars": length, - "large_output_count": 1 if length >= large_output_chars else 0, - } - else: - a["output_count"] += 1 - a["total_chars"] += length - a["max_chars"] = max(a["max_chars"], length) - if length >= large_output_chars: - a["large_output_count"] += 1 - - stats: list[ToolSizeStat] = [] - for name, a in agg.items(): - stats.append( - ToolSizeStat( - tool_name=name, - output_count=a["output_count"], - total_chars=a["total_chars"], - max_chars=a["max_chars"], - avg_chars=a["total_chars"] // a["output_count"], - total_est_tokens=_est_tokens(a["total_chars"]), - large_output_count=a["large_output_count"], - ) - ) - stats.sort(key=lambda s: s.total_chars, reverse=True) - return stats[:top_n] - - -def _iter_blocks(content: str, min_block_chars: int) -> Iterable[str]: - """Yield the distinct fingerprintable lines of one item (deduped in-item).""" - seen: set[str] = set() - for line in content.splitlines(): - block = line.strip() - if len(block) < min_block_chars: - continue - if block in seen: - continue - seen.add(block) - yield block - - -def analyze_llm_bound_blocks( - contents: Iterable[_LLMContent], - *, - salt: str, - min_block_chars: int, - min_repeat: int, - top_n: int, -) -> tuple[list[BlockTypeStat], list[CrossTypeBlockGroup]]: - """Fingerprint LLM-bound blocks and report redundancy. - - Returns (per-type stats, cross-type repeated block groups). All output is - salted hashes / counters / block-type enums -- no raw text. - """ - # block_hash -> {char_length, types: {block_type: occ}} - agg: dict[str, dict] = {} - # block_type -> source item count - item_counts: dict[str, int] = {} - - for item in contents: - bt = item.block_type - item_counts[bt] = item_counts.get(bt, 0) + 1 - for block in _iter_blocks(item.content, min_block_chars): - h = _salted_hash(block, salt) - entry = agg.get(h) - if entry is None: - agg[h] = {"char_length": len(block), "types": {bt: 1}} - else: - entry["types"][bt] = entry["types"].get(bt, 0) + 1 - - # --- per block-type aggregate redundancy ------------------------------ - per_type: dict[str, dict] = {} - for entry in agg.values(): - est = _est_tokens(entry["char_length"]) - for bt, occ in entry["types"].items(): - t = per_type.setdefault( - bt, - { - "block_count": 0, - "unique": 0, - "repeated": 0, - "redundant_tokens": 0, - }, - ) - t["block_count"] += occ - t["unique"] += 1 - if occ >= min_repeat: - t["repeated"] += 1 - t["redundant_tokens"] += est * (occ - 1) - - block_type_stats: list[BlockTypeStat] = [] - for bt in sorted(set(per_type) | set(item_counts)): - t = per_type.get( - bt, {"block_count": 0, "unique": 0, "repeated": 0, "redundant_tokens": 0} - ) - block_type_stats.append( - BlockTypeStat( - block_type=bt, - item_count=item_counts.get(bt, 0), - block_count=t["block_count"], - unique_block_count=t["unique"], - repeated_block_count=t["repeated"], - est_redundant_tokens=t["redundant_tokens"], - ) - ) - - # --- cross-type repeated blocks --------------------------------------- - cross: list[CrossTypeBlockGroup] = [] - for h, entry in agg.items(): - types = entry["types"] - if len(types) < 2: - continue - total_occ = sum(types.values()) - est = _est_tokens(entry["char_length"]) - cross.append( - CrossTypeBlockGroup( - block_hash=h, - block_types=sorted(types.keys()), - type_occurrences=[ - TypeCount(block_type=bt, count=occ) - for bt, occ in sorted(types.items()) - ], - occurrences=total_occ, - char_length=entry["char_length"], - est_tokens=est, - est_wasted_tokens=est * (total_occ - 1), - ) - ) - cross.sort(key=lambda g: g.est_wasted_tokens, reverse=True) - return block_type_stats, cross[:top_n] - - -def classify_router_label( - block_type: str, - content: str, - *, - occurrences: int, - large_output_chars: int, - min_repeat: int, -) -> tuple[str, str]: - """Heuristically assign a worker-routing label + reason code to a block. - - Pure P0 heuristic: no ML, no network, no mutation. Operates on in-memory - text only and returns two low-cardinality enums (``route_label``, - ``reason_code``) -- never the text. The bias is deliberately conservative: - when in doubt, keep. Anything that is a user prompt, a system/skill prompt, - or carries explicit safety/acceptance-constraint language is pinned to - ``policy_must_keep`` and can never become a routable candidate. - """ - low = content.lower() - - # 1. Never-drop by origin: prompts the user/system/skills authored. - if block_type == "user_prompt": - return "policy_must_keep", "user_prompt_never_drop" - if block_type in ("system_prompt", "skill_prompt"): - return "policy_must_keep", "system_or_skill_constraint_never_drop" - - # 2. Never-drop by content: explicit safety / acceptance / hard constraints, - # even inside an assistant or tool block. - if any(cue in low for cue in _SAFETY_CONSTRAINT_CUES): - return "policy_must_keep", "safety_or_acceptance_constraint" - - char_len = len(content) - has_task_hint = any(cue in low for cue in _TASK_HINT_CUES) - - # 3. Short actionable task hints -> keep verbatim. Very large diagnostic - # logs often contain "error:"/"failed"/"traceback"; keep collecting - # them as summarization candidates instead of pinning the whole log. - if has_task_hint and char_len < large_output_chars: - return "direct_task_hint", "actionable_task_signal" - - # 4. Bulky / repeated tool-like material -> routable candidates (advisory). - if block_type in ("tool_result", "assistant_context", "unknown"): - if has_task_hint and char_len >= large_output_chars: - return "summarizable_candidate", "large_actionable_tool_block" - is_large = char_len >= large_output_chars - is_repeated = occurrences >= min_repeat - if is_large and is_repeated: - return "likely_drop_candidate", "large_repeated_tool_block" - if is_repeated: - return "likely_drop_candidate", "repeated_tool_block" - if is_large: - return "summarizable_candidate", "large_single_tool_block" - - # 5. Everything else: keep by default. - return "likely_relevant", "default_keep" - - -def analyze_worker_routing_shadow( - contents: Iterable[_LLMContent], - *, - salt: str, - large_output_chars: int, - min_repeat: int, - top_n: int, - enabled: bool = True, -) -> WorkerRoutingShadow: - """Shadow-mode worker-context routing classifier (P0: data collection only). - - Fingerprints each LLM-bound item, assigns a conservative router label, and - returns aggregate counters + salted hashes for routable candidates. Emits - NO raw text and never mutates/drops context. ``est_candidate_tokens`` is an - advisory upper bound on what a *future* router might route away -- not a - realized saving. - """ - if not enabled: - return WorkerRoutingShadow( - enabled=False, - item_count=0, - classified_block_count=0, - total_occurrences=0, - must_keep_block_count=0, - must_keep_occurrence_count=0, - est_must_keep_tokens=0, - est_candidate_tokens_total=0, - est_drop_candidate_tokens=0, - est_summarizable_candidate_tokens=0, - label_counts=[], - reason_counts=[], - top_candidate_blocks=[], - notes=["worker-routing shadow analysis disabled via flag"], - ) - - # Aggregate occurrences per fingerprint, picking the most must-keep origin - # when one block spans several block types. - agg: dict[str, dict] = {} - item_count = 0 - for item in contents: - content = item.content - if not content: - continue - item_count += 1 - h = _salted_hash(content, salt) - bt = item.block_type - entry = agg.get(h) - if entry is None: - agg[h] = { - "block_type": bt, - "char_length": len(content), - "occurrences": 1, - "content": content, - } - else: - entry["occurrences"] += 1 - cur = entry["block_type"] - bt_pri = _TYPE_KEEP_PRIORITY.get(bt, 0) - cur_pri = _TYPE_KEEP_PRIORITY.get(cur, 0) - if bt_pri > cur_pri or (bt_pri == cur_pri and bt < cur): - entry["block_type"] = bt - - # Classify each unique fingerprint and roll up counters. - label_agg: dict[str, dict] = {} - reason_agg: dict[tuple[str, str, str], dict] = {} - candidates: list[RouterCandidateBlock] = [] - must_keep_blocks = 0 - must_keep_occ = 0 - est_must_keep_tokens = 0 - drop_tokens = 0 - summ_tokens = 0 - - for h, entry in agg.items(): - bt = entry["block_type"] - occ = entry["occurrences"] - char_len = entry["char_length"] - est = _est_tokens(char_len) - total_est = est * occ - label, reason = classify_router_label( - bt, - entry["content"], - occurrences=occ, - large_output_chars=large_output_chars, - min_repeat=min_repeat, - ) - candidate_tokens = total_est if label in _ROUTABLE_LABELS else 0 - - la = label_agg.setdefault( - label, - {"block_count": 0, "occ": 0, "total_est": 0, "candidate": 0}, - ) - la["block_count"] += 1 - la["occ"] += occ - la["total_est"] += total_est - la["candidate"] += candidate_tokens - - ra = reason_agg.setdefault( - (bt, label, reason), - {"block_count": 0, "occ": 0, "total_est": 0, "candidate": 0}, - ) - ra["block_count"] += 1 - ra["occ"] += occ - ra["total_est"] += total_est - ra["candidate"] += candidate_tokens - - if label == "policy_must_keep": - must_keep_blocks += 1 - must_keep_occ += occ - est_must_keep_tokens += total_est - if label == "likely_drop_candidate": - drop_tokens += candidate_tokens - elif label == "summarizable_candidate": - summ_tokens += candidate_tokens - - if candidate_tokens > 0: - candidates.append( - RouterCandidateBlock( - block_hash=h, - block_type=bt, - route_label=label, - reason_code=reason, - occurrences=occ, - char_length=char_len, - est_tokens=est, - est_candidate_tokens=candidate_tokens, - ) - ) - - # Deterministic ordering: label_counts follow the canonical label order; - # reason_counts and candidates sort by a stable key. - label_counts = [ - RouterLabelCount( - route_label=lbl, - block_count=label_agg[lbl]["block_count"], - occurrence_count=label_agg[lbl]["occ"], - total_est_tokens=label_agg[lbl]["total_est"], - est_candidate_tokens=label_agg[lbl]["candidate"], - ) - for lbl in ROUTER_LABELS - if lbl in label_agg - ] - reason_counts = [ - RouterReasonCount( - block_type=bt, - route_label=lbl, - reason_code=reason, - block_count=v["block_count"], - occurrence_count=v["occ"], - total_est_tokens=v["total_est"], - est_candidate_tokens=v["candidate"], - ) - for (bt, lbl, reason), v in sorted(reason_agg.items()) - ] - candidates.sort( - key=lambda c: (c.est_candidate_tokens, c.occurrences, c.block_hash), - reverse=True, - ) - - total_occ = sum(e["occurrences"] for e in agg.values()) - notes = [ - "SHADOW MODE P0: classification only -- no context was dropped, summarized, or mutated", - "route_label/reason_code/block_type are low-cardinality enums; block_hash is a salted SHA-256 fingerprint", - "est_candidate_tokens is ADVISORY (an upper bound for a FUTURE router), not a realized saving", - "user/system/skill prompts and safety/acceptance constraints are pinned to policy_must_keep and never routable", - "classification is conservative: when uncertain, blocks are kept (likely_relevant)", - ] - - return WorkerRoutingShadow( - enabled=True, - item_count=item_count, - classified_block_count=len(agg), - total_occurrences=total_occ, - must_keep_block_count=must_keep_blocks, - must_keep_occurrence_count=must_keep_occ, - est_must_keep_tokens=est_must_keep_tokens, - est_candidate_tokens_total=drop_tokens + summ_tokens, - est_drop_candidate_tokens=drop_tokens, - est_summarizable_candidate_tokens=summ_tokens, - label_counts=label_counts, - reason_counts=reason_counts, - top_candidate_blocks=candidates[:top_n], - notes=notes, - ) - - -# --------------------------------------------------------------------------- -# Parent Aggregation Artifacts — SHADOW MODE (P0 telemetry only) -# --------------------------------------------------------------------------- -# When a parent/orchestrator aggregates results from several workers, the same -# artifact body (a test log, a diff, a file dump, a review summary, ...) is often -# carried into the parent's LLM context once per worker and again in the parent's -# own roll-up -- paying for the same tokens several times. This section collects -# *telemetry only* so a future parent-aggregation dedup can be evaluated offline: -# it groups EXACT artifact bodies by salted content hash, classifies each body -# with a deterministic heuristic kind, and emits low-cardinality metadata + -# counters. It NEVER drops, summarizes, replaces, or mutates any context, and it -# NEVER emits raw artifact text, worker text, tool output, session ids, or -# system prompts. - -# Heuristic P0 artifact kinds. Low-cardinality enums describing the *shape* of an -# aggregation artifact, never its text. Classification is deterministic. -ARTIFACT_KINDS = ( - "test_log", - "terminal_output", - "file_content", - "diff", - "error_trace", - "review_findings", - "benchmark_result", - "worker_summary", - "unknown_large_block", -) - -# Conservative floor: only sizeable blocks are treated as candidate aggregation -# artifacts, so short prompts/hints never enter parent-aggregation telemetry. -DEFAULT_MIN_ARTIFACT_CHARS = 400 - -# Parent aggregation P0 focuses on content produced by workers/tools and then -# carried into the parent context. System/skill/user prompts are analyzed by the -# LLM-bound redundancy and worker-routing sections, but excluding them here keeps -# parent artifact telemetry from being polluted by prompt boilerplate. -PARENT_AGGREGATION_SOURCE_TYPES = ("assistant_context", "tool_result") - - -def classify_artifact_kind(content: str) -> str: - """Deterministically classify a candidate aggregation artifact body. - - Pure P0 heuristic over in-memory text; returns a low-cardinality enum from - ``ARTIFACT_KINDS`` and never the text. The check order is fixed so the same - body always yields the same kind (first match wins). - """ - low = content.lower() - stripped = content.lstrip() - - # 1. Unified diff / patch. - if ( - stripped.startswith("diff --git") - or stripped.startswith("--- a/") - or stripped.startswith("@@ ") - or "\n@@ " in content - or ("\n--- " in content and "\n+++ " in content) - ): - return "diff" - - # 2. Test/pytest log (checked before error_trace: a failing test log may - # embed a traceback but is still fundamentally a test log). - if ( - "pytest" in low - or "test session starts" in low - or " passed in " in low - or " failed in " in low - or ("passed" in low and "failed" in low) - or "=== " in content - ): - return "test_log" - - # 3. Error / exception trace. - if ( - "traceback (most recent call last)" in low - or "\n at " in content - or "stack trace" in low - or ("exception" in low and "error" in low) - ): - return "error_trace" - - # 4. Benchmark / perf result. - if ( - "benchmark" in low - or "ops/sec" in low - or "ops/s" in low - or "req/sec" in low - or "throughput" in low - or "latency" in low - or "iterations/sec" in low - ): - return "benchmark_result" - - # 5. Code-review findings. - if ( - "code review" in low - or "review findings" in low - or "severity:" in low - or "vulnerab" in low - or "## findings" in low - ): - return "review_findings" - - # 6. File content / source dump (cat -n style numbering or code cues). - if ( - "\n 1\t" in content - or "\n 1\t" in content - or "def " in content - or "class " in content - or "\nimport " in content - or "#include" in content - or "function " in content - ): - return "file_content" - - # 7. Worker / aggregation summary. Checked after source-code cues so files - # mentioning workers are still labeled as file_content. - if ( - "## summary" in low - or "in summary" in low - or "summary:" in low - or "tl;dr" in low - or "aggregat" in low - or "worker" in low - ): - return "worker_summary" - - # 8. Terminal / shell session output. - if ( - "\n$ " in content - or stripped.startswith("$ ") - or "\n# " in content - or "user@" in low - or "bash-" in low - or "exit code" in low - ): - return "terminal_output" - - # 9. Fallback: a large block we could not confidently classify. - return "unknown_large_block" - - -@dataclass -class ArtifactSourceCount: - """Provenance counter: occurrences of one artifact body from one source.""" - - source_type: str - count: int - - -@dataclass -class ParentAggregationGroup: - """One EXACT artifact body observed 2+ times across parent/worker contexts. - - Salted hash + counters only -- never the body text. - """ - - content_hash: str - artifact_kind: str - canonical_source_type: str # dominant origin, chosen deterministically - occurrences: int - char_length: int - est_tokens: int - est_duplicate_tokens: int # ADVISORY: (occurrences - 1) * est_tokens - source_type_counts: list[ArtifactSourceCount] # provenance: tool_result xN, ... - - -@dataclass -class ArtifactKindStat: - """Aggregate over all candidate artifact bodies of one kind.""" - - artifact_kind: str - group_count: int # distinct bodies of this kind - occurrence_count: int # total occurrences of those bodies - duplicate_group_count: int # bodies seen >= 2 times - est_tokens: int # sum of est tokens for distinct bodies - est_duplicate_tokens: int # ADVISORY duplicate tokens for this kind - - -@dataclass -class ParentAggregationArtifacts: - """Shadow-mode parent-aggregation artifact report (P0: telemetry only).""" - - enabled: bool - item_count: int # candidate artifact items considered - artifact_body_count: int # distinct bodies (groups) - total_occurrences: int - duplicate_group_count: int - est_total_tokens: int # est tokens for distinct bodies - est_duplicate_tokens: int # ADVISORY duplicate-artifact tokens - by_kind: list[ArtifactKindStat] - source_type_counts: list[ArtifactSourceCount] # provenance across candidates - top_duplicate_groups: list[ParentAggregationGroup] - notes: list[str] = field(default_factory=list) - - -def analyze_parent_aggregation_artifacts( - contents: Iterable[_LLMContent], - *, - salt: str, - min_artifact_chars: int, - top_n: int, - enabled: bool = True, -) -> ParentAggregationArtifacts: - """Group EXACT aggregation-artifact bodies and emit provenance telemetry. - - P0 telemetry/advisory only: no context is dropped, summarized, replaced, or - mutated. Each sizeable LLM-bound block is fingerprinted by EXACT salted - content hash (near-duplicates never group), classified with a deterministic - heuristic kind, and rolled up into low-cardinality metadata + counters. - ``est_duplicate_tokens`` is an advisory upper bound on what a *future* parent - dedup might save -- never a realized saving. No raw artifact/worker/tool/ - system text, and no raw session ids, are ever emitted. - """ - if not enabled: - return ParentAggregationArtifacts( - enabled=False, - item_count=0, - artifact_body_count=0, - total_occurrences=0, - duplicate_group_count=0, - est_total_tokens=0, - est_duplicate_tokens=0, - by_kind=[], - source_type_counts=[], - top_duplicate_groups=[], - notes=["parent-aggregation artifact analysis disabled via flag"], - ) - - # --- group sizeable bodies by EXACT salted content hash ---------------- - groups: dict[str, dict] = {} - item_count = 0 - source_totals: dict[str, int] = {} - for item in contents: - content = item.content - bt = item.block_type - if bt not in PARENT_AGGREGATION_SOURCE_TYPES: - continue - if not content or len(content) < min_artifact_chars: - continue - item_count += 1 - source_totals[bt] = source_totals.get(bt, 0) + 1 - h = _salted_hash(content, salt) - g = groups.get(h) - if g is None: - groups[h] = { - "char_length": len(content), - "occurrences": 1, - "sources": {bt: 1}, - # classify once from in-memory text; never stored/emitted. - "kind": classify_artifact_kind(content), - } - else: - g["occurrences"] += 1 - g["sources"][bt] = g["sources"].get(bt, 0) + 1 - - # --- per-kind rollup + per-group records ------------------------------- - kind_agg: dict[str, dict] = {} - group_records: list[ParentAggregationGroup] = [] - total_occurrences = 0 - est_total_tokens = 0 - est_duplicate_tokens = 0 - duplicate_group_count = 0 - - for h, g in groups.items(): - occ = g["occurrences"] - char_len = g["char_length"] - est = _est_tokens(char_len) - dup_tokens = est * (occ - 1) - kind = g["kind"] - is_dup = occ >= 2 - - total_occurrences += occ - est_total_tokens += est - est_duplicate_tokens += dup_tokens - if is_dup: - duplicate_group_count += 1 - - ka = kind_agg.setdefault( - kind, - {"groups": 0, "occ": 0, "dups": 0, "est": 0, "dup_tokens": 0}, - ) - ka["groups"] += 1 - ka["occ"] += occ - ka["est"] += est - ka["dup_tokens"] += dup_tokens - if is_dup: - ka["dups"] += 1 - - if is_dup: - # Provenance counts, sorted by source_type for determinism. - source_counts = [ - ArtifactSourceCount(source_type=st, count=c) - for st, c in sorted(g["sources"].items()) - ] - # Canonical source: dominant origin, tie-broken alphabetically. - canonical = min( - g["sources"].items(), key=lambda kv: (-kv[1], kv[0]) - )[0] - group_records.append( - ParentAggregationGroup( - content_hash=h, - artifact_kind=kind, - canonical_source_type=canonical, - occurrences=occ, - char_length=char_len, - est_tokens=est, - est_duplicate_tokens=dup_tokens, - source_type_counts=source_counts, - ) - ) - - by_kind = [ - ArtifactKindStat( - artifact_kind=kind, - group_count=kind_agg[kind]["groups"], - occurrence_count=kind_agg[kind]["occ"], - duplicate_group_count=kind_agg[kind]["dups"], - est_tokens=kind_agg[kind]["est"], - est_duplicate_tokens=kind_agg[kind]["dup_tokens"], - ) - for kind in ARTIFACT_KINDS - if kind in kind_agg - ] - source_type_counts = [ - ArtifactSourceCount(source_type=st, count=c) - for st, c in sorted(source_totals.items()) - ] - group_records.sort( - key=lambda g: (g.est_duplicate_tokens, g.occurrences, g.content_hash), - reverse=True, - ) - - notes = [ - "SHADOW MODE P0: telemetry only -- no aggregation artifact was deduped, replaced, summarized, or mutated", - "artifact_kind/source_type/canonical_source_type are low-cardinality enums; content_hash is a salted SHA-256 fingerprint", - "grouping is EXACT (same salted content hash): near-duplicate artifacts never group", - "est_duplicate_tokens is ADVISORY ((occurrences-1) * est_tokens), an upper bound for a FUTURE parent dedup -- not a realized saving", - "provenance source_type_counts show how many copies came from each parent/worker output origin (assistant_context, tool_result)", - ] - - return ParentAggregationArtifacts( - enabled=True, - item_count=item_count, - artifact_body_count=len(groups), - total_occurrences=total_occurrences, - duplicate_group_count=duplicate_group_count, - est_total_tokens=est_total_tokens, - est_duplicate_tokens=est_duplicate_tokens, - by_kind=by_kind, - source_type_counts=source_type_counts, - top_duplicate_groups=group_records[:top_n], - notes=notes, - ) - - -# --------------------------------------------------------------------------- -# Build + write -# --------------------------------------------------------------------------- - - -def build_report( - *, - date: str, - since_hours: int, - salt: str, - tool_messages: list[_ToolMessage], - heavy_sessions: list[HeavySession], - telemetry: TelemetryCoverage, - llm_contents: list[_LLMContent] | None = None, - all_sessions: bool = False, - min_block_chars: int = DEFAULT_MIN_BLOCK_CHARS, - min_block_repeat: int = DEFAULT_MIN_BLOCK_REPEAT, - large_output_chars: int = DEFAULT_LARGE_OUTPUT_CHARS, - top_n: int = DEFAULT_TOP_N, - worker_routing_shadow: bool = True, - parent_aggregation_shadow: bool = True, - min_artifact_chars: int = DEFAULT_MIN_ARTIFACT_CHARS, -) -> OpportunityReport: - dups = detect_exact_duplicate_tool_outputs(tool_messages, salt=salt, top_n=top_n) - blocks = detect_repeated_blocks( - tool_messages, - salt=salt, - min_block_chars=min_block_chars, - min_repeat=min_block_repeat, - top_n=top_n, - ) - sizes = summarize_tool_sizes( - tool_messages, large_output_chars=large_output_chars, top_n=top_n - ) - - llm_contents = llm_contents or [] - block_type_stats, cross_groups = analyze_llm_bound_blocks( - llm_contents, - salt=salt, - min_block_chars=min_block_chars, - min_repeat=min_block_repeat, - top_n=top_n, - ) - - worker_routing = analyze_worker_routing_shadow( - llm_contents, - salt=salt, - large_output_chars=large_output_chars, - min_repeat=min_block_repeat, - top_n=top_n, - enabled=worker_routing_shadow, - ) - - parent_aggregation = analyze_parent_aggregation_artifacts( - llm_contents, - salt=salt, - min_artifact_chars=min_artifact_chars, - top_n=top_n, - enabled=parent_aggregation_shadow, - ) - - total_chars = sum(len(m.content) for m in tool_messages) - dup_wasted = sum(d.est_wasted_tokens for d in dups) - block_wasted = sum(b.est_wasted_tokens for b in blocks) - cross_wasted = sum(g.est_wasted_tokens for g in cross_groups) - - notes = [ - "content-aware analysis: message/tool text was hashed in-memory only and never written to reports", - "all identifiers are salted SHA-256 fingerprints; counters are aggregates", - "wasted-token figures are heuristic estimates (chars/4); validate before acting", - "session 'source', 'tool_name', and block_type are emitted verbatim as low-cardinality enums, not raw text", - "llm-bound scan covers only content sent to the LLM: system/skill prompts, active user/assistant/tool messages", - "worker-routing section is SHADOW MODE P0: it labels blocks for a future router but never drops/summarizes context", - "parent-aggregation section is SHADOW MODE P0 telemetry: it groups exact artifact bodies but never dedups/replaces context", - ] - if all_sessions: - notes.append("all-sessions mode: time window ignored; scanned all non-archived sessions/active messages") - if not tool_messages: - notes.append("no tool-output messages observed in the selected window") - if not llm_contents: - notes.append("no llm-bound content observed in the selected window") - - return OpportunityReport( - date=date, - since_hours=since_hours, - all_sessions=all_sessions, - salt_fingerprint=_salt_fingerprint(salt), - tool_message_count=len(tool_messages), - total_tool_output_chars=total_chars, - total_tool_output_est_tokens=_est_tokens(total_chars), - exact_duplicate_groups=dups, - duplicate_tool_output_groups=len(dups), - duplicate_tool_output_wasted_tokens=dup_wasted, - repeated_block_count=len(blocks), - repeated_block_wasted_tokens=block_wasted, - repeated_blocks=blocks, - large_tool_outputs_by_tool=sizes, - heavy_sessions=heavy_sessions, - telemetry=telemetry, - llm_bound_item_count=len(llm_contents), - llm_block_types=block_type_stats, - cross_type_block_groups=cross_groups, - cross_type_wasted_tokens=cross_wasted, - worker_routing=worker_routing, - parent_aggregation=parent_aggregation, - notes=notes, - ) - - -def _assert_no_forbidden_keys(data: dict) -> None: - """Defensive guard: ensure no forbidden raw-content key reached the output.""" - - def walk(obj): - if isinstance(obj, dict): - for k, v in obj.items(): - if k in FORBIDDEN_OUTPUT_KEYS: - raise RuntimeError(f"refusing to emit forbidden key: {k}") - walk(v) - elif isinstance(obj, list): - for item in obj: - walk(item) - - walk(data) - - -def write_report(report: OpportunityReport, out_dir: Path) -> tuple[Path, Path]: - out_dir.mkdir(parents=True, exist_ok=True) - data = asdict(report) - _assert_no_forbidden_keys(data) - - json_path = out_dir / f"opportunities_{report.date}.json" - md_path = out_dir / f"opportunities_{report.date}.md" - json_path.write_text( - json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8" - ) - - t = report.telemetry - window = "all sessions (no time window)" if report.all_sessions else f"last {report.since_hours}h" - md = [ - f"# ContextPilot Hermes opportunity scan — {report.date}", - "", - f"Window: {window}", - f"Salt fingerprint: `{report.salt_fingerprint}`", - "", - "## Summary", - f"- Tool-output messages: {report.tool_message_count}", - f"- Total tool-output tokens (est): {report.total_tool_output_est_tokens}", - f"- Exact duplicate groups: {report.duplicate_tool_output_groups} " - f"(~{report.duplicate_tool_output_wasted_tokens} wasted tokens)", - f"- Repeated blocks: {report.repeated_block_count} " - f"(~{report.repeated_block_wasted_tokens} wasted tokens)", - f"- LLM-bound items scanned: {report.llm_bound_item_count}", - f"- Cross-type repeated blocks: {len(report.cross_type_block_groups)} " - f"(~{report.cross_type_wasted_tokens} wasted tokens)", - f"- Telemetry: {t.events} events, ~{t.tokens_saved} tokens saved, " - f"coverage {t.coverage_ratio_pct}%", - f"- Worker routing (shadow): {report.worker_routing.classified_block_count} blocks " - f"classified, {report.worker_routing.must_keep_block_count} must-keep, " - f"~{report.worker_routing.est_candidate_tokens_total} advisory candidate tokens", - f"- Parent aggregation (shadow): {report.parent_aggregation.duplicate_group_count} " - f"duplicate artifact groups, " - f"~{report.parent_aggregation.est_duplicate_tokens} advisory duplicate tokens", - "", - "## LLM-bound redundancy by block type", - ] - for bt in report.llm_block_types: - md.append( - f"- {bt.block_type}: items={bt.item_count} blocks={bt.block_count} " - f"unique={bt.unique_block_count} repeated={bt.repeated_block_count} " - f"~redundant={bt.est_redundant_tokens} tokens" - ) - md.append("") - md.append("## Cross-type repeated blocks (same block, multiple sources)") - for g in report.cross_type_block_groups: - spread = ", ".join(f"{tc.block_type}x{tc.count}" for tc in g.type_occurrences) - md.append( - f"- `{g.block_hash}` types=[{', '.join(g.block_types)}] ({spread}) " - f"chars={g.char_length} ~wasted={g.est_wasted_tokens} tokens" - ) - md.append("") - md.append("## Top exact-duplicate tool outputs") - for d in report.exact_duplicate_groups: - md.append( - f"- `{d.content_hash}` tool={d.tool_name} x{d.occurrences} " - f"chars={d.char_length} ~wasted={d.est_wasted_tokens} tokens" - ) - md.append("") - md.append("## Top repeated blocks") - for b in report.repeated_blocks: - md.append( - f"- `{b.block_hash}` x{b.occurrences} chars={b.char_length} " - f"~wasted={b.est_wasted_tokens} tokens" - ) - md.append("") - md.append("## Large tool outputs by tool") - for s in report.large_tool_outputs_by_tool: - md.append( - f"- {s.tool_name}: count={s.output_count} total_chars={s.total_chars} " - f"max={s.max_chars} avg={s.avg_chars} large(>=thresh)={s.large_output_count}" - ) - md.append("") - md.append("## Heavy sessions (hashed)") - for h in report.heavy_sessions: - md.append( - f"- `{h.session_hash}` source={h.source} input={h.input_tokens} " - f"output={h.output_tokens} msgs={h.message_count} tools={h.tool_call_count} " - f"apis={h.api_call_count}" - ) - md.append("") - md.append("## Telemetry coverage") - md.extend( - [ - f"- Events: {t.events}", - f"- Tokens saved: {t.tokens_saved} (chars {t.chars_saved})", - f"- Avg tokens saved / event: {t.avg_tokens_saved_per_event}", - f"- Coverage ratio: {t.coverage_ratio_pct}%", - f"- Malformed records skipped: {t.malformed_records_skipped}", - ] - ) - md.append("") - wr = report.worker_routing - md.append("## Worker Context Routing — shadow mode (P0, advisory only)") - if not wr.enabled: - md.append("- disabled") - else: - md.append( - f"- Items classified: {wr.item_count} " - f"(distinct fingerprints: {wr.classified_block_count}, " - f"occurrences: {wr.total_occurrences})" - ) - md.append( - f"- Must-keep: {wr.must_keep_block_count} blocks / " - f"{wr.must_keep_occurrence_count} occurrences " - f"(~{wr.est_must_keep_tokens} tokens, never routable)" - ) - md.append( - f"- Advisory candidate tokens: ~{wr.est_candidate_tokens_total} " - f"(drop ~{wr.est_drop_candidate_tokens}, " - f"summarize ~{wr.est_summarizable_candidate_tokens}) — NOT a realized saving" - ) - md.append("") - md.append("### Router labels") - for lc in wr.label_counts: - md.append( - f"- {lc.route_label}: blocks={lc.block_count} " - f"occ={lc.occurrence_count} tokens={lc.total_est_tokens} " - f"~candidate={lc.est_candidate_tokens}" - ) - md.append("") - md.append("### Reason codes (block_type / label / reason)") - for rc in wr.reason_counts: - md.append( - f"- {rc.block_type} / {rc.route_label} / {rc.reason_code}: " - f"blocks={rc.block_count} occ={rc.occurrence_count} " - f"tokens={rc.total_est_tokens} ~candidate={rc.est_candidate_tokens}" - ) - md.append("") - md.append("### Top routable-candidate blocks (hashed)") - for cb in wr.top_candidate_blocks: - md.append( - f"- `{cb.block_hash}` type={cb.block_type} " - f"label={cb.route_label} reason={cb.reason_code} " - f"x{cb.occurrences} chars={cb.char_length} ~candidate={cb.est_candidate_tokens}" - ) - md.append("") - pa = report.parent_aggregation - md.append("## Parent Aggregation Artifacts — shadow mode") - if not pa.enabled: - md.append("- disabled") - else: - md.append( - f"- Candidate artifact items: {pa.item_count} " - f"(distinct bodies: {pa.artifact_body_count}, " - f"occurrences: {pa.total_occurrences})" - ) - md.append( - f"- Duplicate artifact groups: {pa.duplicate_group_count} " - f"(~{pa.est_duplicate_tokens} advisory duplicate tokens of " - f"~{pa.est_total_tokens} distinct-body tokens) — NOT a realized saving, " - f"payloads are unchanged" - ) - md.append("") - md.append("### By artifact kind") - for ks in pa.by_kind: - md.append( - f"- {ks.artifact_kind}: bodies={ks.group_count} " - f"occ={ks.occurrence_count} dup_groups={ks.duplicate_group_count} " - f"tokens={ks.est_tokens} ~dup={ks.est_duplicate_tokens}" - ) - md.append("") - md.append("### Provenance (artifact source types)") - for sc in pa.source_type_counts: - md.append(f"- {sc.source_type}: {sc.count}") - md.append("") - md.append("### Top duplicate artifact groups (hashed)") - for g in pa.top_duplicate_groups: - spread = ", ".join( - f"{sc.source_type}x{sc.count}" for sc in g.source_type_counts - ) - md.append( - f"- `{g.content_hash}` kind={g.artifact_kind} " - f"canonical={g.canonical_source_type} x{g.occurrences} " - f"({spread}) chars={g.char_length} ~dup={g.est_duplicate_tokens} tokens" - ) - md.append("") - md.append("## Notes") - for note in report.notes: - md.append(f"- {note}") - md_path.write_text("\n".join(md) + "\n", encoding="utf-8") - return json_path, md_path - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--state-db", type=Path, default=Path("/root/.hermes/state.db")) - parser.add_argument( - "--telemetry-file", - type=Path, - default=Path.home() / ".hermes" / "contextpilot" / "telemetry.jsonl", - help="metadata-only ContextPilot telemetry file", - ) - parser.add_argument( - "--out-dir", type=Path, default=Path.home() / "contextpilot" / "opportunities" - ) - parser.add_argument("--since-hours", type=int, default=24) - parser.add_argument( - "--all-sessions", - action="store_true", - help="ignore --since-hours; scan all non-archived sessions and active messages", - ) - parser.add_argument( - "--salt", - default="contextpilot-hermes-opportunity-v1", - help="salt for stable per-install content/session fingerprints", - ) - parser.add_argument("--date", default=dt.date.today().isoformat()) - parser.add_argument("--min-block-chars", type=int, default=DEFAULT_MIN_BLOCK_CHARS) - parser.add_argument("--min-block-repeat", type=int, default=DEFAULT_MIN_BLOCK_REPEAT) - parser.add_argument( - "--large-output-chars", type=int, default=DEFAULT_LARGE_OUTPUT_CHARS - ) - parser.add_argument("--top-n", type=int, default=DEFAULT_TOP_N) - parser.add_argument( - "--disable-worker-routing-shadow", - action="store_true", - help=( - "skip the shadow-mode Worker Context Routing classification " - "(P0 data collection; enabled by default, never prunes context)" - ), - ) - parser.add_argument( - "--disable-parent-aggregation", - action="store_true", - help=( - "skip the shadow-mode Parent Aggregation Artifact telemetry " - "(P0 telemetry only; enabled by default, never dedups/replaces context)" - ), - ) - parser.add_argument( - "--min-artifact-chars", type=int, default=DEFAULT_MIN_ARTIFACT_CHARS - ) - args = parser.parse_args() - - if not args.state_db.exists(): - raise SystemExit(f"Hermes state DB not found: {args.state_db}") - - # Harden for unattended cron use: never dump a traceback (which would echo - # the DB path / SQL); emit only the exception class name and a non-zero code. - try: - tool_messages = load_tool_messages( - args.state_db, since_hours=args.since_hours, all_sessions=args.all_sessions - ) - llm_contents = load_llm_bound_content( - args.state_db, since_hours=args.since_hours, all_sessions=args.all_sessions - ) - heavy_sessions = load_heavy_sessions( - args.state_db, - since_hours=args.since_hours, - salt=args.salt, - top_n=args.top_n, - all_sessions=args.all_sessions, - ) - total_input = total_input_tokens( - args.state_db, since_hours=args.since_hours, all_sessions=args.all_sessions - ) - telemetry = parse_telemetry( - args.telemetry_file, - since_hours=args.since_hours, - total_input_tokens=total_input, - all_sessions=args.all_sessions, - ) - report = build_report( - date=args.date, - since_hours=args.since_hours, - salt=args.salt, - tool_messages=tool_messages, - heavy_sessions=heavy_sessions, - telemetry=telemetry, - llm_contents=llm_contents, - all_sessions=args.all_sessions, - min_block_chars=args.min_block_chars, - min_block_repeat=args.min_block_repeat, - large_output_chars=args.large_output_chars, - top_n=args.top_n, - worker_routing_shadow=not args.disable_worker_routing_shadow, - parent_aggregation_shadow=not args.disable_parent_aggregation, - min_artifact_chars=args.min_artifact_chars, - ) - json_path, md_path = write_report(report, args.out_dir) - except Exception as exc: # noqa: BLE001 - cron-safe: report class only, no payload - print(json.dumps({"ok": False, "error": type(exc).__name__})) - return 1 - - print( - json.dumps( - {"ok": True, "json": str(json_path), "markdown": str(md_path)}, - ensure_ascii=False, - ) - ) - return 0 - - if __name__ == "__main__": raise SystemExit(main()) diff --git a/scripts/build_provenance_linker_dataset.py b/scripts/build_provenance_linker_dataset.py new file mode 100644 index 0000000..8ae97f3 --- /dev/null +++ b/scripts/build_provenance_linker_dataset.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Build an independent provenance-linker dataset. + +Modes: +* ``synthetic``: committed-safe toy examples with gold claim→evidence links. +* ``trace``: local raw export from an existing trace-validation JSONL corpus. + +The output schema is independent from artifact-dedup validation: it is designed +for training/shadowing a general provenance linker that maps assistant claims to +supporting evidence blocks. It does not mutate online payloads. +""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from contextpilot.provenance_linking import ( # noqa: E402 + ProvenanceBlock, + ProvenanceClaim, + ProvenanceExample, + ProvenanceLink, + example_from_trace_case, + shadow_link_claims, + to_jsonable, + write_jsonl, +) + + +def _claim(block: ProvenanceBlock, text: str, cid: str) -> ProvenanceClaim: + start = block.text.index(text) + return ProvenanceClaim(cid, block.block_id, text, start, start + len(text)) + + +def _span(block: ProvenanceBlock, text: str) -> tuple[int, int]: + start = block.text.index(text) + return start, start + len(text) + + +def synthetic_examples() -> list[ProvenanceExample]: + examples: list[ProvenanceExample] = [] + + # Coding is one example domain, not the design itself. + e1_tool = ProvenanceBlock( + "coding_tool_pytest", + "tool_result", + "pytest output\n46 passed, 2 warnings in 0.33s\nfull suite: 592 passed, 37 skipped in 19.02s\n", + {"domain": "coding", "tool": "pytest"}, + ) + e1_review = ProvenanceBlock( + "coding_worker_review", + "worker_output", + "Read-only review: PASS. No correctness blockers. Minor naming nit only.\n", + {"domain": "coding", "worker": "reviewer"}, + ) + e1_assistant = ProvenanceBlock( + "coding_parent_summary", + "assistant_context", + "Full suite passed: 592 passed, 37 skipped. Read-only review PASS, so PR can continue.", + {"domain": "coding"}, + ) + c1 = _claim(e1_assistant, "Full suite passed: 592 passed, 37 skipped.", "claim_coding_tests_passed") + c2 = _claim(e1_assistant, "Read-only review PASS, so PR can continue.", "claim_coding_pr_continue") + s1, t1 = _span(e1_tool, "592 passed, 37 skipped") + s2, t2 = _span(e1_review, "PASS. No correctness blockers") + examples.append( + ProvenanceExample( + "synthetic_coding_review", + "coding", + [e1_tool, e1_review, e1_assistant], + [c1, c2], + [ + ProvenanceLink(c1.claim_id, e1_tool.block_id, s1, t1, "supports", 1.0, "gold"), + ProvenanceLink(c2.claim_id, e1_review.block_id, s2, t2, "supports", 1.0, "gold"), + ], + ) + ) + + e2_web = ProvenanceBlock( + "research_web_source", + "web_result", + "Paper abstract: the system uses retrieval plus citation verification to reduce unsupported claims.", + {"domain": "research"}, + ) + e2_worker = ProvenanceBlock( + "research_worker_summary", + "worker_output", + "Worker finding: citation verification catches unsupported claims before final answer.", + {"domain": "research"}, + ) + e2_assistant = ProvenanceBlock( + "research_parent_summary", + "assistant_context", + "The method reduces unsupported claims by using retrieval plus citation verification.", + {"domain": "research"}, + ) + c3 = _claim(e2_assistant, e2_assistant.text, "claim_research_attribution") + s3, t3 = _span(e2_web, "retrieval plus citation verification") + examples.append( + ProvenanceExample( + "synthetic_research_claim", + "research", + [e2_web, e2_worker, e2_assistant], + [c3], + [ProvenanceLink(c3.claim_id, e2_web.block_id, s3, t3, "summarized_support", 1.0, "gold")], + ) + ) + + e3_sensor = ProvenanceBlock( + "ops_metric_source", + "tool_result", + "monitor: p95 latency = 183ms, error_rate = 0.02%, queue_depth = 4", + {"domain": "ops"}, + ) + e3_assistant = ProvenanceBlock( + "ops_summary", + "assistant_context", + "Latency is healthy: p95 latency = 183ms and error_rate = 0.02%.", + {"domain": "ops"}, + ) + c4 = _claim(e3_assistant, e3_assistant.text, "claim_ops_latency_healthy") + s4, t4 = _span(e3_sensor, "p95 latency = 183ms") + examples.append( + ProvenanceExample( + "synthetic_ops_metrics", + "ops", + [e3_sensor, e3_assistant], + [c4], + [ProvenanceLink(c4.claim_id, e3_sensor.block_id, s4, t4, "extracted_support", 1.0, "gold")], + ) + ) + return examples + + +def build_from_trace_jsonl(path: Path, *, limit: int | None) -> list[ProvenanceExample]: + examples = [] + for line in path.read_text().splitlines(): + if not line.strip(): + continue + examples.append(example_from_trace_case(json.loads(line))) + if limit is not None and len(examples) >= limit: + break + return examples + + +def manifest_for(examples: list[ProvenanceExample], *, mode: str, output: Path) -> dict: + shadow_counts = [len(shadow_link_claims(ex)) for ex in examples] + return { + "schema_version": 1, + "dataset_kind": "provenance_linking", + "mode": mode, + "output": str(output), + "example_count": len(examples), + "block_count": sum(len(ex.blocks) for ex in examples), + "claim_count": sum(len(ex.claims) for ex in examples), + "gold_link_count": sum(len(ex.gold_links) for ex in examples), + "shadow_link_count": sum(shadow_counts), + "privacy_note": "synthetic is commit-safe; trace mode may contain raw local content and must stay local/gitignored", + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--mode", choices=["synthetic", "trace"], default="synthetic") + parser.add_argument("--trace-jsonl", type=Path, help="trace-validation JSONL for trace mode") + parser.add_argument("--limit", type=int, default=None) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--with-shadow", action="store_true", help="include shadow_links alongside gold/raw examples") + parser.add_argument( + "--allow-unsafe-trace-output", + action="store_true", + help="allow trace-mode raw output outside ~/contextpilot/provenance_datasets (not recommended)", + ) + args = parser.parse_args() + + if args.mode == "synthetic": + examples = synthetic_examples() + else: + safe_root = (Path.home() / "contextpilot" / "provenance_datasets").resolve() + out_resolved = args.output.resolve() + if not args.allow_unsafe_trace_output and safe_root not in [out_resolved, *out_resolved.parents]: + raise SystemExit( + "trace mode writes raw content; use an output under " + f"{safe_root} or pass --allow-unsafe-trace-output explicitly" + ) + if args.trace_jsonl is None: + raise SystemExit("--trace-jsonl is required for --mode trace") + examples = build_from_trace_jsonl(args.trace_jsonl, limit=args.limit) + + rows = [to_jsonable(ex, shadow_links=shadow_link_claims(ex) if args.with_shadow else None) for ex in examples] + write_jsonl(args.output, rows) + manifest = manifest_for(examples, mode=args.mode, output=args.output) + manifest_path = args.output.with_suffix(args.output.suffix + ".manifest.json") + manifest_path.write_text(json.dumps(manifest, indent=2, ensure_ascii=False) + "\n") + print(json.dumps({"ok": True, **manifest, "manifest": str(manifest_path)}, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/build_trace_validation_set.py b/scripts/build_trace_validation_set.py new file mode 100644 index 0000000..bb8fb97 --- /dev/null +++ b/scripts/build_trace_validation_set.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Build a trace-derived ContextPilot validation set from a local Hermes DB. + +Thin wrapper around :mod:`contextpilot.trace_validation.builder`. Reads the +Hermes state DB read-only and writes a FIXED JSONL corpus (raw content, +local-only / gitignored) plus a privacy-safe manifest sidecar. + +Examples:: + + # last 24h, conservative sampling, default local output dir + python scripts/build_trace_validation_set.py + + # heavier sessions only, all history, to a chosen private dir + python scripts/build_trace_validation_set.py --all-sessions \\ + --min-input-tokens 20000 --limit 50 --out .contextpilot_validation + + # exclude system/skill prompts from the corpus + python scripts/build_trace_validation_set.py --no-system-prompt +""" +from __future__ import annotations + +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[1] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from contextpilot.trace_validation.builder import main # noqa: E402 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/contextpilot_savings.py b/scripts/contextpilot_savings.py index 2403d38..2a3f485 100644 --- a/scripts/contextpilot_savings.py +++ b/scripts/contextpilot_savings.py @@ -51,8 +51,15 @@ def summarize_telemetry( "window_start_iso": None, "events": 0, "chars_saved": 0, - "tokens_saved": 0, - "avg_tokens_per_event": None, + # Token savings are tokenizer-measured only. Legacy ``tokens_saved`` + # fields from older telemetry are ignored rather than treated as char/4 + # proxies. + "actual_token_status": "unavailable", + "actual_token_events": 0, + "actual_tokens_before": 0, + "actual_tokens_after": 0, + "actual_tokens_saved": 0, + "actual_tokenizer_backends": [], "skipped_lines": 0, } @@ -69,8 +76,12 @@ def summarize_telemetry( events = 0 chars = 0 - tokens = 0 skipped = 0 + actual_events = 0 + actual_before = 0 + actual_after = 0 + actual_saved = 0 + actual_backends: set[str] = set() with telemetry_path.open("r", encoding="utf-8", errors="replace") as f: for line in f: line = line.strip() @@ -95,24 +106,37 @@ def summarize_telemetry( if not isinstance(cs, (int, float)) or cs < 0: skipped += 1 continue - saved_tokens = record.get("tokens_saved") - if isinstance(saved_tokens, (int, float)) and saved_tokens < 0: - skipped += 1 - continue events += 1 chars += int(cs) - tokens += ( - int(saved_tokens) - if isinstance(saved_tokens, (int, float)) - else int(cs) // 4 - ) + + # Tokenizer measurement, only when the writer marked it as + # available. Anything else (missing/unavailable) is left out -- we + # never substitute the chars/4 estimate into the actual-token totals. + if record.get("actual_token_status") == "available": + ats = record.get("actual_tokens_saved") + if isinstance(ats, (int, float)): + actual_events += 1 + actual_saved += int(ats) + atb = record.get("actual_tokens_before") + if isinstance(atb, (int, float)): + actual_before += int(atb) + ata = record.get("actual_tokens_after") + if isinstance(ata, (int, float)): + actual_after += int(ata) + backend = record.get("actual_tokenizer_backend") + if isinstance(backend, str) and backend: + actual_backends.add(backend) result["events"] = events result["chars_saved"] = chars - result["tokens_saved"] = tokens result["skipped_lines"] = skipped - if events > 0: - result["avg_tokens_per_event"] = round(tokens / events, 1) + if actual_events > 0: + result["actual_token_status"] = "available" + result["actual_token_events"] = actual_events + result["actual_tokens_before"] = actual_before + result["actual_tokens_after"] = actual_after + result["actual_tokens_saved"] = actual_saved + result["actual_tokenizer_backends"] = sorted(actual_backends) return result @@ -154,16 +178,27 @@ def render_text(summary: Dict[str, Any]) -> str: ) lines = [ - f"ContextPilot token savings ({window})", - f" Events: {summary['events']}", - f" Chars saved: {summary['chars_saved']:,}", - f" Estimated tokens saved: ~{summary['tokens_saved']:,}", + f"ContextPilot savings ({window})", + f" Events: {summary['events']}", + f" Chars saved: {summary['chars_saved']:,}", ] - if summary["avg_tokens_per_event"] is not None: + # Token savings are shown ONLY when telemetry recorded tokenizer counts. + # Legacy derived fields are ignored; no chars/4 fallback is displayed. + if summary["actual_token_status"] == "available": + backends = ", ".join(summary["actual_tokenizer_backends"]) or "unknown" + lines.append( + f" Actual tokens saved (tokenizer): {summary['actual_tokens_saved']:,}" + ) + lines.append( + f" backend: {backends} | status: available | " + f"events: {summary['actual_token_events']}" + ) + else: lines.append( - f" Avg tokens/event: ~{summary['avg_tokens_per_event']:,}" + " Actual tokens saved (tokenizer): unavailable " + "(no exact tokenizer backend recorded)" ) - lines.append(f" Telemetry file: {path}") + lines.append(f" Telemetry file: {path}") if summary["skipped_lines"]: lines.append( f" (skipped {summary['skipped_lines']} malformed telemetry line(s))" @@ -173,7 +208,10 @@ def render_text(summary: Dict[str, Any]) -> str: def main(argv: list[str] | None = None) -> int: parser = argparse.ArgumentParser( - description="Show how many tokens ContextPilot saved (metadata-only).", + description=( + "Show ContextPilot processed-payload savings (metadata-only); " + "exact tokenizer tokens are shown only when telemetry recorded them." + ), ) parser.add_argument( "--telemetry-file", diff --git a/scripts/evaluate_artifact_precision.py b/scripts/evaluate_artifact_precision.py new file mode 100644 index 0000000..be68213 --- /dev/null +++ b/scripts/evaluate_artifact_precision.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +"""Run a reproducible synthetic precision/recall check for artifact canary rewrites. + +This is intentionally *not* a product/model precision benchmark. It is a small, +hand-labeled synthetic self-consistency suite that checks the current exact +whole-body, fenced-block, and declared source-span rewrite gates against planted +positive/negative cases. Top-level metric names are prefixed with ``synthetic`` +so they are not confused with field precision on real traces. +""" +from __future__ import annotations + +import argparse +import json +import os +import pathlib +import sys +import time +from dataclasses import dataclass +from typing import Iterable + +REPO_ROOT = pathlib.Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from contextpilot.hermes_opportunities.artifact_dedup_canary import ( + ARTIFACT_DEDUP_DISABLE_ENV, + ARTIFACT_DEDUP_MODE_ENV, + ARTIFACT_DEDUP_CANARY_REFERENCE_TEMPLATE, + ArtifactSpanLink, + apply_artifact_dedup_canary, + dangling_artifact_references, +) +from contextpilot.hermes_opportunities.models import _LLMContent + +SALT = "precision-eval-salt" +MIN = 40 +ART = ("Synthetic artifact payload alpha bravo charlie delta echo foxtrot. " * 6).strip() +SHORT = "Short duplicate body just above forty chars." +FENCE = ( + "```log\n" + + ("synthetic repeated fenced worker output alpha bravo charlie\n" * 8).rstrip("\n") + + "\n```" +) +SPAN = ("declared source span line alpha bravo charlie\n" * 8).rstrip("\n") + + +@dataclass(frozen=True) +class LabeledCase: + name: str + expected_replacements: int + items: list[_LLMContent] + span_links: list[ArtifactSpanLink] + + +def _content(block_type: str, text: str) -> _LLMContent: + return _LLMContent(block_type, text) + + +def _clone(items: Iterable[_LLMContent]) -> list[_LLMContent]: + return [_LLMContent(item.block_type, item.content) for item in items] + + +def _span_case(*, exact: bool = True, forward: bool = False, oob: bool = False) -> tuple[list[_LLMContent], list[ArtifactSpanLink]]: + src = "tool pre\n" + SPAN + "\ntool post\n" + target_span = SPAN if exact else SPAN.replace("charlie", "changed", 1) + tgt = "parent pre\n" + target_span + "\nparent post\n" + source_start = src.index(SPAN) + source_end = source_start + len(SPAN) + target_start = tgt.index(target_span) + target_end = target_start + len(target_span) + items = [_content("tool_result", src), _content("assistant_context", tgt)] + if forward: + link = ArtifactSpanLink(1, target_start, target_end, 0, source_start, source_end) + elif oob: + link = ArtifactSpanLink(0, source_start, len(src) + 100, 1, target_start, target_end) + else: + link = ArtifactSpanLink(0, source_start, source_end, 1, target_start, target_end) + return items, [link] + + +def build_cases() -> list[LabeledCase]: + cases: list[LabeledCase] = [ + LabeledCase("whole_tool_exact_duplicate", 1, [_content("tool_result", ART), _content("tool_result", ART)], []), + LabeledCase("whole_cross_type_exact_duplicate", 1, [_content("tool_result", ART), _content("assistant_context", ART)], []), + LabeledCase("whole_near_duplicate_not_mutated", 0, [_content("tool_result", ART), _content("tool_result", ART + " changed")], []), + LabeledCase("fenced_internal_duplicate", 1, [_content("assistant_context", "before\n" + FENCE + "\nmiddle\n" + FENCE + "\nafter")], []), + LabeledCase("two_fenced_duplicates", 2, [_content("assistant_context", "a\n" + FENCE + "\nb\n" + FENCE + "\nc\n" + FENCE + "\nd")], []), + LabeledCase("protected_user_system_duplicates", 0, [_content("user_ctx", ART), _content("system_ctx", ART)], []), + LabeledCase("short_duplicate_never_grow", 0, [_content("tool_result", SHORT), _content("tool_result", SHORT)], []), + LabeledCase("unterminated_fence_not_mutated", 0, [_content("assistant_context", ("prefix\n```log\n" + ("unterminated line alpha bravo charlie\n" * 8)) * 2)], []), + LabeledCase("copied_plain_span_without_declared_link", 0, [_content("tool_result", "tool\n" + SPAN + "\nend"), _content("assistant_context", "parent\n" + SPAN + "\nend")], []), + LabeledCase("protected_duplicate_tool_vs_user", 0, [_content("tool_result", ART), _content("user_ctx", ART)], []), + LabeledCase("protected_duplicate_tool_vs_system", 0, [_content("tool_result", ART), _content("system_ctx", ART)], []), + ] + items, links = _span_case(exact=True) + cases.append(LabeledCase("declared_source_span_exact", 1, items, links)) + items, links = _span_case(exact=False) + cases.append(LabeledCase("declared_span_content_differs", 0, items, links)) + items, links = _span_case(exact=True, forward=True) + cases.append(LabeledCase("forward_span_link_rejected", 0, items, links)) + items, links = _span_case(exact=True, oob=True) + cases.append(LabeledCase("oob_span_link_rejected", 0, items, links)) + items, links = _span_case(exact=True) + # Duplicate declaration for the same target is deduplicated to one event. + cases.append(LabeledCase("duplicate_span_declaration_counts_once", 1, items, links + links)) + return cases + + +def _mode_gate_checks() -> dict[str, bool]: + base = [_content("tool_result", ART), _content("tool_result", ART)] + off_items = _clone(base) + off = apply_artifact_dedup_canary(off_items, salt=SALT, min_block_chars=MIN, mode="off") + shadow_items = _clone(base) + shadow = apply_artifact_dedup_canary(shadow_items, salt=SALT, min_block_chars=MIN, mode="shadow") + disable_items = _clone(base) + old_mode = os.environ.get(ARTIFACT_DEDUP_MODE_ENV) + old_disable = os.environ.get(ARTIFACT_DEDUP_DISABLE_ENV) + try: + os.environ[ARTIFACT_DEDUP_MODE_ENV] = "canary" + os.environ[ARTIFACT_DEDUP_DISABLE_ENV] = "1" + disabled = apply_artifact_dedup_canary(disable_items, salt=SALT, min_block_chars=MIN) + finally: + if old_mode is None: + os.environ.pop(ARTIFACT_DEDUP_MODE_ENV, None) + else: + os.environ[ARTIFACT_DEDUP_MODE_ENV] = old_mode + if old_disable is None: + os.environ.pop(ARTIFACT_DEDUP_DISABLE_ENV, None) + else: + os.environ[ARTIFACT_DEDUP_DISABLE_ENV] = old_disable + return { + "off_no_mutation": not off.mutated and [i.content for i in off_items] == [i.content for i in base], + "shadow_no_mutation": not shadow.mutated and [i.content for i in shadow_items] == [i.content for i in base], + "disable_env_no_mutation": not disabled.mutated and [i.content for i in disable_items] == [i.content for i in base], + } + + +def _validation_gate_checks() -> dict[str, bool]: + forged_ref = ARTIFACT_DEDUP_CANARY_REFERENCE_TEMPLATE.replace("", "tool_result").replace("", "deadbeef") + forged = [_content("assistant_context", forged_ref)] + return { + "forged_reference_detected": dangling_artifact_references(forged, salt=SALT) == [0], + } + + +def build_report() -> dict: + rows = [] + tp = fp = fn = tn = 0 + expected_total = predicted_total = chars_saved = 0 + for case in build_cases(): + items = _clone(case.items) + result = apply_artifact_dedup_canary( + items, + salt=SALT, + min_block_chars=MIN, + mode="canary", + span_links=case.span_links, + ) + actual = result.blocks_replaced + expected = case.expected_replacements + dangling = dangling_artifact_references(items, salt=SALT, span_links=case.span_links) + case_pass = actual == expected and not dangling + this_tp = min(actual, expected) + this_fp = max(0, actual - expected) + this_fn = max(0, expected - actual) + this_tn = 1 if actual == 0 and expected == 0 else 0 + tp += this_tp + fp += this_fp + fn += this_fn + tn += this_tn + expected_total += expected + predicted_total += actual + chars_saved += result.chars_saved + rows.append( + { + "name": case.name, + "expected_replacements": expected, + "actual_replacements": actual, + "pass": case_pass, + "chars_saved": result.chars_saved, + "span_replacements": result.span_blocks_replaced, + "dangling": dangling, + } + ) + + return { + "schema_version": 1, + "generated_at": time.strftime("%Y-%m-%dT%H:%M:%S%z"), + "corpus": "synthetic_labeled_artifact_precision_v1", + "claim_scope": "synthetic exact/provenance gate self-consistency; not field/model/product precision", + "case_count": len(rows), + "synthetic_event_tp": tp, + "synthetic_event_fp": fp, + "synthetic_event_fn": fn, + "synthetic_negative_case_tn": tn, + "synthetic_negative_case_fpr": fp / (fp + tn) if fp + tn else 0.0, + "synthetic_event_precision": tp / (tp + fp) if tp + fp else 1.0, + "synthetic_event_recall": tp / (tp + fn) if tp + fn else 1.0, + "synthetic_case_accuracy": sum(1 for row in rows if row["pass"]) / len(rows), + "predicted_replacements": predicted_total, + "expected_replacements": expected_total, + "synthetic_realized_chars_saved": chars_saved, + "mode_gate_checks": _mode_gate_checks(), + "validation_gate_checks": _validation_gate_checks(), + "rows": rows, + } + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output", type=pathlib.Path, help="Optional JSON output path") + args = parser.parse_args() + report = build_report() + text = json.dumps(report, indent=2, ensure_ascii=False) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text + "\n") + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/evaluate_provenance_shadow.py b/scripts/evaluate_provenance_shadow.py new file mode 100644 index 0000000..3dff800 --- /dev/null +++ b/scripts/evaluate_provenance_shadow.py @@ -0,0 +1,32 @@ +#!/usr/bin/env python3 +"""Evaluate the shadow provenance linker on a provenance-linking JSONL dataset.""" +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +from contextpilot.provenance_linking import evaluate_shadow, read_jsonl_examples # noqa: E402 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("dataset", type=Path) + parser.add_argument("--output", type=Path) + args = parser.parse_args() + report = evaluate_shadow(read_jsonl_examples(args.dataset)) + text = json.dumps(report, indent=2, ensure_ascii=False) + if args.output: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(text + "\n") + print(text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/hermes_contextpilot_monitor.py b/scripts/hermes_contextpilot_monitor.py index 3bd9483..24357ee 100644 --- a/scripts/hermes_contextpilot_monitor.py +++ b/scripts/hermes_contextpilot_monitor.py @@ -18,11 +18,11 @@ from typing import Iterable SAVINGS_RE = re.compile( - r"\[ContextPilot\].*?saved\s+(?P\d+)\s+chars\s+\(~(?P\d+)\s+tokens\)" + r"\[ContextPilot\].*?saved\s+(?P\d+)\s+chars" ) SESSION_RE = re.compile( r"\[ContextPilot\]\s+Session\s+(?P[^:]+):\s+(?P\d+)\s+turns,\s+" - r"(?P\d+)\s+chars\s+saved\s+\(~(?P\d+)\s+tokens\)" + r"(?P\d+)\s+chars\s+saved" ) FORBIDDEN_COLUMNS = { @@ -71,6 +71,7 @@ class DailyReport: contextpilot_log_events: int contextpilot_telemetry_events: int contextpilot_savings_source: str + contextpilot_token_status: str contextpilot_chars_saved: int contextpilot_tokens_saved: int estimated_input_token_reduction_pct: float @@ -191,24 +192,30 @@ def parse_contextpilot_savings(log_path: Path, *, since_hours: int) -> tuple[int continue events += 1 chars += int(m.group("chars")) - tokens += int(m.group("tokens")) + # Gateway log lines are char-only for token accounting. Older logs used + # to include a ``(~N tokens)`` suffix derived from chars/4; daily reports + # must not consume that estimate. Token savings come only from + # tokenizer-measured telemetry (``actual_tokens_saved``). return events, chars, tokens -def parse_contextpilot_telemetry(telemetry_path: Path, *, since_hours: int) -> tuple[int, int, int]: +def parse_contextpilot_telemetry(telemetry_path: Path, *, since_hours: int) -> tuple[int, int, int, int]: """Aggregate the plugin's metadata-only telemetry file. - Returns (events, chars_saved, tokens_saved). The file is JSON-lines, one - numeric record per saved turn; it never contains message content, prompts, - or tool payloads, so we only read numeric counters here. + Returns (events, chars_saved, actual_tokens_saved, actual_token_events). + The file is JSON-lines, one numeric record per saved turn; it never contains + message content, prompts, or tool payloads, so we only read numeric counters + here. Token savings are counted only from tokenizer-measured + ``actual_tokens_saved`` records. """ if not telemetry_path or not telemetry_path.exists(): - return 0, 0, 0 + return 0, 0, 0, 0 cutoff = dt.datetime.now(dt.timezone.utc).timestamp() - since_hours * 3600 events = 0 chars = 0 tokens = 0 + actual_token_events = 0 with telemetry_path.open("r", encoding="utf-8", errors="replace") as f: for line in f: line = line.strip() @@ -226,11 +233,14 @@ def parse_contextpilot_telemetry(telemetry_path: Path, *, since_hours: int) -> t cs = record.get("chars_saved") if not isinstance(cs, (int, float)): continue - saved_tokens = record.get("tokens_saved") events += 1 chars += int(cs) - tokens += int(saved_tokens) if isinstance(saved_tokens, (int, float)) else int(cs) // 4 - return events, chars, tokens + if record.get("actual_token_status") == "available": + saved = record.get("actual_tokens_saved") + if isinstance(saved, (int, float)): + tokens += int(saved) + actual_token_events += 1 + return events, chars, tokens, actual_token_events def build_report( @@ -239,7 +249,7 @@ def build_report( date: str, since_hours: int, log_stats: tuple[int, int, int], - telemetry_stats: tuple[int, int, int] = (0, 0, 0), + telemetry_stats: tuple[int, int, int, int] = (0, 0, 0, 0), ) -> DailyReport: rows = list(metrics) source_counts: dict[str, int] = {} @@ -247,26 +257,34 @@ def build_report( source_counts[row.source or "unknown"] = source_counts.get(row.source or "unknown", 0) + 1 total_input = sum(r.input_tokens for r in rows) - log_events, log_chars, log_tokens = log_stats - tel_events, tel_chars, tel_tokens = telemetry_stats + log_events, log_chars, _log_tokens_ignored = log_stats + tel_events, tel_chars, tel_tokens, tel_token_events = telemetry_stats # Prefer the local telemetry file when present: it is the authoritative, - # log-independent source. Logs are a fallback and are NOT summed on top - # (both record the same turns, so summing would double-count). + # log-independent source. Logs are a fallback for char savings only; token + # savings must be tokenizer-measured and are never derived from log chars. if tel_events > 0: - events, saved_chars, saved_tokens = tel_events, tel_chars, tel_tokens + events, saved_chars = tel_events, tel_chars savings_source = "telemetry" else: - events, saved_chars, saved_tokens = log_events, log_chars, log_tokens - savings_source = "gateway-log" + events, saved_chars = log_events, log_chars + savings_source = "gateway-log-chars-only" + + if tel_token_events > 0: + saved_tokens = tel_tokens + token_status = "available" + else: + saved_tokens = 0 + token_status = "unavailable" denominator = total_input + saved_tokens - reduction = (saved_tokens / denominator * 100.0) if denominator else 0.0 + reduction = (saved_tokens / denominator * 100.0) if token_status == "available" and denominator else 0.0 notes: list[str] = [ "metadata-only: did not read messages.content, sessions.system_prompt, reasoning, or tool payloads", "accuracy gate is observational here; apply code/config changes only after separate golden-eval pass", f"contextpilot savings source: {savings_source} (telemetry={tel_events} events, log={log_events} events)", + f"contextpilot token savings status: {token_status} (tokenizer-measured events={tel_token_events})", ] if not rows: notes.append("no sessions observed in the selected window") @@ -292,6 +310,7 @@ def build_report( contextpilot_log_events=log_events, contextpilot_telemetry_events=tel_events, contextpilot_savings_source=savings_source, + contextpilot_token_status=token_status, contextpilot_chars_saved=saved_chars, contextpilot_tokens_saved=saved_tokens, estimated_input_token_reduction_pct=round(reduction, 2), @@ -308,6 +327,22 @@ def write_report(report: DailyReport, out_dir: Path) -> tuple[Path, Path]: data = asdict(report) json_path.write_text(json.dumps(data, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + if report.contextpilot_token_status == "available": + token_saved_line = ( + f"- ContextPilot saved tokens (tokenizer): {report.contextpilot_tokens_saved} " + f"tokens ({report.contextpilot_chars_saved} chars metadata)" + ) + reduction_line = ( + f"- Input-token reduction from tokenizer telemetry: " + f"{report.estimated_input_token_reduction_pct}%" + ) + else: + token_saved_line = ( + f"- ContextPilot saved tokens (tokenizer): unavailable " + f"({report.contextpilot_chars_saved} chars metadata only)" + ) + reduction_line = "- Input-token reduction from tokenizer telemetry: unavailable" + md = [ f"# ContextPilot Hermes monitor — {report.date}", "", @@ -318,11 +353,12 @@ def write_report(report: DailyReport, out_dir: Path) -> tuple[Path, Path]: f"- Input tokens: {report.total_input_tokens}", f"- Output tokens: {report.total_output_tokens}", f"- Tool calls: {report.total_tool_calls}", - f"- ContextPilot saved: ~{report.contextpilot_tokens_saved} tokens ({report.contextpilot_chars_saved} chars)", + token_saved_line, f"- ContextPilot savings source: {report.contextpilot_savings_source} " - f"(telemetry events={report.contextpilot_telemetry_events}, log events={report.contextpilot_log_events})", - f"- Estimated input-token reduction: {report.estimated_input_token_reduction_pct}%", - f"- Estimated cost: ${report.estimated_cost_usd:.4f}", + f"(telemetry events={report.contextpilot_telemetry_events}, log events={report.contextpilot_log_events}, " + f"token status={report.contextpilot_token_status})", + reduction_line, + f"- Session cost field total: ${report.estimated_cost_usd:.4f}", "", "## Top sources", ] diff --git a/scripts/run_trace_validation.py b/scripts/run_trace_validation.py new file mode 100644 index 0000000..910d39d --- /dev/null +++ b/scripts/run_trace_validation.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Run ContextPilot trace validation against a fixed local corpus. + +Thin wrapper around :mod:`contextpilot.trace_validation.runner`. Replays the +JSONL corpus through ContextPilot's optimization in a baseline (``off``) mode vs +a configured candidate mode, checks accuracy-preservation invariants, prints a +privacy-safe JSON/Markdown summary, and exits non-zero on any gate failure. + +Examples:: + + # validate the canary candidate against a corpus, JSON gate report + python scripts/run_trace_validation.py \\ + ~/contextpilot/validation_sets/validation_set_2026-06-14.jsonl \\ + --candidate-mode canary + + # use the resolved CONTEXTPILOT_PROMPT_DEDUP_MODE env, Markdown output + CONTEXTPILOT_PROMPT_DEDUP_MODE=canary \\ + python scripts/run_trace_validation.py --format markdown + + # with exact-token accounting + python scripts/run_trace_validation.py \\ + --candidate-mode canary --tokenizer tiktoken:cl100k_base +""" +from __future__ import annotations + +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[1] +if str(_REPO_ROOT) not in sys.path: + sys.path.insert(0, str(_REPO_ROOT)) + +from contextpilot.trace_validation.runner import main # noqa: E402 + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/contextpilot-savings/SKILL.md b/skills/contextpilot-savings/SKILL.md index bd8211b..5579390 100644 --- a/skills/contextpilot-savings/SKILL.md +++ b/skills/contextpilot-savings/SKILL.md @@ -93,8 +93,8 @@ Vary the command to match the request: (Combine with `--all-time` or `--since-hours N` as needed.) -The script prints events, chars saved, estimated tokens saved, the time window, -and average tokens per event. It imports no Hermes internals. +The script prints events, chars saved, telemetry tokens saved, the time window, +and average tokens per event. ### Step 3 — Summarize in plain language diff --git a/tests/fixtures/trace_validation/synthetic_cases.jsonl b/tests/fixtures/trace_validation/synthetic_cases.jsonl new file mode 100644 index 0000000..0cb10a0 --- /dev/null +++ b/tests/fixtures/trace_validation/synthetic_cases.jsonl @@ -0,0 +1,3 @@ +{"schema_version": 1, "case_id": "synth00000000aaaa", "source": "synthetic", "input_tokens": 4096, "message_count": 4, "messages": [{"role": "system", "block_type": "skill_prompt", "content": "Synthetic reusable skill paragraph that explains how the demo helper reformats sample markdown tables into neat aligned columns for readers.\nSynthetic reusable skill paragraph that explains how the demo helper reformats sample markdown tables into neat aligned columns for readers.\nSynthetic reusable skill paragraph that explains how the demo helper reformats sample markdown tables into neat aligned columns for readers."}, {"role": "user", "block_type": "user_prompt", "content": "Please summarize the synthetic quarterly figures attached above for the demo."}, {"role": "assistant", "block_type": "assistant_context", "content": "Here is the synthetic summary of the demo quarterly figures as requested."}, {"role": "tool", "block_type": "tool_result", "content": "{\"synthetic_metric\": 42, \"label\": \"demo only, not real data\"}"}]} +{"schema_version": 1, "case_id": "synth00000000bbbb", "source": "synthetic", "input_tokens": 2048, "message_count": 4, "messages": [{"role": "system", "block_type": "system_prompt", "content": "Synthetic system narration paragraph describing the demo assistant persona and the general tone it adopts."}, {"role": "user", "block_type": "user_prompt", "content": "What is the synthetic weather in the demo city today?"}, {"role": "assistant", "block_type": "assistant_context", "content": "The synthetic demo weather report indicates clear skies for the example city."}, {"role": "tool", "block_type": "tool_result", "content": "synthetic_weather=clear; demo_temperature=21C; note=fabricated_for_tests"}]} +{"schema_version": 1, "case_id": "synth00000000cccc", "source": "synthetic", "input_tokens": 1024, "message_count": 2, "messages": [{"role": "system", "block_type": "skill_prompt", "content": "You must double check the synthetic example output before sharing it with the demo readers in this flow.\nYou must double check the synthetic example output before sharing it with the demo readers in this flow.\nYou must double check the synthetic example output before sharing it with the demo readers in this flow."}, {"role": "user", "block_type": "user_prompt", "content": "Run the synthetic demo check now please."}]} diff --git a/tests/test_analyzer_lightweight_provenance.py b/tests/test_analyzer_lightweight_provenance.py new file mode 100644 index 0000000..296bda9 --- /dev/null +++ b/tests/test_analyzer_lightweight_provenance.py @@ -0,0 +1,130 @@ +"""RED-phase tests for the lightweight provenance profiler / token monitor. + +These tests pin two requirements for using ContextPilot purely as a standalone +token monitor / profiler against a Hermes state DB: + +1. ``scripts/analyze_hermes_context_opportunities.py`` must import and expose its + public API even when SciPy is not installed. SciPy is a heavy, optional + ContextPilot dependency that ``contextpilot/__init__`` pulls in transitively + (``contextpilot.pipeline`` -> ``contextpilot.context_index`` -> + ``scipy.cluster.hierarchy``). The analyzer only reads Hermes' SQLite state + DB and never needs the RAG pipeline, so a missing SciPy must not break it. + This reproduces the real ``ModuleNotFoundError: No module named 'scipy'`` + observed when running the analyzer inside the Hermes venv. + +2. The analyzer must offer a privacy-safe provenance profile that aggregates + token usage per source (the token-monitor view) using numeric aggregates and + low-cardinality source enums only -- never raw content, prompts, reasoning, + or raw session ids/hashes. + +Both tests fail today (RED): importing the script triggers +``contextpilot/__init__``, which eagerly imports ``contextpilot.pipeline`` and +hence SciPy, so the script cannot even load when SciPy is absent. +""" +from __future__ import annotations + +import dataclasses +import importlib.util +import sys +from pathlib import Path + +MODULE_PATH = ( + Path(__file__).resolve().parents[1] + / "scripts" + / "analyze_hermes_context_opportunities.py" +) + + +class _BlockScipyFinder: + """Meta-path finder that makes ``scipy`` look uninstalled. + + Raising ``ModuleNotFoundError`` from ``find_spec`` reproduces exactly what + the Hermes venv does at import time, regardless of whether SciPy happens to + be installed in the test environment. + """ + + def find_spec(self, fullname, path=None, target=None): + if fullname == "scipy" or fullname.startswith("scipy."): + raise ModuleNotFoundError( + f"No module named 'scipy' (blocked for test: {fullname})" + ) + return None + + +def _purge(prefixes): + for name in list(sys.modules): + if any(name == p or name.startswith(p + ".") for p in prefixes): + del sys.modules[name] + + +_PURGE_PREFIXES = ["scipy", "contextpilot", "analyze_hermes_context_opportunities"] + + +def _load_analyzer_without_scipy(): + """Load the analyzer script by file path with ``scipy`` forced absent.""" + finder = _BlockScipyFinder() + saved_modules = { + name: module + for name, module in sys.modules.items() + if any(name == p or name.startswith(p + ".") for p in _PURGE_PREFIXES) + } + sys.meta_path.insert(0, finder) + _purge(_PURGE_PREFIXES) + try: + spec = importlib.util.spec_from_file_location( + "analyze_hermes_context_opportunities", MODULE_PATH + ) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + finally: + try: + sys.meta_path.remove(finder) + except ValueError: + pass + # Restore the import cache exactly as it was so this import-isolation + # test cannot perturb later multiprocessing/pickling tests that depend + # on module object identity. + _purge(_PURGE_PREFIXES) + sys.modules.update(saved_modules) + + +def test_analyzer_imports_without_scipy(): + module = _load_analyzer_without_scipy() + # The token-monitor entry points must all be reachable without SciPy. + assert callable(module.main) + assert callable(module.load_tool_messages) + assert callable(module.load_heavy_sessions) + assert callable(module.build_report) + + +def test_provenance_profile_is_privacy_safe(): + module = _load_analyzer_without_scipy() + build_provenance_profile = getattr(module, "build_provenance_profile", None) + assert callable(build_provenance_profile), ( + "analyzer must expose build_provenance_profile() for the token-monitor view" + ) + HeavySession = module.HeavySession + sessions = [ + HeavySession("hash-a", "discord", 1000, 200, 6, 4, 3), + HeavySession("hash-b", "discord", 500, 100, 4, 2, 2), + HeavySession("hash-c", "slack", 300, 50, 3, 1, 1), + ] + profile = build_provenance_profile(sessions) + + by_source = {e.source: e for e in profile.by_source} + assert set(by_source) == {"discord", "slack"} + assert by_source["discord"].input_tokens == 1500 + assert by_source["discord"].output_tokens == 300 + assert by_source["discord"].session_count == 2 + assert by_source["slack"].input_tokens == 300 + assert by_source["slack"].session_count == 1 + + # Provenance output is numeric aggregates + low-cardinality source enums only; + # no raw content/prompts/reasoning and no raw session ids/hashes may leak. + data = dataclasses.asdict(profile) + module._assert_no_forbidden_keys(data) + blob = repr(data) + for raw_hash in ("hash-a", "hash-b", "hash-c"): + assert raw_hash not in blob diff --git a/tests/test_artifact_dedup_canary.py b/tests/test_artifact_dedup_canary.py new file mode 100644 index 0000000..a9b4792 --- /dev/null +++ b/tests/test_artifact_dedup_canary.py @@ -0,0 +1,897 @@ +"""RED-phase tests for the provenance-aware tool-artifact reuse canary. + +This canary is the second (and, like the prompt-dedup canary, default-OFF) +runtime mutation path. Where the prompt-dedup canary rewrites *skill_prompt* +lines, this one dedups whole **artifact bodies** carried by ``tool_result`` and +``assistant_context`` items: it keeps the FIRST full artifact body verbatim and +replaces a later EXACT duplicate body (regardless of which of the two artifact +block types it appears in -- provenance-aware) with a deterministic, strictly +shorter reference string that points back at the canonical body via a salted +hash. + +The safety contract these tests pin: + +1. ``off`` (default) and ``shadow`` never mutate the payload; only ``canary`` + replaces a later exact-duplicate artifact body, always keeping the first one. +2. Only ``tool_result`` / ``assistant_context`` artifact bodies are mutable; + ``system_prompt`` / ``user_prompt`` / ``skill_prompt`` (and any other + non-artifact content) are protected and survive byte-identical. +3. A reference is valid only if it resolves to an EARLIER canonical full body in + the same payload; the runner/validation gate fails a dangling reference or a + protected-content mutation. +4. Telemetry / the runner report carry REALIZED ``chars_saved`` only when an + actual mutation happened, and never any raw artifact content. + +Production is implemented in ``contextpilot.hermes_opportunities.artifact_dedup_canary``. +Fixtures are synthetic only. +""" +import json + +from contextpilot.hermes_opportunities.artifact_dedup_canary import ( + ARTIFACT_DEDUP_CANARY_REFERENCE_TEMPLATE, + ARTIFACT_DEDUP_CLASS, + ARTIFACT_DEDUP_DISABLE_ENV, + ARTIFACT_DEDUP_MODE_ENV, + MUTABLE_ARTIFACT_BLOCK_TYPES, + ArtifactDedupCanaryResult, + ArtifactSpanLink, + apply_artifact_dedup_canary, + build_artifact_canary_telemetry_record, + dangling_artifact_references, + resolve_artifact_dedup_mode, + _artifact_reference_string, + _segment_fenced_blocks, +) +from contextpilot.hermes_opportunities.models import _LLMContent +from contextpilot.hermes_opportunities.privacy import _salted_hash +from contextpilot.trace_validation.runner import ( + ARTIFACT_INVARIANT_NAMES, + assert_report_privacy_safe, + render_markdown, + report_to_dict, + run_artifact_validation, +) + +SALT = "test-artifact-salt" +MIN = 40 + +# A synthetic artifact body comfortably longer than the reference placeholder so +# a replacement actually shrinks the payload. Free of secrets/raw user text. +LONG_ARTIFACT = ( + "Synthetic tool artifact body: rows=128 cols=8 checksum=alpha-bravo-charlie " + "delta-echo. Summary of the synthetic computation produced purely for this " + "test fixture and nothing else, padded to comfortably exceed the reference." +) +# A non-artifact protected block (system narration); duplicated to prove the +# canary never touches it. +SYS_BLOCK = ( + "Synthetic system narration describing the assistant persona and the general " + "tone it should adopt across replies in this fixture." +) +# Just over min_block_chars but shorter than any reference placeholder, so a +# replacement would GROW the payload and must be skipped. +SHORT_ARTIFACT = "Short synthetic artifact body just over forty chars." +LONG_FENCE_BLOCK = ( + "```log\n" + "synthetic provenance artifact line 001: worker output checksum=alpha\n" + "synthetic provenance artifact line 002: worker output checksum=bravo\n" + "synthetic provenance artifact line 003: worker output checksum=charlie\n" + "```" +) +FENCED_PARENT_ARTIFACT = ( + "Parent aggregation summary before first artifact.\n" + f"{LONG_FENCE_BLOCK}\n" + "Short prose between artifacts must survive byte-identical.\n" + f"{LONG_FENCE_BLOCK}\n" + "Parent aggregation summary after duplicate artifact." +) +SOURCE_SPAN_BLOCK = ( + "worker-span-line-001 provenance payload alpha bravo charlie\n" + "worker-span-line-002 provenance payload delta echo foxtrot\n" + "worker-span-line-003 provenance payload golf hotel india" +) +SOURCE_SPAN_TOOL = ( + "tool preamble stays canonical\n" + f"{SOURCE_SPAN_BLOCK}\n" + "tool epilogue stays canonical" +) +SOURCE_SPAN_PARENT = ( + "parent summary before copied worker span\n" + f"{SOURCE_SPAN_BLOCK}\n" + "parent summary after copied worker span" +) + + +def _ref(body: str, *, canonical_type: str = "tool_result") -> str: + """The deterministic reference string a canary would emit for ``body``.""" + return _artifact_reference_string(canonical_type, _salted_hash(body, SALT)) + + +def _ref_len(canonical_type: str = "tool_result") -> int: + return len(_ref(LONG_ARTIFACT, canonical_type=canonical_type)) + + +def _block_ref(block: str, *, canonical_type: str = "tool_result#block") -> str: + return _artifact_reference_string(canonical_type, _salted_hash(block, SALT)) + + +def _source_span_link() -> ArtifactSpanLink: + src_start = SOURCE_SPAN_TOOL.index(SOURCE_SPAN_BLOCK) + tgt_start = SOURCE_SPAN_PARENT.index(SOURCE_SPAN_BLOCK) + return ArtifactSpanLink( + source_index=0, + source_start=src_start, + source_end=src_start + len(SOURCE_SPAN_BLOCK), + target_index=1, + target_start=tgt_start, + target_end=tgt_start + len(SOURCE_SPAN_BLOCK), + ) + + +def _span_ref(span: str, *, canonical_type: str = "tool_result#span") -> str: + return _artifact_reference_string(canonical_type, _salted_hash(span, SALT)) + + +# --------------------------------------------------------------------------- +# Mode resolution + escape hatch (default OFF) +# --------------------------------------------------------------------------- + + +def test_mode_defaults_to_off(monkeypatch): + monkeypatch.delenv(ARTIFACT_DEDUP_MODE_ENV, raising=False) + monkeypatch.delenv(ARTIFACT_DEDUP_DISABLE_ENV, raising=False) + assert resolve_artifact_dedup_mode() == "off" + + +def test_mode_reads_env_values(): + assert resolve_artifact_dedup_mode({ARTIFACT_DEDUP_MODE_ENV: "shadow"}) == "shadow" + assert resolve_artifact_dedup_mode({ARTIFACT_DEDUP_MODE_ENV: "CANARY"}) == "canary" + # Unknown / garbage values fall back to the safe default. + assert resolve_artifact_dedup_mode({ARTIFACT_DEDUP_MODE_ENV: "aggressive"}) == "off" + + +def test_disable_env_is_a_kill_switch(): + env = {ARTIFACT_DEDUP_MODE_ENV: "canary", ARTIFACT_DEDUP_DISABLE_ENV: "1"} + assert resolve_artifact_dedup_mode(env) == "off" + + +# --------------------------------------------------------------------------- +# (1) off (default) and shadow never mutate +# --------------------------------------------------------------------------- + + +def test_default_off_does_not_change_payload(monkeypatch): + monkeypatch.delenv(ARTIFACT_DEDUP_MODE_ENV, raising=False) + monkeypatch.delenv(ARTIFACT_DEDUP_DISABLE_ENV, raising=False) + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + before = [c.content for c in contents] + result = apply_artifact_dedup_canary(contents, salt=SALT, min_block_chars=MIN) + assert result.mode == "off" + assert result.mutated is False + assert result.blocks_replaced == 0 + assert result.chars_saved == 0 + assert result.item_count == 0 # off does not even scan + assert [c.content for c in contents] == before # byte-identical + + +def test_disable_env_blocks_mutation_even_with_canary_set(monkeypatch): + monkeypatch.setenv(ARTIFACT_DEDUP_MODE_ENV, "canary") + monkeypatch.setenv(ARTIFACT_DEDUP_DISABLE_ENV, "true") + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + before = [c.content for c in contents] + result = apply_artifact_dedup_canary(contents, salt=SALT, min_block_chars=MIN) + assert result.mode == "off" + assert [c.content for c in contents] == before + + +def test_shadow_measures_duplicates_without_mutating(): + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + before = [c.content for c in contents] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="shadow" + ) + assert [c.content for c in contents] == before # never mutated + assert result.mode == "shadow" + assert result.mutated is False + assert result.blocks_replaced == 0 + assert result.chars_saved == 0 + # Advisory: one eligible duplicate group; later occurrence chars measured. + assert result.candidate_group_count == 1 + assert result.candidate_chars == len(LONG_ARTIFACT) + + +# --------------------------------------------------------------------------- +# (1) canary keeps the first full body, replaces later exact duplicates +# --------------------------------------------------------------------------- + + +def test_canary_keeps_first_full_and_replaces_later_duplicate(): + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert contents[0].content == LONG_ARTIFACT # first kept verbatim + assert contents[1].content == _ref(LONG_ARTIFACT) # later replaced + assert result.mode == "canary" + assert result.mutated is True + assert result.artifact_dedup_class == ARTIFACT_DEDUP_CLASS + assert result.blocks_replaced == 1 + assert result.chars_saved == len(LONG_ARTIFACT) - _ref_len() + + +def test_canary_three_occurrences_keeps_first_replaces_two(): + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("assistant_context", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert contents[0].content == LONG_ARTIFACT # canonical kept + assert contents[1].content == _ref(LONG_ARTIFACT) # later dup replaced + assert contents[2].content == _ref(LONG_ARTIFACT) + assert result.blocks_replaced == 2 + assert result.chars_saved == 2 * (len(LONG_ARTIFACT) - _ref_len()) + + +def test_canary_dedups_across_artifact_types_provenance_canonical_is_first(): + # tool_result first, assistant_context second: the duplicate spans both + # artifact types. The first (tool_result) is canonical; the reference left in + # the assistant_context records that canonical provenance. + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("assistant_context", LONG_ARTIFACT), + ] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert contents[0].content == LONG_ARTIFACT + # Reference records the canonical (first) provenance == tool_result. + assert contents[1].content == _ref(LONG_ARTIFACT, canonical_type="tool_result") + assert "tool_result" in contents[1].content + assert "assistant_context" not in contents[1].content + assert result.blocks_replaced == 1 + + +def test_segment_fenced_blocks_round_trips_and_marks_closed_fences(): + segments = _segment_fenced_blocks(FENCED_PARENT_ARTIFACT) + assert "".join(text for _kind, text in segments) == FENCED_PARENT_ARTIFACT + assert [kind for kind, _text in segments].count("fence") == 2 + + +def test_canary_replaces_later_exact_duplicate_fenced_block_inside_artifact_body(): + contents = [_LLMContent("assistant_context", FENCED_PARENT_ARTIFACT)] + + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + + assert contents[0].content.count(LONG_FENCE_BLOCK) == 1 + expected_ref = _block_ref(LONG_FENCE_BLOCK, canonical_type="assistant_context#block") + assert expected_ref in contents[0].content + assert "Short prose between artifacts must survive byte-identical." in contents[0].content + assert result.blocks_replaced == 1 + assert result.chars_saved == len(LONG_FENCE_BLOCK) - len(expected_ref) + assert dangling_artifact_references(contents, salt=SALT) == [] + + +def test_canary_replaces_duplicate_fenced_block_across_artifact_types(): + first = f"tool output wrapper\n{LONG_FENCE_BLOCK}\nend" + second = f"assistant rollup wrapper\n{LONG_FENCE_BLOCK}\nend" + contents = [ + _LLMContent("tool_result", first), + _LLMContent("assistant_context", second), + ] + + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + + assert contents[0].content == first + assert LONG_FENCE_BLOCK not in contents[1].content + assert _block_ref(LONG_FENCE_BLOCK, canonical_type="tool_result#block") in contents[1].content + assert result.blocks_replaced == 1 + + +def test_whole_body_canonical_is_not_registered_before_internal_block_rewrite(): + contents = [ + _LLMContent("assistant_context", FENCED_PARENT_ARTIFACT), + _LLMContent("assistant_context", FENCED_PARENT_ARTIFACT), + ] + + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + + assert result.blocks_replaced >= 2 + # A later reference must never point at the pre-mutation whole-body hash after + # the first body was internally rewritten; every emitted reference resolves + # to an earlier full fenced block/body still present in the payload. + assert dangling_artifact_references(contents, salt=SALT) == [] + + +def test_unterminated_fence_is_treated_as_prose_and_not_mutated(): + body = "prefix\n```log\n" + ("unterminated synthetic artifact line\n" * 8) + contents = [_LLMContent("tool_result", body + body)] + before = contents[0].content + + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + + assert contents[0].content == before + assert result.blocks_replaced == 0 + + +# --------------------------------------------------------------------------- +# Level 2: declared source-span provenance (metadata-driven, not discovery) +# --------------------------------------------------------------------------- + + +def test_source_span_canary_replaces_declared_parent_span_only(): + contents = [ + _LLMContent("tool_result", SOURCE_SPAN_TOOL), + _LLMContent("assistant_context", SOURCE_SPAN_PARENT), + ] + link = _source_span_link() + + result = apply_artifact_dedup_canary( + contents, + salt=SALT, + min_block_chars=MIN, + mode="canary", + span_links=[link], + ) + + expected_ref = _span_ref(SOURCE_SPAN_BLOCK) + assert contents[0].content == SOURCE_SPAN_TOOL + assert contents[1].content == SOURCE_SPAN_PARENT.replace(SOURCE_SPAN_BLOCK, expected_ref) + assert result.span_blocks_replaced == 1 + assert result.span_chars_saved == len(SOURCE_SPAN_BLOCK) - len(expected_ref) + assert result.blocks_replaced == 1 + assert result.chars_saved == result.span_chars_saved + assert dangling_artifact_references(contents, salt=SALT, span_links=[link]) == [] + + +def test_source_span_shadow_measures_without_mutating(): + contents = [ + _LLMContent("tool_result", SOURCE_SPAN_TOOL), + _LLMContent("assistant_context", SOURCE_SPAN_PARENT), + ] + before = [c.content for c in contents] + + result = apply_artifact_dedup_canary( + contents, + salt=SALT, + min_block_chars=MIN, + mode="shadow", + span_links=[_source_span_link()], + ) + + assert [c.content for c in contents] == before + assert result.span_blocks_replaced == 0 + assert result.span_candidate_count == 1 + assert result.span_candidate_chars == len(SOURCE_SPAN_BLOCK) + + +def test_source_span_mismatch_or_forward_link_is_not_mutated(): + mismatch_parent = SOURCE_SPAN_PARENT.replace("alpha", "ALPHA", 1) + contents = [ + _LLMContent("tool_result", SOURCE_SPAN_TOOL), + _LLMContent("assistant_context", mismatch_parent), + ] + result = apply_artifact_dedup_canary( + contents, + salt=SALT, + min_block_chars=MIN, + mode="canary", + span_links=[_source_span_link()], + ) + assert contents[1].content == mismatch_parent + assert result.span_blocks_replaced == 0 + + forward = ArtifactSpanLink(1, 0, len(SOURCE_SPAN_BLOCK), 0, 0, len(SOURCE_SPAN_BLOCK)) + before = [c.content for c in contents] + result = apply_artifact_dedup_canary( + contents, + salt=SALT, + min_block_chars=MIN, + mode="canary", + span_links=[forward], + ) + assert [c.content for c in contents] == before + assert result.span_blocks_replaced == 0 + + +def test_source_span_rejects_protected_or_inline_scope(): + inline_parent = SOURCE_SPAN_PARENT.replace("\n" + SOURCE_SPAN_BLOCK + "\n", SOURCE_SPAN_BLOCK) + contents = [ + _LLMContent("tool_result", SOURCE_SPAN_TOOL), + _LLMContent("assistant_context", inline_parent), + _LLMContent("user_prompt", SOURCE_SPAN_PARENT), + ] + inline_start = inline_parent.index(SOURCE_SPAN_BLOCK) + links = [ + ArtifactSpanLink(0, SOURCE_SPAN_TOOL.index(SOURCE_SPAN_BLOCK), SOURCE_SPAN_TOOL.index(SOURCE_SPAN_BLOCK) + len(SOURCE_SPAN_BLOCK), 1, inline_start, inline_start + len(SOURCE_SPAN_BLOCK)), + ArtifactSpanLink(0, SOURCE_SPAN_TOOL.index(SOURCE_SPAN_BLOCK), SOURCE_SPAN_TOOL.index(SOURCE_SPAN_BLOCK) + len(SOURCE_SPAN_BLOCK), 2, SOURCE_SPAN_PARENT.index(SOURCE_SPAN_BLOCK), SOURCE_SPAN_PARENT.index(SOURCE_SPAN_BLOCK) + len(SOURCE_SPAN_BLOCK)), + ] + before = [c.content for c in contents] + + result = apply_artifact_dedup_canary( + contents, + salt=SALT, + min_block_chars=MIN, + mode="canary", + span_links=links, + ) + + assert [c.content for c in contents] == before + assert result.span_blocks_replaced == 0 + + +def test_canary_reference_carries_no_raw_artifact_body(): + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + apply_artifact_dedup_canary(contents, salt=SALT, min_block_chars=MIN, mode="canary") + ref_line = contents[1].content + # Only a low-cardinality provenance enum + salted hash, never the body. + assert "tool_result" in ref_line + assert _salted_hash(LONG_ARTIFACT, SALT) in ref_line + assert LONG_ARTIFACT not in ref_line + assert len(ref_line) < len(LONG_ARTIFACT) # always shrinks + + +def test_canary_never_grows_payload_for_short_duplicate(): + assert len(SHORT_ARTIFACT) >= MIN + assert len(SHORT_ARTIFACT) < _ref_len() # reference would be longer + contents = [ + _LLMContent("tool_result", SHORT_ARTIFACT), + _LLMContent("tool_result", SHORT_ARTIFACT), + ] + before = [c.content for c in contents] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert [c.content for c in contents] == before # left alone (would grow) + assert result.blocks_replaced == 0 + assert result.chars_saved == 0 + + +# --------------------------------------------------------------------------- +# (2) only tool_result / assistant_context are mutable; others protected +# --------------------------------------------------------------------------- + + +def test_mutable_set_is_exactly_the_two_artifact_types(): + assert set(MUTABLE_ARTIFACT_BLOCK_TYPES) == {"tool_result", "assistant_context"} + + +def test_canary_leaves_non_artifact_block_types_untouched(): + # Duplicate bodies in every PROTECTED block type must survive byte-identical + # and must not even be scanned as artifact candidates. + contents = [ + _LLMContent("system_prompt", SYS_BLOCK), + _LLMContent("system_prompt", SYS_BLOCK), + _LLMContent("user_prompt", LONG_ARTIFACT), + _LLMContent("user_prompt", LONG_ARTIFACT), + _LLMContent("skill_prompt", LONG_ARTIFACT), + _LLMContent("skill_prompt", LONG_ARTIFACT), + _LLMContent("unknown", LONG_ARTIFACT), + _LLMContent("unknown", LONG_ARTIFACT), + ] + before = [c.content for c in contents] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert [c.content for c in contents] == before + assert result.blocks_replaced == 0 + assert result.item_count == 0 # no artifact items present to scan + + +def test_canary_does_not_dedup_artifact_against_protected_duplicate(): + # The same body appears in a protected user_prompt AND a tool_result. The + # artifact occurrence has no EARLIER artifact canonical, so nothing is + # replaced (a reference may only point at an artifact canonical body). + contents = [ + _LLMContent("user_prompt", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + before = [c.content for c in contents] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert [c.content for c in contents] == before + assert result.blocks_replaced == 0 + + +def test_assistant_context_is_a_mutable_artifact(): + contents = [ + _LLMContent("assistant_context", LONG_ARTIFACT), + _LLMContent("assistant_context", LONG_ARTIFACT), + ] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert contents[0].content == LONG_ARTIFACT + assert contents[1].content == _ref(LONG_ARTIFACT, canonical_type="assistant_context") + assert result.blocks_replaced == 1 + + +# --------------------------------------------------------------------------- +# (3) reference resolution: must point at an earlier canonical full body +# --------------------------------------------------------------------------- + + +def test_canary_output_has_no_dangling_references(): + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("assistant_context", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + apply_artifact_dedup_canary(contents, salt=SALT, min_block_chars=MIN, mode="canary") + # Every reference the canary emitted resolves to an earlier canonical body. + assert dangling_artifact_references(contents, salt=SALT) == [] + + +def test_resolution_accepts_reference_after_its_canonical_body(): + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), # canonical full body + _LLMContent("tool_result", _ref(LONG_ARTIFACT)), # resolves to index 0 + ] + assert dangling_artifact_references(contents, salt=SALT) == [] + + +def test_resolution_flags_reference_with_no_earlier_canonical(): + # A reference whose canonical body never appears earlier is dangling. + contents = [ + _LLMContent("tool_result", _ref(LONG_ARTIFACT)), + ] + assert dangling_artifact_references(contents, salt=SALT) == [0] + + +def test_resolution_flags_reference_before_its_canonical_body(): + # The canonical body exists, but only AFTER the reference -> still dangling. + contents = [ + _LLMContent("tool_result", _ref(LONG_ARTIFACT)), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + assert dangling_artifact_references(contents, salt=SALT) == [0] + + +# --------------------------------------------------------------------------- +# (4) telemetry: realized chars_saved only on actual mutation, no raw content +# --------------------------------------------------------------------------- + + +def test_telemetry_records_no_savings_when_off(): + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="off" + ) + record = build_artifact_canary_telemetry_record(result) + assert record["artifact_dedup_mode"] == "off" + assert record["artifact_dedup_blocks_replaced"] == 0 + assert record["artifact_dedup_chars_saved"] == 0 + assert record["chars_saved"] == 0 + + +def test_telemetry_records_no_savings_in_shadow(): + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="shadow" + ) + record = build_artifact_canary_telemetry_record(result) + assert record["artifact_dedup_mode"] == "shadow" + # Shadow contributes nothing to the realized chars_saved total. + assert record["artifact_dedup_chars_saved"] == 0 + assert record["chars_saved"] == 0 + + +def test_telemetry_records_realized_savings_in_canary(): + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("assistant_context", LONG_ARTIFACT), + ] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + record = build_artifact_canary_telemetry_record(result) + expected = 2 * (len(LONG_ARTIFACT) - _ref_len()) + assert record["artifact_dedup_mode"] == "canary" + assert record["artifact_dedup_class"] == ARTIFACT_DEDUP_CLASS + assert record["artifact_dedup_blocks_replaced"] == 2 + assert record["artifact_dedup_chars_saved"] == expected + assert record["chars_saved"] == expected + + +def test_telemetry_is_metadata_only_no_artifact_text(): + contents = [ + _LLMContent("tool_result", LONG_ARTIFACT), + _LLMContent("tool_result", LONG_ARTIFACT), + ] + result = apply_artifact_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + record = build_artifact_canary_telemetry_record(result) + blob = json.dumps(record) + assert LONG_ARTIFACT not in blob + assert set(record) == { + "artifact_dedup_mode", + "artifact_dedup_class", + "artifact_dedup_blocks_replaced", + "artifact_span_blocks_replaced", + "artifact_span_chars_saved", + "artifact_dedup_chars_saved", + "chars_saved", + } + for key in ( + "artifact_dedup_blocks_replaced", + "artifact_span_blocks_replaced", + "artifact_span_chars_saved", + "artifact_dedup_chars_saved", + "chars_saved", + ): + assert isinstance(record[key], int) + + +# --------------------------------------------------------------------------- +# (3)+(4) runner/validation gate over synthetic artifact cases +# --------------------------------------------------------------------------- + + +def _artifact_cases() -> list[dict]: + """Two synthetic cases, each with an exact-duplicate artifact body.""" + return [ + { + "case_id": "syn-art-1", + "source": "synthetic", + "messages": [ + {"role": "system", "block_type": "system_prompt", "content": SYS_BLOCK}, + {"role": "tool", "block_type": "tool_result", "content": LONG_ARTIFACT}, + { + "role": "user", + "block_type": "user_prompt", + "content": "Synthetic user question referencing the artifact above.", + }, + # exact duplicate of the earlier tool_result -> replaced by ref. + {"role": "tool", "block_type": "tool_result", "content": LONG_ARTIFACT}, + ], + }, + { + "case_id": "syn-art-2", + "source": "synthetic", + "messages": [ + { + "role": "assistant", + "block_type": "assistant_context", + "content": LONG_ARTIFACT, + }, + # cross-type duplicate -> canonical is the assistant_context above. + {"role": "tool", "block_type": "tool_result", "content": LONG_ARTIFACT}, + ], + }, + ] + + +def test_artifact_runner_passes_on_synthetic_duplicate_artifacts(): + report = run_artifact_validation( + _artifact_cases(), + baseline_mode="off", + candidate_mode="canary", + salt=SALT, + min_block_chars=MIN, + date="2026-06-15", + ) + assert report.passed is True + assert report.failed_cases == 0 + assert report.invariant_names == ARTIFACT_INVARIANT_NAMES + assert report.total_blocks_replaced == 2 # one duplicate per case + assert report.total_chars_saved > 0 + assert any(c.mutated for c in report.cases) + # No tokenizer backend configured by default. + assert report.tokenizer_status == "unavailable" + assert report.total_actual_tokens_saved is None + + +def test_artifact_runner_shadow_passes_without_realized_savings(): + report = run_artifact_validation( + _artifact_cases(), + baseline_mode="off", + candidate_mode="shadow", + salt=SALT, + min_block_chars=MIN, + date="2026-06-15", + ) + assert report.passed is True + assert report.total_blocks_replaced == 0 + assert report.total_chars_saved == 0 + assert all(not c.mutated for c in report.cases) + + +def test_artifact_runner_accepts_source_span_that_ends_with_reference_suffix_char(): + source_span = ( + "worker copied JSON-ish payload line 001 alpha bravo charlie delta echo\n" + "worker copied JSON-ish payload line 002 foxtrot golf hotel india juliet\n" + "worker copied JSON-ish payload line 003 kilo lima mike november oscar]" + ) + tool = "tool wrapper before\n" + source_span + "\ntool wrapper after" + parent = "parent summary before\n" + source_span + "\nparent summary after" + link = ArtifactSpanLink( + source_index=0, + source_start=tool.index(source_span), + source_end=tool.index(source_span) + len(source_span), + target_index=1, + target_start=parent.index(source_span), + target_end=parent.index(source_span) + len(source_span), + ) + case = { + "case_id": "syn-art-span-bracket", + "source": "synthetic", + "span_links": [link.__dict__], + "messages": [ + {"role": "tool", "block_type": "tool_result", "content": tool}, + {"role": "assistant", "block_type": "assistant_context", "content": parent}, + ], + } + + report = run_artifact_validation( + [case], + baseline_mode="off", + candidate_mode="canary", + salt=SALT, + min_block_chars=MIN, + date="2026-06-15", + ) + + assert report.passed is True + assert report.failed_cases == 0 + assert report.total_blocks_replaced == 1 + assert report.total_chars_saved > 0 + + +def test_artifact_runner_report_is_privacy_safe(): + report = run_artifact_validation( + _artifact_cases(), + baseline_mode="off", + candidate_mode="canary", + salt=SALT, + min_block_chars=MIN, + date="2026-06-15", + ) + report_dict = report_to_dict(report) + raw_needles = [ + LONG_ARTIFACT, + SYS_BLOCK, + "Synthetic user question referencing the artifact above.", + ] + assert_report_privacy_safe(report_dict, raw_needles) + blob = json.dumps(report_dict, ensure_ascii=False) + md = render_markdown(report) + for needle in raw_needles: + assert needle not in blob + assert needle not in md + + +def test_artifact_runner_fails_on_dangling_reference(): + # A candidate that replaces the FIRST (canonical) artifact body with a + # reference leaves a dangling reference: nothing earlier resolves it. + def bad_dangling(messages, *, mode, salt, min_block_chars): + out = [dict(m) for m in messages] + if mode == "bad": + for m in out: + if m["block_type"] in MUTABLE_ARTIFACT_BLOCK_TYPES: + m["content"] = _artifact_reference_string( + m["block_type"], _salted_hash("no-such-canonical", salt) + ) + break + result = ArtifactDedupCanaryResult( + mode="canary", + artifact_dedup_class=ARTIFACT_DEDUP_CLASS, + mutated=True, + item_count=2, + candidate_group_count=1, + candidate_chars=len(LONG_ARTIFACT), + blocks_replaced=1, + chars_saved=len(LONG_ARTIFACT) - _ref_len(), + ) + return out, result + result = ArtifactDedupCanaryResult( + mode="off", + artifact_dedup_class=ARTIFACT_DEDUP_CLASS, + mutated=False, + item_count=0, + candidate_group_count=0, + candidate_chars=0, + blocks_replaced=0, + chars_saved=0, + ) + return out, result + + report = run_artifact_validation( + _artifact_cases()[:1], + baseline_mode="off", + candidate_mode="bad", + salt=SALT, + min_block_chars=MIN, + date="2026-06-15", + optimize_fn=bad_dangling, + ) + assert report.passed is False + assert report.failed_cases == 1 + assert "artifact_reference_resolvable" in report.cases[0].failed_invariants + + +def test_artifact_runner_fails_on_protected_mutation(): + # A candidate that rewrites a protected (non-artifact) user_prompt must fail + # the protected-content and mutation-scope invariants. + def bad_protected(messages, *, mode, salt, min_block_chars): + out = [dict(m) for m in messages] + if mode == "bad": + for m in out: + if m["block_type"] == "user_prompt": + m["content"] = "[dropped]" + break + result = ArtifactDedupCanaryResult( + mode="canary", + artifact_dedup_class=ARTIFACT_DEDUP_CLASS, + mutated=True, + item_count=2, + candidate_group_count=0, + candidate_chars=0, + blocks_replaced=1, + chars_saved=1, + ) + return out, result + result = ArtifactDedupCanaryResult( + mode="off", + artifact_dedup_class=ARTIFACT_DEDUP_CLASS, + mutated=False, + item_count=0, + candidate_group_count=0, + candidate_chars=0, + blocks_replaced=0, + chars_saved=0, + ) + return out, result + + report = run_artifact_validation( + _artifact_cases()[:1], + baseline_mode="off", + candidate_mode="bad", + salt=SALT, + min_block_chars=MIN, + date="2026-06-15", + optimize_fn=bad_protected, + ) + assert report.passed is False + failed = report.cases[0].failed_invariants + assert "protected_content_preserved" in failed + assert "artifact_mutation_scope_allowed" in failed + + +def test_reference_template_is_low_cardinality_placeholder_only(): + # The template carries only / placeholders -- never content. + assert "" in ARTIFACT_DEDUP_CANARY_REFERENCE_TEMPLATE + assert "" in ARTIFACT_DEDUP_CANARY_REFERENCE_TEMPLATE diff --git a/tests/test_artifact_precision_eval.py b/tests/test_artifact_precision_eval.py new file mode 100644 index 0000000..9caa9a5 --- /dev/null +++ b/tests/test_artifact_precision_eval.py @@ -0,0 +1,53 @@ +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + + +REPO_ROOT = Path(__file__).resolve().parents[1] +MODULE_PATH = REPO_ROOT / "scripts" / "evaluate_artifact_precision.py" +spec = importlib.util.spec_from_file_location("evaluate_artifact_precision", MODULE_PATH) +assert spec is not None and spec.loader is not None +evaluator = importlib.util.module_from_spec(spec) +sys.modules[spec.name] = evaluator +spec.loader.exec_module(evaluator) +build_report = evaluator.build_report + + +def test_synthetic_artifact_precision_report_is_namespaced_and_reproducible(): + report = build_report() + + assert report["corpus"] == "synthetic_labeled_artifact_precision_v1" + assert "precision" not in report + assert "recall" not in report + assert report["synthetic_event_precision"] == 1.0 + assert report["synthetic_event_recall"] == 1.0 + assert report["synthetic_case_accuracy"] == 1.0 + assert report["case_count"] >= 15 + assert report["synthetic_event_tp"] == report["expected_replacements"] + assert report["synthetic_event_fp"] == 0 + assert report["synthetic_event_fn"] == 0 + assert report["mode_gate_checks"] == { + "off_no_mutation": True, + "shadow_no_mutation": True, + "disable_env_no_mutation": True, + } + assert report["synthetic_negative_case_fpr"] == 0.0 + assert report["validation_gate_checks"]["forged_reference_detected"] is True + + +def test_synthetic_artifact_precision_cli_writes_json(tmp_path): + out = tmp_path / "precision.json" + completed = subprocess.run( + [sys.executable, "scripts/evaluate_artifact_precision.py", "--output", str(out)], + cwd=REPO_ROOT, + text=True, + capture_output=True, + check=True, + ) + + stdout_report = json.loads(completed.stdout) + file_report = json.loads(out.read_text()) + assert file_report == stdout_report + assert stdout_report["synthetic_event_precision"] == 1.0 diff --git a/tests/test_contextpilot_savings.py b/tests/test_contextpilot_savings.py index 9baf22a..dcc790b 100644 --- a/tests/test_contextpilot_savings.py +++ b/tests/test_contextpilot_savings.py @@ -23,7 +23,7 @@ def test_missing_file_is_safe_and_prompts_user(): summary = savings.summarize_telemetry(Path("/no/such/telemetry.jsonl"), since_hours=24) assert summary["file_exists"] is False assert summary["events"] == 0 - assert summary["tokens_saved"] == 0 + assert summary["actual_token_status"] == "unavailable" text = savings.render_text(summary) assert "No ContextPilot telemetry found" in text assert "Restart Hermes" in text @@ -36,7 +36,7 @@ def test_time_window_filtering(tmp_path): _write_jsonl( tel, [ - {"ts": now - 3600, "type": "turn", "chars_saved": 400, "tokens_saved": 100}, + {"ts": now - 3600, "type": "turn", "chars_saved": 400, "actual_token_status": "available", "actual_tokenizer_backend": "tiktoken:cl100k_base", "actual_tokens_saved": 100, "actual_tokens_before": 200, "actual_tokens_after": 100}, {"ts": now - 7200, "type": "turn", "chars_saved": 200, "tokens_saved": 50}, # ~2 days ago: outside a 24h window. {"ts": now - 48 * 3600, "type": "turn", "chars_saved": 999999, "tokens_saved": 999999}, @@ -48,8 +48,8 @@ def test_time_window_filtering(tmp_path): summary = savings.summarize_telemetry(tel, since_hours=24) assert summary["events"] == 2 assert summary["chars_saved"] == 600 - assert summary["tokens_saved"] == 150 - assert summary["avg_tokens_per_event"] == 75.0 + assert summary["actual_tokens_saved"] == 100 + assert summary["actual_token_events"] == 1 assert summary["window_start_iso"] is not None assert summary["skipped_lines"] == 1 @@ -60,7 +60,7 @@ def test_all_time_mode_includes_old_records(tmp_path): _write_jsonl( tel, [ - {"ts": now - 3600, "type": "turn", "chars_saved": 400, "tokens_saved": 100}, + {"ts": now - 3600, "type": "turn", "chars_saved": 400, "actual_token_status": "available", "actual_tokenizer_backend": "tiktoken:cl100k_base", "actual_tokens_saved": 100, "actual_tokens_before": 200, "actual_tokens_after": 100}, {"ts": now - 48 * 3600, "type": "turn", "chars_saved": 200, "tokens_saved": 50}, # No timestamp at all should still count in all-time mode. {"type": "turn", "chars_saved": 40, "tokens_saved": 10}, @@ -72,7 +72,7 @@ def test_all_time_mode_includes_old_records(tmp_path): assert summary["window_start_iso"] is None assert summary["events"] == 3 assert summary["chars_saved"] == 640 - assert summary["tokens_saved"] == 160 + assert summary["actual_tokens_saved"] == 100 def test_malformed_lines_skipped_and_counted(tmp_path): @@ -81,7 +81,7 @@ def test_malformed_lines_skipped_and_counted(tmp_path): _write_jsonl( tel, [ - {"ts": now, "type": "turn", "chars_saved": 400, "tokens_saved": 100}, + {"ts": now, "type": "turn", "chars_saved": 400, "actual_token_status": "available", "actual_tokenizer_backend": "tiktoken:cl100k_base", "actual_tokens_saved": 100, "actual_tokens_before": 200, "actual_tokens_after": 100}, "this is not json", "[1, 2, 3]", # valid json, but not a dict {"ts": now, "type": "turn", "chars_saved": "not a number"}, # bad field @@ -92,11 +92,10 @@ def test_malformed_lines_skipped_and_counted(tmp_path): ], ) summary = savings.summarize_telemetry(tel, since_hours=None) - assert summary["events"] == 2 - assert summary["chars_saved"] == 500 - # 100 (explicit) + 100//4 (derived) = 125 - assert summary["tokens_saved"] == 125 - assert summary["skipped_lines"] == 5 + assert summary["events"] == 3 + assert summary["chars_saved"] == 510 + assert summary["actual_tokens_saved"] == 100 + assert summary["skipped_lines"] == 4 def test_json_output_schema_and_no_raw_content(tmp_path, capsys): @@ -111,7 +110,7 @@ def test_json_output_schema_and_no_raw_content(tmp_path, capsys): "ts": now, "type": "turn", "chars_saved": 400, - "tokens_saved": 100, + "actual_token_status": "available", "actual_tokenizer_backend": "tiktoken:cl100k_base", "actual_tokens_saved": 100, "actual_tokens_before": 200, "actual_tokens_after": 100, "content": "SECRET CONVERSATION TEXT", "system_prompt": "SECRET SYSTEM PROMPT", }, @@ -130,13 +129,19 @@ def test_json_output_schema_and_no_raw_content(tmp_path, capsys): "window_start_iso", "events", "chars_saved", - "tokens_saved", - "avg_tokens_per_event", + "actual_token_status", + "actual_token_events", + "actual_tokens_before", + "actual_tokens_after", + "actual_tokens_saved", + "actual_tokenizer_backends", "skipped_lines", } assert set(data.keys()) == expected_keys assert data["events"] == 1 - assert data["tokens_saved"] == 100 + assert data["actual_tokens_saved"] == 100 + assert data["actual_token_status"] == "available" + assert data["actual_tokenizer_backends"] == ["tiktoken:cl100k_base"] assert "SECRET CONVERSATION TEXT" not in out assert "SECRET SYSTEM PROMPT" not in out @@ -146,21 +151,66 @@ def test_text_output_renders_savings(tmp_path, capsys): now = time.time() _write_jsonl( tel, - [{"ts": now, "type": "turn", "chars_saved": 400, "tokens_saved": 100}], + [{"ts": now, "type": "turn", "chars_saved": 400, "actual_token_status": "available", "actual_tokenizer_backend": "tiktoken:cl100k_base", "actual_tokens_saved": 100, "actual_tokens_before": 200, "actual_tokens_after": 100}], ) rc = savings.main(["--telemetry-file", str(tel), "--since-hours", "24"]) assert rc == 0 out = capsys.readouterr().out - assert "ContextPilot token savings (last 24h)" in out - assert "Estimated tokens saved" in out + assert "ContextPilot savings (last 24h)" in out + assert "Est. tokens saved (chars/4, derived)" not in out + assert "Telemetry tokens saved" not in out + assert "Actual tokens saved (tokenizer): 100" in out assert str(tel) in out +def test_actual_tokenizer_tokens_surfaced_separately(tmp_path, capsys): + """Tokenizer fields are aggregated; legacy token fields are ignored.""" + tel = tmp_path / "telemetry.jsonl" + now = time.time() + _write_jsonl( + tel, + [ + { + "ts": now, + "type": "turn", + "chars_saved": 400, + "actual_token_status": "available", + "actual_tokenizer_backend": "tiktoken:cl100k_base", + "actual_tokens_before": 90, + "actual_tokens_after": 30, + "actual_tokens_saved": 60, + }, + # A record with no exact tokenizer must not pollute the actual totals. + { + "ts": now, + "type": "turn", + "chars_saved": 200, + "tokens_saved": 50, + "actual_token_status": "unavailable", + }, + ], + ) + summary = savings.summarize_telemetry(tel, since_hours=None) + assert summary["events"] == 2 + assert summary["actual_token_status"] == "available" + assert summary["actual_token_events"] == 1 + assert summary["actual_tokens_before"] == 90 + assert summary["actual_tokens_after"] == 30 + assert summary["actual_tokens_saved"] == 60 + assert summary["actual_tokenizer_backends"] == ["tiktoken:cl100k_base"] + + text = savings.render_text(summary) + assert "Est. tokens saved (chars/4, derived): 150" not in text + assert "Actual tokens saved (tokenizer): 60" in text + assert "tiktoken:cl100k_base" in text + assert "status: available" in text + + def test_no_events_in_window_message(tmp_path, capsys): tel = tmp_path / "telemetry.jsonl" _write_jsonl( tel, - [{"ts": 1000.0, "type": "turn", "chars_saved": 400, "tokens_saved": 100}], + [{"ts": 1000.0, "type": "turn", "chars_saved": 400, "actual_token_status": "available", "actual_tokenizer_backend": "tiktoken:cl100k_base", "actual_tokens_saved": 100, "actual_tokens_before": 200, "actual_tokens_after": 100}], ) rc = savings.main(["--telemetry-file", str(tel), "--since-hours", "24"]) assert rc == 0 diff --git a/tests/test_hermes_context_opportunity_analyzer.py b/tests/test_hermes_context_opportunity_analyzer.py index fc997fc..3227012 100644 --- a/tests/test_hermes_context_opportunity_analyzer.py +++ b/tests/test_hermes_context_opportunity_analyzer.py @@ -183,8 +183,8 @@ def test_malformed_telemetry_tolerated(tmp_path): tel.write_text( "\n".join( [ - json.dumps({"ts": FAR_FUTURE, "chars_saved": 400, "tokens_saved": 100}), - json.dumps({"ts": FAR_FUTURE, "chars_saved": 200}), # missing tokens_saved + json.dumps({"ts": FAR_FUTURE, "chars_saved": 400, "actual_token_status": "available", "actual_tokens_saved": 100}), + json.dumps({"ts": FAR_FUTURE, "chars_saved": 200}), # valid char metadata, token unavailable "this is not json at all", json.dumps([1, 2, 3]), # not a dict json.dumps({"ts": FAR_FUTURE, "note": "no counters here"}), @@ -196,10 +196,11 @@ def test_malformed_telemetry_tolerated(tmp_path): ) report = _analyze(db, tmp_path, telemetry=tel) t = report.telemetry - # Two valid records aggregated; second infers tokens from chars (200//4=50). + # Two valid char-metadata records aggregated; only tokenizer-measured token + # telemetry is counted, and missing actual tokens remain unavailable/zero. assert t.events == 2 assert t.chars_saved == 600 - assert t.tokens_saved == 150 + assert t.tokens_saved == 100 # Non-json, non-dict, and missing-counter lines are skipped, not fatal. assert t.malformed_records_skipped == 3 assert t.coverage_ratio_pct > 0 @@ -939,3 +940,229 @@ def test_worker_routing_intact_alongside_parent_aggregation(tmp_path): assert report.worker_routing.est_drop_candidate_tokens > 0 # And parent aggregation independently sees the same body as a duplicate. assert report.parent_aggregation.duplicate_group_count == 1 + + +# --------------------------------------------------------------------------- +# Prompt duplicate shadow (system/skill prompts only; advisory only) +# --------------------------------------------------------------------------- + + +def test_prompt_duplicate_shadow_detects_system_skill_duplicates(): + line = "This is a sufficiently long duplicated instruction line here." + sys_unique = "A completely unique system instruction line that is long." + skill_unique = "Skill body unique line that is also clearly long enough." + contents = [ + analyzer._LLMContent( + block_type="system_prompt", content=f"{line}\n{sys_unique}\n{line}" + ), + analyzer._LLMContent(block_type="skill_prompt", content=f"{line}\n{skill_unique}"), + # Non-prompt duplicates must be ignored by this prompt-only section. + analyzer._LLMContent(block_type="tool_result", content=f"{line}\n{line}"), + analyzer._LLMContent(block_type="user_prompt", content=f"{line}\n{line}"), + ] + shadow = analyzer.detect_prompt_duplicate_blocks( + contents, salt="s", min_block_chars=40, top_n=20 + ) + assert shadow.enabled + assert shadow.item_count == 2 # only system + skill items scanned + assert shadow.scanned_block_types == ["system_prompt", "skill_prompt"] + # `line` appears 2x (system) + 1x (skill) = 3 across prompt types only. + assert shadow.duplicate_group_count == 1 + grp = shadow.top_duplicate_blocks[0] + assert grp.occurrences == 3 + assert grp.block_types == ["skill_prompt", "system_prompt"] + assert grp.chars_duplicated == (3 - 1) * len(line) + assert shadow.total_chars_duplicated == grp.chars_duplicated + # Advisory token figure is exactly chars/4, never an actual token count. + assert ( + shadow.advisory_est_duplicate_tokens_chars_div_4 + == shadow.total_chars_duplicated // 4 + ) + assert ( + grp.advisory_est_duplicate_tokens_chars_div_4 == grp.chars_duplicated // 4 + ) + # Occurrences are broken out per prompt type. + types = {tc.block_type: tc for tc in shadow.by_block_type} + assert set(types) == {"system_prompt", "skill_prompt"} + assert types["system_prompt"].occurrence_count == 2 + assert types["skill_prompt"].occurrence_count == 1 + + +def test_prompt_duplicate_shadow_in_report_no_leak_and_advisory(tmp_path): + db = tmp_path / "state.db" + secret_line = "SECRET-PROMPT-LINE-THAT-REPEATS-AND-IS-PLENTY-LONG" + other_line = "some other distinct system instruction text here now" + sys_prompt = f"{secret_line}\n{other_line}\n{secret_line}" + _make_db( + db, + [("tool", "irrelevant tool output", "Bash")], + sessions=[("raw-session-id", "discord", None, 1, 1, 100, 10, 1, sys_prompt)], + ) + report = _analyze(db, tmp_path) + pd = report.prompt_duplicates + assert pd.enabled + assert pd.duplicate_group_count == 1 + assert pd.total_chars_duplicated == len(secret_line) + # Advisory figures are NOT folded into realized telemetry savings. + assert report.telemetry.chars_saved == 0 + assert pd.total_chars_duplicated > 0 + + json_path, md_path = analyzer.write_report(report, tmp_path / "out") + md_text = md_path.read_text(encoding="utf-8") + blob = json_path.read_text(encoding="utf-8") + md_text + # Raw prompt text must never appear in the report. + assert secret_line not in blob + assert other_line not in blob + # Section is present and clearly labelled advisory / not-realized. + assert "Prompt duplicate blocks" in md_text + assert "advisory" in md_text.lower() + assert "NOT a realized saving" in md_text or "NOT realized savings" in md_text + + +def test_prompt_duplicate_shadow_can_be_disabled(tmp_path): + db = tmp_path / "state.db" + _make_db(db, [("tool", "out", "Bash")]) + tool_messages = analyzer.load_tool_messages(db, since_hours=WIDE_WINDOW) + llm = analyzer.load_llm_bound_content(db, since_hours=WIDE_WINDOW) + heavy = analyzer.load_heavy_sessions( + db, since_hours=WIDE_WINDOW, salt="s", top_n=20 + ) + tel = analyzer.parse_telemetry( + tmp_path / "none.jsonl", since_hours=WIDE_WINDOW, total_input_tokens=0 + ) + report = analyzer.build_report( + date="2100-01-01", + since_hours=24, + salt="s", + tool_messages=tool_messages, + heavy_sessions=heavy, + telemetry=tel, + llm_contents=llm, + prompt_duplicate_shadow=False, + ) + assert report.prompt_duplicates.enabled is False + _, md_path = analyzer.write_report(report, tmp_path / "out") + # Section still renders, marked disabled; report writing stays healthy. + md_text = md_path.read_text(encoding="utf-8") + assert "Prompt duplicate blocks" in md_text + assert "disabled" in md_text + + +# --------------------------------------------------------------------------- +# Prompt dedup A/B simulation (offline only; no replacement) +# --------------------------------------------------------------------------- + + +def test_prompt_dedup_ab_simulates_candidate_classes_without_tokenizer(): + skill_line = "Skill duplicate instruction line long enough for hashing." + sys_line = "System duplicate instruction line long enough for hashing." + cross_line = "Cross prompt duplicate instruction line long enough for hashing." + contents = [ + analyzer._LLMContent( + block_type="skill_prompt", + content=f"{skill_line}\n{skill_line}\n{cross_line}", + ), + analyzer._LLMContent( + block_type="system_prompt", + content=f"{sys_line}\n{sys_line}\n{cross_line}", + ), + analyzer._LLMContent( + block_type="tool_result", + content=f"{skill_line}\n{skill_line}", + ), + ] + sim = analyzer.simulate_prompt_dedup_ab( + contents, salt="s", min_block_chars=40, tokenizer=None + ) + assert sim.enabled + assert sim.item_count == 2 + assert sim.tokenizer_status == "unavailable" + classes = {c.candidate_class: c for c in sim.classes} + assert classes["same_type_skill_prompt_only"].candidate_group_count == 1 + assert classes["same_type_skill_prompt_only"].replacement_occurrence_count == 1 + assert classes["same_type_system_prompt_only"].candidate_group_count == 1 + assert classes["cross_type_system_skill"].candidate_group_count == 1 + for cls in classes.values(): + assert cls.actual_tokens_before is None + assert cls.actual_tokens_after is None + assert cls.actual_tokens_delta is None + # Tool duplicates with the same text are ignored by this prompt-only harness. + assert all("tool" not in c.candidate_class for c in sim.classes) + + +def test_prompt_dedup_ab_uses_injected_tokenizer_only_when_available(): + line = "Skill duplicate instruction line long enough for tokenizer counting." + fake = analyzer.TokenizerBackend( + name="fake:chars", + count=lambda text: len(text), + ) + sim = analyzer.simulate_prompt_dedup_ab( + [ + analyzer._LLMContent( + block_type="skill_prompt", content=f"{line}\n{line}\n{line}" + ) + ], + salt="s", + min_block_chars=40, + tokenizer=fake, + ) + assert sim.tokenizer_status == "available" + assert sim.tokenizer_backend == "fake:chars" + cls = {c.candidate_class: c for c in sim.classes}["same_type_skill_prompt_only"] + assert cls.actual_tokens_before == 3 * len(line) + assert cls.actual_tokens_after is not None + assert cls.actual_tokens_delta == cls.actual_tokens_before - cls.actual_tokens_after + + +def test_prompt_dedup_ab_report_no_leak_and_not_realized(tmp_path): + db = tmp_path / "state.db" + secret_line = "SECRET-PROMPT-AB-LINE-THAT-REPEATS-AND-IS-LONG-ENOUGH" + sys_prompt = f"{secret_line}\n{secret_line}" + _make_db( + db, + [("tool", "irrelevant tool output", "Bash")], + sessions=[("raw-session-id", "discord", None, 1, 1, 100, 10, 1, sys_prompt)], + ) + report = _analyze(db, tmp_path) + ab = report.prompt_dedup_ab + assert ab.enabled + cls = {c.candidate_class: c for c in ab.classes}["same_type_system_prompt_only"] + assert cls.candidate_group_count == 1 + assert cls.replacement_occurrence_count == 1 + # A/B simulation is not realized telemetry savings. + assert report.telemetry.chars_saved == 0 + + json_path, md_path = analyzer.write_report(report, tmp_path / "out") + blob = json_path.read_text(encoding="utf-8") + md_path.read_text(encoding="utf-8") + assert secret_line not in blob + assert "Prompt dedup A/B simulation" in blob + assert "OFFLINE SIMULATION ONLY" in blob + assert "NOT realized savings" in blob + + +def test_prompt_dedup_ab_can_be_disabled(tmp_path): + db = tmp_path / "state.db" + _make_db(db, [("tool", "out", "Bash")]) + tool_messages = analyzer.load_tool_messages(db, since_hours=WIDE_WINDOW) + llm = analyzer.load_llm_bound_content(db, since_hours=WIDE_WINDOW) + heavy = analyzer.load_heavy_sessions( + db, since_hours=WIDE_WINDOW, salt="s", top_n=20 + ) + tel = analyzer.parse_telemetry( + tmp_path / "none.jsonl", since_hours=WIDE_WINDOW, total_input_tokens=0 + ) + report = analyzer.build_report( + date="2100-01-01", + since_hours=24, + salt="s", + tool_messages=tool_messages, + heavy_sessions=heavy, + telemetry=tel, + llm_contents=llm, + prompt_dedup_ab=False, + ) + assert report.prompt_dedup_ab.enabled is False + _, md_path = analyzer.write_report(report, tmp_path / "out") + md_text = md_path.read_text(encoding="utf-8") + assert "Prompt dedup A/B simulation" in md_text + assert "disabled" in md_text diff --git a/tests/test_hermes_contextpilot_monitor.py b/tests/test_hermes_contextpilot_monitor.py index ba56bb2..9907371 100644 --- a/tests/test_hermes_contextpilot_monitor.py +++ b/tests/test_hermes_contextpilot_monitor.py @@ -106,14 +106,32 @@ def test_monitor_reads_metadata_only_and_hashes_session_ids(tmp_path): data = json.loads(json_path.read_text(encoding="utf-8")) md = md_path.read_text(encoding="utf-8") assert data["session_count"] == 1 - assert data["contextpilot_tokens_saved"] == 100 - assert data["estimated_input_token_reduction_pct"] > 0 + assert data["contextpilot_tokens_saved"] == 0 + assert data["contextpilot_token_status"] == "unavailable" + assert data["estimated_input_token_reduction_pct"] == 0 + assert "~100 tokens" not in md + assert "tokenizer): unavailable" in md assert "raw-session-id" not in md assert "DO NOT READ ME" not in md assert "SECRET SYSTEM PROMPT" not in md assert data["top_token_sessions"][0]["session_hash"] != "raw-session-id" +def test_parse_contextpilot_savings_accepts_char_only_log(tmp_path): + log = tmp_path / "gateway.log" + log.write_text( + "2026-01-01 INFO [ContextPilot] Turn 2: saved 400 chars | " + "cumulative: 400 chars\n", + encoding="utf-8", + ) + + events, chars, tokens = monitor.parse_contextpilot_savings(log, since_hours=24) + + assert events == 1 + assert chars == 400 + assert tokens == 0 + + def _write_telemetry(path, records): path.write_text( "\n".join(json.dumps(r) for r in records) + "\n", encoding="utf-8" @@ -127,9 +145,9 @@ def test_parse_telemetry_aggregates_recent_records(tmp_path): tel, [ {"ts": far_future, "type": "turn", "session": "s1", "turn": 1, - "chars_saved": 400, "tokens_saved": 100}, + "chars_saved": 400, "actual_token_status": "available", "actual_tokens_saved": 100}, {"ts": far_future, "type": "turn", "session": "s1", "turn": 2, - "chars_saved": 200, "tokens_saved": 50}, + "chars_saved": 200, "actual_token_status": "available", "actual_tokens_saved": 50}, # Stale record far in the past must be excluded by the window. {"ts": 1000.0, "type": "turn", "session": "s0", "turn": 1, "chars_saved": 999999, "tokens_saved": 999999}, @@ -137,14 +155,15 @@ def test_parse_telemetry_aggregates_recent_records(tmp_path): ], ) - events, chars, tokens = monitor.parse_contextpilot_telemetry(tel, since_hours=24) + events, chars, tokens, token_events = monitor.parse_contextpilot_telemetry(tel, since_hours=24) assert events == 2 assert chars == 600 assert tokens == 150 + assert token_events == 2 def test_parse_telemetry_missing_file_is_safe(tmp_path): - assert monitor.parse_contextpilot_telemetry(tmp_path / "nope.jsonl", since_hours=24) == (0, 0, 0) + assert monitor.parse_contextpilot_telemetry(tmp_path / "nope.jsonl", since_hours=24) == (0, 0, 0, 0) def test_build_report_prefers_telemetry_over_logs(tmp_path): @@ -157,7 +176,7 @@ def test_build_report_prefers_telemetry_over_logs(tmp_path): date="2100-01-01", since_hours=24, log_stats=(5, 4000, 1000), - telemetry_stats=(2, 600, 150), + telemetry_stats=(2, 600, 150, 2), ) # Telemetry is authoritative when present; logs are not summed on top. assert report.contextpilot_tokens_saved == 150 @@ -165,6 +184,7 @@ def test_build_report_prefers_telemetry_over_logs(tmp_path): assert report.contextpilot_telemetry_events == 2 assert report.contextpilot_log_events == 5 assert report.contextpilot_savings_source == "telemetry" + assert report.contextpilot_token_status == "available" def test_build_report_falls_back_to_logs_without_telemetry(tmp_path): @@ -177,7 +197,8 @@ def test_build_report_falls_back_to_logs_without_telemetry(tmp_path): date="2100-01-01", since_hours=24, log_stats=(5, 4000, 1000), - telemetry_stats=(0, 0, 0), + telemetry_stats=(0, 0, 0, 0), ) - assert report.contextpilot_tokens_saved == 1000 - assert report.contextpilot_savings_source == "gateway-log" + assert report.contextpilot_tokens_saved == 0 + assert report.contextpilot_token_status == "unavailable" + assert report.contextpilot_savings_source == "gateway-log-chars-only" diff --git a/tests/test_hermes_plugin_patch.py b/tests/test_hermes_plugin_patch.py index 74bcd19..9366962 100644 --- a/tests/test_hermes_plugin_patch.py +++ b/tests/test_hermes_plugin_patch.py @@ -228,7 +228,6 @@ def test_optimize_writes_metadata_only_telemetry_line(monkeypatch, tmp_path): # Numeric/metadata only — savings recorded. assert record["chars_saved"] > 0 - assert record["tokens_saved"] == record["chars_saved"] // 4 assert record["turn"] == 1 assert record["session_hash"] == module._hash_text("session-XYZ") assert "session" not in record @@ -242,6 +241,84 @@ def test_optimize_writes_metadata_only_telemetry_line(monkeypatch, tmp_path): assert forbidden.isdisjoint(record.keys()) +def test_telemetry_records_payload_chars_and_unavailable_tokenizer_status(monkeypatch, tmp_path): + """Before/after payload chars are actual; no char/4 token proxy is emitted.""" + import json + + module, _ = _load_plugin_module(monkeypatch) + monkeypatch.setattr(module, "_check_reorder", lambda: False) + monkeypatch.setattr(module, "_CONTEXTPILOT_AVAILABLE", False) + monkeypatch.setattr(module, "dedup_chat_completions", _saving_dedup) + # Force the exact tokenizer OFF so this case is deterministic everywhere. + monkeypatch.setenv("CONTEXTPILOT_DISABLE_EXACT_TOKENIZER", "1") + + telemetry = tmp_path / "telemetry.jsonl" + monkeypatch.setenv("CONTEXTPILOT_TELEMETRY_FILE", str(telemetry)) + + engine = module.ContextPilotEngine() + messages = [ + {"role": "user", "content": "read file"}, + {"role": "tool", "tool_call_id": "call_1", "content": "FULL TOOL RESULT"}, + ] + _out, stats = engine.optimize_api_messages(messages) + + record = json.loads(telemetry.read_text(encoding="utf-8").splitlines()[0]) + + # Actual processed-payload before/after char measurement. + assert record["payload_chars_before"] > record["payload_chars_after"] + assert ( + record["payload_chars_saved"] + == record["payload_chars_before"] - record["payload_chars_after"] + ) + # No tokenizer -> a clear status and NO fabricated token numbers. + assert "tokens_saved" not in record + assert "tokens_saved_method" not in record + assert record["actual_token_status"] == "unavailable" + assert "actual_tokens_before" not in record + assert "actual_tokens_after" not in record + assert "actual_tokens_saved" not in record + # Returned stats expose the same payload-char measurement. + assert stats["payload_chars_saved"] == record["payload_chars_saved"] + + +def test_telemetry_records_exact_tokens_when_backend_available(monkeypatch, tmp_path): + """When an exact tokenizer backend is present, actual token fields are emitted.""" + import json + + module, _ = _load_plugin_module(monkeypatch) + monkeypatch.setattr(module, "_check_reorder", lambda: False) + monkeypatch.setattr(module, "_CONTEXTPILOT_AVAILABLE", False) + monkeypatch.setattr(module, "dedup_chat_completions", _saving_dedup) + + # Inject a deterministic fake exact tokenizer (1 token per 3 chars). + def fake_counter(text): + return len(text) // 3 + + fake_counter._backend = "fake:test-encoding" + monkeypatch.setattr(module, "_get_exact_tokenizer", lambda: fake_counter) + + telemetry = tmp_path / "telemetry.jsonl" + monkeypatch.setenv("CONTEXTPILOT_TELEMETRY_FILE", str(telemetry)) + + engine = module.ContextPilotEngine() + messages = [ + {"role": "user", "content": "read file"}, + {"role": "tool", "tool_call_id": "call_1", "content": "FULL TOOL RESULT"}, + ] + engine.optimize_api_messages(messages) + + record = json.loads(telemetry.read_text(encoding="utf-8").splitlines()[0]) + + assert record["actual_token_status"] == "available" + assert record["actual_tokenizer_backend"] == "fake:test-encoding" + assert record["actual_tokens_before"] >= record["actual_tokens_after"] + assert ( + record["actual_tokens_saved"] + == record["actual_tokens_before"] - record["actual_tokens_after"] + ) + assert "tokens_saved_method" not in record + + def test_optimize_telemetry_skipped_when_nothing_saved(monkeypatch, tmp_path): module, _ = _load_plugin_module(monkeypatch) monkeypatch.setattr(module, "_check_reorder", lambda: False) @@ -284,3 +361,236 @@ def test_optimize_survives_unwritable_telemetry_path(monkeypatch, tmp_path): out, stats = engine.optimize_api_messages(messages) assert out[1]["content"] == "REF" assert stats["chars_saved"] > 0 + + +def _zero_dedup(body, **kwargs): + return SimpleNamespace( + chars_saved=0, + blocks_deduped=0, + blocks_total=0, + system_blocks_matched=0, + ) + + +def test_prompt_dedup_canary_default_off_does_not_mutate_runtime(monkeypatch, tmp_path): + module, _ = _load_plugin_module(monkeypatch) + monkeypatch.setattr(module, "_check_reorder", lambda: False) + monkeypatch.setattr(module, "_CONTEXTPILOT_AVAILABLE", False) + monkeypatch.setattr(module, "dedup_chat_completions", _zero_dedup) + telemetry = tmp_path / "telemetry.jsonl" + monkeypatch.setenv("CONTEXTPILOT_TELEMETRY_FILE", str(telemetry)) + monkeypatch.delenv("CONTEXTPILOT_PROMPT_DEDUP_MODE", raising=False) + + repeated = ( + "Reusable examples paragraph for skill notes with enough descriptive filler " + "to make the reference shorter than the duplicate body in this test." + ) + content = f"Use this skill when testing.\n{repeated}\n{repeated}" + engine = module.ContextPilotEngine() + out, stats = engine.optimize_api_messages([{"role": "system", "content": content}]) + + assert out[0]["content"] == content + assert stats["prompt_dedup_mode"] == "off" + assert stats["prompt_dedup_chars_saved"] == 0 + assert not telemetry.exists() + + +def test_prompt_dedup_canary_mutates_only_skill_prompt_runtime(monkeypatch, tmp_path): + import json + + module, _ = _load_plugin_module(monkeypatch) + monkeypatch.setattr(module, "_check_reorder", lambda: False) + monkeypatch.setattr(module, "_CONTEXTPILOT_AVAILABLE", False) + monkeypatch.setattr(module, "dedup_chat_completions", _zero_dedup) + monkeypatch.setenv("CONTEXTPILOT_PROMPT_DEDUP_MODE", "canary") + telemetry = tmp_path / "telemetry.jsonl" + monkeypatch.setenv("CONTEXTPILOT_TELEMETRY_FILE", str(telemetry)) + + repeated = ( + "Reusable examples paragraph for skill notes with enough descriptive filler " + "to make the reference shorter than the duplicate body in this test." + ) + skill_content = f"Use this skill when testing.\n{repeated}\n{repeated}" + ordinary_system = "ordinary system heading\nordinary system text stays untouched" + user_content = f"{repeated}\n{repeated}" + + engine = module.ContextPilotEngine() + out, stats = engine.optimize_api_messages( + [ + {"role": "system", "content": skill_content}, + {"role": "system", "content": ordinary_system}, + {"role": "user", "content": user_content}, + ] + ) + + assert repeated in out[0]["content"] # first occurrence kept + assert out[0]["content"].count(repeated) == 1 + assert "ContextPilot dedup: duplicate skill_prompt block omitted" in out[0]["content"] + # Ordinary system and user content are untouched. + assert out[1]["content"] == ordinary_system + assert out[2]["content"] == user_content + assert stats["prompt_dedup_mode"] == "canary" + assert stats["prompt_dedup_blocks_replaced"] == 1 + assert stats["prompt_dedup_chars_saved"] > 0 + assert stats["chars_saved"] == stats["prompt_dedup_chars_saved"] + + record = json.loads(telemetry.read_text(encoding="utf-8").splitlines()[0]) + assert record["prompt_dedup_mode"] == "canary" + assert record["prompt_dedup_class"] == "same_type_skill_prompt_only" + assert record["prompt_dedup_blocks_replaced"] == 1 + assert record["prompt_dedup_chars_saved"] == stats["prompt_dedup_chars_saved"] + raw = telemetry.read_text(encoding="utf-8") + assert repeated not in raw + assert "Use this skill" not in raw + + +def test_prompt_dedup_canary_does_not_replace_cross_type_or_denylisted_runtime(monkeypatch): + module, _ = _load_plugin_module(monkeypatch) + monkeypatch.setattr(module, "_check_reorder", lambda: False) + monkeypatch.setattr(module, "_CONTEXTPILOT_AVAILABLE", False) + monkeypatch.setattr(module, "dedup_chat_completions", _zero_dedup) + monkeypatch.setenv("CONTEXTPILOT_PROMPT_DEDUP_MODE", "canary") + + cross = ( + "Shared examples paragraph across prompts with enough descriptive filler " + "to be tempting but cross hierarchy should stay unchanged." + ) + denied = ( + "This duplicate line contains secret handling details and enough filler " + "to be long but should be blocked by denylist." + ) + skill_content = f"Use this skill when testing.\n{cross}\n{denied}\n{denied}" + ordinary_system = f"ordinary system heading\n{cross}" + + engine = module.ContextPilotEngine() + out, stats = engine.optimize_api_messages( + [ + {"role": "system", "content": skill_content}, + {"role": "system", "content": ordinary_system}, + ] + ) + + assert out[0]["content"] == skill_content + assert out[1]["content"] == ordinary_system + assert stats["prompt_dedup_chars_saved"] == 0 + assert stats["prompt_dedup_blocks_replaced"] == 0 + + + +def test_artifact_dedup_canary_default_off_does_not_mutate_runtime(monkeypatch, tmp_path): + module, _ = _load_plugin_module(monkeypatch) + monkeypatch.setattr(module, "_check_reorder", lambda: False) + monkeypatch.setattr(module, "_CONTEXTPILOT_AVAILABLE", False) + monkeypatch.setattr(module, "dedup_chat_completions", _zero_dedup) + monkeypatch.delenv("CONTEXTPILOT_ARTIFACT_DEDUP_MODE", raising=False) + telemetry = tmp_path / "telemetry.jsonl" + monkeypatch.setenv("CONTEXTPILOT_TELEMETRY_FILE", str(telemetry)) + + repeated = "pytest terminal output line showing repeated failure details\n" * 12 + messages = [ + {"role": "tool", "tool_call_id": "call_1", "content": repeated}, + {"role": "tool", "tool_call_id": "call_2", "content": repeated}, + ] + + engine = module.ContextPilotEngine() + out, stats = engine.optimize_api_messages(messages) + + assert out[0]["content"] == repeated + assert out[1]["content"] == repeated + assert stats["artifact_dedup_mode"] == "off" + assert stats["artifact_dedup_chars_saved"] == 0 + assert not telemetry.exists() + + +def test_artifact_dedup_canary_mutates_repeated_tool_artifacts_runtime(monkeypatch, tmp_path): + import json + + module, _ = _load_plugin_module(monkeypatch) + monkeypatch.setattr(module, "_check_reorder", lambda: False) + monkeypatch.setattr(module, "_CONTEXTPILOT_AVAILABLE", False) + monkeypatch.setattr(module, "dedup_chat_completions", _zero_dedup) + monkeypatch.setenv("CONTEXTPILOT_ARTIFACT_DEDUP_MODE", "canary") + telemetry = tmp_path / "telemetry.jsonl" + monkeypatch.setenv("CONTEXTPILOT_TELEMETRY_FILE", str(telemetry)) + + repeated = "pytest terminal output line showing repeated failure details\n" * 12 + user_same = repeated + messages = [ + {"role": "tool", "tool_call_id": "call_1", "content": repeated}, + {"role": "assistant", "content": "ordinary assistant response stays untouched"}, + {"role": "tool", "tool_call_id": "call_2", "content": repeated}, + {"role": "user", "content": user_same}, + ] + + engine = module.ContextPilotEngine() + out, stats = engine.optimize_api_messages(messages) + + assert out[0]["content"] == repeated # canonical full copy kept + assert out[1]["content"] == "ordinary assistant response stays untouched" + assert "ContextPilot artifact dedup: duplicate" in out[2]["content"] + assert repeated not in out[2]["content"] + assert out[3]["content"] == user_same # protected same text is untouched + assert stats["artifact_dedup_mode"] == "canary" + assert stats["artifact_dedup_blocks_replaced"] == 1 + assert stats["artifact_dedup_chars_saved"] > 0 + assert stats["chars_saved"] == stats["artifact_dedup_chars_saved"] + + record = json.loads(telemetry.read_text(encoding="utf-8").splitlines()[0]) + assert record["artifact_dedup_mode"] == "canary" + assert record["artifact_dedup_class"] == "same_payload_exact_artifact_body" + assert record["artifact_dedup_blocks_replaced"] == 1 + assert record["artifact_dedup_chars_saved"] == stats["artifact_dedup_chars_saved"] + raw = telemetry.read_text(encoding="utf-8") + assert repeated not in raw + assert "pytest terminal output" not in raw + + +def test_artifact_dedup_canary_runs_when_contextpilot_package_init_unimportable( + monkeypatch, tmp_path +): + """Regression: the canary must load via direct-file loading even when the + ``contextpilot`` package ``__init__`` cannot be imported (e.g. scipy missing + in the Hermes/plugin runtime). Previously the apply helpers imported + ``contextpilot.hermes_opportunities.*`` directly, which executed the heavy + package ``__init__`` and silently fell back to ``artifact_dedup_mode=off``. + """ + import builtins + + module, _ = _load_plugin_module(monkeypatch) + monkeypatch.setattr(module, "_check_reorder", lambda: False) + monkeypatch.setattr(module, "_CONTEXTPILOT_AVAILABLE", False) + monkeypatch.setattr(module, "dedup_chat_completions", _zero_dedup) + monkeypatch.setenv("CONTEXTPILOT_ARTIFACT_DEDUP_MODE", "canary") + telemetry = tmp_path / "telemetry.jsonl" + monkeypatch.setenv("CONTEXTPILOT_TELEMETRY_FILE", str(telemetry)) + + # Force a fresh load attempt and simulate the unimportable package. + monkeypatch.setattr(module, "_canary_modules", None) + for mod_name in list(sys.modules): + if mod_name == "contextpilot" or mod_name.startswith("contextpilot."): + monkeypatch.delitem(sys.modules, mod_name, raising=False) + + real_import = builtins.__import__ + + def _poisoned_import(name, *args, **kwargs): + if name == "contextpilot" or name.startswith("contextpilot."): + raise ImportError("simulated: contextpilot package __init__ (scipy) unavailable") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _poisoned_import) + + repeated = "pytest terminal output line showing repeated failure details\n" * 12 + messages = [ + {"role": "tool", "tool_call_id": "call_1", "content": repeated}, + {"role": "tool", "tool_call_id": "call_2", "content": repeated}, + ] + + engine = module.ContextPilotEngine() + out, stats = engine.optimize_api_messages(messages) + + assert out[0]["content"] == repeated # canonical full copy kept + assert "ContextPilot artifact dedup: duplicate" in out[1]["content"] + assert repeated not in out[1]["content"] + assert stats["artifact_dedup_mode"] == "canary" + assert stats["artifact_dedup_blocks_replaced"] == 1 + assert stats["artifact_dedup_chars_saved"] > 0 diff --git a/tests/test_prompt_dedup_canary.py b/tests/test_prompt_dedup_canary.py new file mode 100644 index 0000000..6bf8d91 --- /dev/null +++ b/tests/test_prompt_dedup_canary.py @@ -0,0 +1,304 @@ +"""Tests for the default-off prompt-dedup canary (runtime prompt mutation). + +The canary is the only place ContextPilot may actually rewrite prompt text. These +tests pin the safety gate: default off (no mutation), canary touches ONLY +same_type_skill_prompt_only duplicates (first occurrence kept), never touches +system/cross-type/user/assistant/tool content, honours the safety denylist and +the escape-hatch kill switch, never grows the payload, and emits metadata-only +telemetry with no raw prompt text. +""" +import json + +from contextpilot.hermes_opportunities import ( + CANARY_DEDUP_CLASS, + PROMPT_DEDUP_DISABLE_ENV, + PROMPT_DEDUP_MODE_ENV, + apply_prompt_dedup_canary, + build_canary_telemetry_record, + resolve_prompt_dedup_mode, + _LLMContent, +) +from contextpilot.hermes_opportunities.prompt_dedup_canary import ( + _reference_string, + _salted_hash, +) + +SALT = "test-salt" +MIN = 40 + +# A benign skill block, comfortably longer than the reference placeholder so a +# replacement actually saves characters, and free of any denylist keyword. +LONG_SKILL = ( + "Example reusable skill paragraph describing how the helper reformats " + "markdown tables into neat aligned columns for the reader." +) +SYS_BLOCK = ( + "Plain system narration paragraph describing the assistant persona and the " + "general tone it should adopt across replies." +) + + +def _ref_len() -> int: + return len(_reference_string("skill_prompt", _salted_hash(LONG_SKILL, SALT))) + + +# --------------------------------------------------------------------------- +# Mode resolution + escape hatch +# --------------------------------------------------------------------------- + + +def test_mode_defaults_to_off(monkeypatch): + monkeypatch.delenv(PROMPT_DEDUP_MODE_ENV, raising=False) + monkeypatch.delenv(PROMPT_DEDUP_DISABLE_ENV, raising=False) + assert resolve_prompt_dedup_mode() == "off" + + +def test_mode_reads_env_values(): + assert resolve_prompt_dedup_mode({PROMPT_DEDUP_MODE_ENV: "shadow"}) == "shadow" + assert resolve_prompt_dedup_mode({PROMPT_DEDUP_MODE_ENV: "CANARY"}) == "canary" + # Unknown / garbage values fall back to the safe default. + assert resolve_prompt_dedup_mode({PROMPT_DEDUP_MODE_ENV: "aggressive"}) == "off" + + +def test_disable_env_is_a_kill_switch(): + env = {PROMPT_DEDUP_MODE_ENV: "canary", PROMPT_DEDUP_DISABLE_ENV: "1"} + assert resolve_prompt_dedup_mode(env) == "off" + + +# --------------------------------------------------------------------------- +# Default off never mutates +# --------------------------------------------------------------------------- + + +def test_default_off_does_not_change_payload(monkeypatch): + monkeypatch.delenv(PROMPT_DEDUP_MODE_ENV, raising=False) + monkeypatch.delenv(PROMPT_DEDUP_DISABLE_ENV, raising=False) + contents = [_LLMContent("skill_prompt", f"{LONG_SKILL}\n{LONG_SKILL}\n{LONG_SKILL}")] + before = contents[0].content + result = apply_prompt_dedup_canary(contents, salt=SALT, min_block_chars=MIN) + assert result.mode == "off" + assert result.mutated is False + assert result.blocks_replaced == 0 + assert result.chars_saved == 0 + assert contents[0].content == before # payload byte-identical + + +def test_disable_env_blocks_mutation_even_with_canary_set(monkeypatch): + monkeypatch.setenv(PROMPT_DEDUP_MODE_ENV, "canary") + monkeypatch.setenv(PROMPT_DEDUP_DISABLE_ENV, "true") + contents = [_LLMContent("skill_prompt", f"{LONG_SKILL}\n{LONG_SKILL}")] + before = contents[0].content + result = apply_prompt_dedup_canary(contents, salt=SALT, min_block_chars=MIN) + assert result.mode == "off" + assert contents[0].content == before + + +# --------------------------------------------------------------------------- +# Canary replaces only same_type_skill_prompt_only duplicates +# --------------------------------------------------------------------------- + + +def test_canary_replaces_later_skill_duplicates_keeps_first(): + contents = [_LLMContent("skill_prompt", f"{LONG_SKILL}\n{LONG_SKILL}\n{LONG_SKILL}")] + result = apply_prompt_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + lines = contents[0].content.split("\n") + assert lines[0] == LONG_SKILL # first occurrence kept verbatim + assert lines[1] != LONG_SKILL and lines[2] != LONG_SKILL # later ones replaced + assert lines[1] == lines[2] # deterministic reference string + assert result.mode == "canary" + assert result.mutated is True + assert result.blocks_replaced == 2 + assert result.prompt_dedup_class == CANARY_DEDUP_CLASS + assert result.chars_saved == 2 * (len(LONG_SKILL) - _ref_len()) + + +def test_canary_replacement_carries_no_raw_content(): + contents = [_LLMContent("skill_prompt", f"{LONG_SKILL}\n{LONG_SKILL}")] + apply_prompt_dedup_canary(contents, salt=SALT, min_block_chars=MIN, mode="canary") + ref_line = contents[0].content.split("\n")[1] + # The reference holds only a type enum + salted hash, never the block text. + assert "skill_prompt" in ref_line + assert LONG_SKILL not in ref_line + + +def test_canary_replicates_across_two_skill_items(): + # Same block in two separate skill_prompt items: first item keeps it, the + # occurrence in the second item is replaced. + a = _LLMContent("skill_prompt", LONG_SKILL) + b = _LLMContent("skill_prompt", LONG_SKILL) + result = apply_prompt_dedup_canary( + [a, b], salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert a.content == LONG_SKILL + assert b.content != LONG_SKILL + assert result.blocks_replaced == 1 + + +# --------------------------------------------------------------------------- +# Canary must NOT touch other classes / roles +# --------------------------------------------------------------------------- + + +def test_canary_leaves_system_only_duplicates_untouched(): + contents = [_LLMContent("system_prompt", f"{SYS_BLOCK}\n{SYS_BLOCK}\n{SYS_BLOCK}")] + before = contents[0].content + result = apply_prompt_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert contents[0].content == before + assert result.blocks_replaced == 0 + assert result.candidate_block_count == 0 + + +def test_canary_leaves_cross_type_duplicates_untouched(): + # The same block appears in BOTH a system and a skill prompt -> cross-type, + # never eligible for the skill-only canary. + skill = _LLMContent("skill_prompt", f"{LONG_SKILL}\n{LONG_SKILL}") + system = _LLMContent("system_prompt", LONG_SKILL) + skill_before = skill.content + result = apply_prompt_dedup_canary( + [skill, system], salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert skill.content == skill_before + assert result.blocks_replaced == 0 + assert result.candidate_block_count == 0 + + +def test_canary_leaves_user_assistant_tool_untouched(): + contents = [ + _LLMContent("user_prompt", f"{LONG_SKILL}\n{LONG_SKILL}"), + _LLMContent("assistant_context", f"{LONG_SKILL}\n{LONG_SKILL}"), + _LLMContent("tool_result", f"{LONG_SKILL}\n{LONG_SKILL}"), + ] + befores = [c.content for c in contents] + result = apply_prompt_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert [c.content for c in contents] == befores + assert result.blocks_replaced == 0 + # Non system/skill items are not even scanned for candidates. + assert result.item_count == 0 + + +# --------------------------------------------------------------------------- +# Safety denylist + payload-growth guard +# --------------------------------------------------------------------------- + + +def test_denylisted_blocks_are_not_replaced(): + danger = ( + "You must always follow this required safety rule precisely and never " + "skip it under any circumstances whatsoever here." + ) + contents = [_LLMContent("skill_prompt", f"{danger}\n{danger}\n{danger}")] + before = contents[0].content + result = apply_prompt_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert contents[0].content == before # untouched + assert result.blocks_replaced == 0 + assert result.denylisted_block_count == 1 + + +def test_canary_never_grows_payload_for_short_duplicates(): + # A duplicate shorter than the reference placeholder would grow if replaced, + # so it is left alone. + short = "Short but over forty chars skill helper line." + assert len(short) < _ref_len() + contents = [_LLMContent("skill_prompt", f"{short}\n{short}\n{short}")] + before = contents[0].content + result = apply_prompt_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + assert contents[0].content == before + assert result.blocks_replaced == 0 + assert result.chars_saved == 0 + + +# --------------------------------------------------------------------------- +# Shadow mode measures but does not mutate +# --------------------------------------------------------------------------- + + +def test_shadow_measures_candidates_without_mutating(): + contents = [_LLMContent("skill_prompt", f"{LONG_SKILL}\n{LONG_SKILL}\n{LONG_SKILL}")] + before = contents[0].content + result = apply_prompt_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="shadow" + ) + assert contents[0].content == before # never mutated + assert result.mode == "shadow" + assert result.mutated is False + assert result.blocks_replaced == 0 + assert result.candidate_block_count == 1 + assert result.candidate_chars == 2 * len(LONG_SKILL) + + +# --------------------------------------------------------------------------- +# Telemetry: metadata-only, savings only when a real mutation happened +# --------------------------------------------------------------------------- + + +def test_telemetry_records_no_savings_when_off(): + contents = [_LLMContent("skill_prompt", f"{LONG_SKILL}\n{LONG_SKILL}")] + result = apply_prompt_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="off" + ) + record = build_canary_telemetry_record(result) + assert record["prompt_dedup_mode"] == "off" + assert record["prompt_dedup_chars_saved"] == 0 + assert record["prompt_dedup_blocks_replaced"] == 0 + assert record["chars_saved"] == 0 + + +def test_telemetry_records_no_savings_in_shadow(): + contents = [_LLMContent("skill_prompt", f"{LONG_SKILL}\n{LONG_SKILL}")] + result = apply_prompt_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="shadow" + ) + record = build_canary_telemetry_record(result) + assert record["prompt_dedup_mode"] == "shadow" + # Shadow contributes nothing to the realized chars_saved total. + assert record["chars_saved"] == 0 + assert record["prompt_dedup_chars_saved"] == 0 + + +def test_telemetry_records_realized_savings_in_canary(): + contents = [_LLMContent("skill_prompt", f"{LONG_SKILL}\n{LONG_SKILL}\n{LONG_SKILL}")] + result = apply_prompt_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + record = build_canary_telemetry_record(result) + expected = 2 * (len(LONG_SKILL) - _ref_len()) + assert record["prompt_dedup_mode"] == "canary" + assert record["prompt_dedup_class"] == CANARY_DEDUP_CLASS + assert record["prompt_dedup_blocks_replaced"] == 2 + assert record["prompt_dedup_chars_saved"] == expected + # The aggregate counter includes prompt dedup only because a mutation occurred. + assert record["chars_saved"] == expected + + +def test_telemetry_is_metadata_only_no_prompt_text(): + contents = [_LLMContent("skill_prompt", f"{LONG_SKILL}\n{LONG_SKILL}")] + result = apply_prompt_dedup_canary( + contents, salt=SALT, min_block_chars=MIN, mode="canary" + ) + record = build_canary_telemetry_record(result) + blob = json.dumps(record) + assert LONG_SKILL not in blob + # Only low-cardinality enums + integer counters are present. + assert set(record) == { + "prompt_dedup_mode", + "prompt_dedup_class", + "prompt_dedup_blocks_replaced", + "prompt_dedup_chars_saved", + "chars_saved", + } + for key in ( + "prompt_dedup_blocks_replaced", + "prompt_dedup_chars_saved", + "chars_saved", + ): + assert isinstance(record[key], int) diff --git a/tests/test_provenance_linking_shadow.py b/tests/test_provenance_linking_shadow.py new file mode 100644 index 0000000..f79b494 --- /dev/null +++ b/tests/test_provenance_linking_shadow.py @@ -0,0 +1,110 @@ +import json +import subprocess +import sys +from pathlib import Path + +from contextpilot.provenance_linking import ( + ProvenanceBlock, + ProvenanceExample, + evaluate_shadow, + example_from_trace_case, + extract_claims, + read_jsonl_examples, + shadow_link_claims, +) +from scripts.build_provenance_linker_dataset import synthetic_examples + +REPO_ROOT = Path(__file__).resolve().parents[1] +SYNTHETIC = REPO_ROOT / "datasets/provenance_linking/synthetic_v1.jsonl" + + +def test_claim_extraction_and_shadow_linking_are_offline_only(): + tool = ProvenanceBlock("tool_1", "tool_result", "pytest: 592 passed, 37 skipped") + assistant = ProvenanceBlock( + "assistant_1", + "assistant_context", + "Full suite passed: 592 passed, 37 skipped.", + ) + claims = extract_claims(assistant) + assert len(claims) == 1 + + example = ProvenanceExample("ex", "coding", [tool, assistant], claims) + links = shadow_link_claims(example) + assert len(links) == 1 + assert links[0].evidence_block_id == "tool_1" + assert links[0].method == "shadow_lexical_v1" + # The linker only emits metadata; it never mutates source/claim text. + assert tool.text == "pytest: 592 passed, 37 skipped" + assert assistant.text == "Full suite passed: 592 passed, 37 skipped." + + +def test_synthetic_provenance_dataset_is_training_ready_and_shadow_measured(): + examples = read_jsonl_examples(SYNTHETIC) + assert len(examples) == 3 + assert sum(len(ex.gold_links) for ex in examples) == 4 + assert {ex.domain for ex in examples} == {"coding", "research", "ops"} + + report = evaluate_shadow(examples) + assert report["claim_scope"] == "shadow claim→evidence linking; does not mutate online context" + assert report["shadow_link_tp"] >= 3 + assert report["shadow_link_precision"] >= 0.7 + assert report["shadow_link_recall"] >= 0.7 + # This is deliberately a baseline, not a solved model: enough signal to train, + # but not pretending rules solve the general provenance problem. + assert report["shadow_link_precision"] < 1.0 or report["shadow_link_recall"] < 1.0 + + +def test_dataset_builder_and_shadow_eval_clis(tmp_path): + dataset = tmp_path / "prov.jsonl" + built = subprocess.run( + [ + sys.executable, + "scripts/build_provenance_linker_dataset.py", + "--mode", + "synthetic", + "--with-shadow", + "--output", + str(dataset), + ], + cwd=REPO_ROOT, + text=True, + capture_output=True, + check=True, + ) + manifest = json.loads(built.stdout) + assert manifest["dataset_kind"] == "provenance_linking" + assert manifest["gold_link_count"] == 4 + assert dataset.exists() + assert dataset.with_suffix(dataset.suffix + ".manifest.json").exists() + + report_path = tmp_path / "report.json" + evaluated = subprocess.run( + [sys.executable, "scripts/evaluate_provenance_shadow.py", str(dataset), "--output", str(report_path)], + cwd=REPO_ROOT, + text=True, + capture_output=True, + check=True, + ) + assert json.loads(evaluated.stdout) == json.loads(report_path.read_text()) + + +def test_trace_case_converts_to_general_provenance_example(): + raw = { + "case_id": "case_1", + "messages": [ + {"role": "tool", "block_type": "tool_result", "content": "pytest says tests passed: 46 passed"}, + {"role": "assistant", "block_type": "assistant_context", "content": "Tests passed: 46 passed."}, + ], + } + example = example_from_trace_case(raw) + assert example.example_id == "case_1" + assert len(example.blocks) == 2 + assert len(example.claims) == 1 + assert shadow_link_claims(example) + + +def test_synthetic_examples_match_committed_dataset_shape(): + generated = synthetic_examples() + committed = read_jsonl_examples(SYNTHETIC) + assert [ex.example_id for ex in generated] == [ex.example_id for ex in committed] + assert [len(ex.gold_links) for ex in generated] == [len(ex.gold_links) for ex in committed] diff --git a/tests/test_trace_validation_builder.py b/tests/test_trace_validation_builder.py new file mode 100644 index 0000000..05e3f43 --- /dev/null +++ b/tests/test_trace_validation_builder.py @@ -0,0 +1,220 @@ +"""Tests for the trace-validation-set builder: redaction, privacy, sampling. + +The builder reads a Hermes-shaped SQLite DB read-only and exports a fixed JSONL +corpus (raw content, local-only) plus a privacy-safe manifest. These tests pin: +case ids are salted (never the raw session id); the manifest is metadata-only +(no raw content) and passes the forbidden-key guard; conservative sampling +honours --limit / --min-input-tokens / --min-messages; --no-system-prompt drops +system/skill prompts; and the corpus is the only place raw content appears. +""" +import json +import sqlite3 +from pathlib import Path + +from contextpilot.hermes_opportunities.privacy import _salt_fingerprint, _salted_hash +from contextpilot.trace_validation.builder import ( + build_manifest, + load_trace_cases, + main as build_main, + write_validation_set, +) + +SALT = "test-trace-salt" + +SKILL_LINE = ( + "Synthetic reusable skill paragraph that explains how the demo helper " + "reformats sample markdown tables into neat aligned columns for readers." +) +USER_LINE = "Please summarize the synthetic figures for the demo run." +TOOL_LINE = '{"synthetic_metric": 42, "label": "demo only"}' +SYS_LINE = "Synthetic system narration describing the demo persona and tone." + + +def _make_db(path: Path) -> None: + """Create a minimal Hermes-shaped DB with two sessions.""" + conn = sqlite3.connect(path) + conn.execute( + "CREATE TABLE sessions (id TEXT, source TEXT, input_tokens INTEGER, " + "system_prompt TEXT, started_at REAL, archived INTEGER, message_count INTEGER)" + ) + conn.execute( + "CREATE TABLE messages (id INTEGER PRIMARY KEY, session_id TEXT, role TEXT, " + "content TEXT, tool_name TEXT, timestamp REAL, active INTEGER)" + ) + now = 1_900_000_000.0 + conn.execute( + "INSERT INTO sessions VALUES (?,?,?,?,?,?,?)", + ("sess-heavy", "synthetic", 9000, SYS_LINE, now, 0, 3), + ) + conn.execute( + "INSERT INTO sessions VALUES (?,?,?,?,?,?,?)", + ("sess-light", "synthetic", 500, None, now, 0, 2), + ) + # Heavy session messages (ordered by id). + conn.execute( + "INSERT INTO messages VALUES (?,?,?,?,?,?,?)", + (1, "sess-heavy", "user", USER_LINE, None, now, 1), + ) + conn.execute( + "INSERT INTO messages VALUES (?,?,?,?,?,?,?)", + (2, "sess-heavy", "tool", TOOL_LINE, "calc", now, 1), + ) + # An inactive message that must be skipped. + conn.execute( + "INSERT INTO messages VALUES (?,?,?,?,?,?,?)", + (3, "sess-heavy", "user", "INACTIVE should not appear", None, now, 0), + ) + conn.execute( + "INSERT INTO messages VALUES (?,?,?,?,?,?,?)", + (4, "sess-light", "user", "light user message for the demo", None, now, 1), + ) + conn.commit() + conn.close() + + +def test_case_ids_are_salted_not_raw(tmp_path): + db = tmp_path / "state.db" + _make_db(db) + cases = load_trace_cases(db, since_hours=24, salt=SALT, limit=10, all_sessions=True) + ids = {c.case_id for c in cases} + assert "sess-heavy" not in ids and "sess-light" not in ids + assert _salted_hash("sess-heavy", SALT) in ids + + +def test_heavy_session_ordered_first_and_inactive_skipped(tmp_path): + db = tmp_path / "state.db" + _make_db(db) + cases = load_trace_cases(db, since_hours=24, salt=SALT, limit=10, all_sessions=True) + # Ordered by input_tokens desc -> heavy first. + assert cases[0].input_tokens == 9000 + heavy = cases[0] + # system(skill/plain) + user + tool, inactive message dropped. + contents = [m.content for m in heavy.messages] + assert SYS_LINE in contents + assert USER_LINE in contents + assert "INACTIVE should not appear" not in contents + + +def test_min_input_tokens_filters_light_session(tmp_path): + db = tmp_path / "state.db" + _make_db(db) + cases = load_trace_cases( + db, since_hours=24, salt=SALT, limit=10, all_sessions=True, min_input_tokens=1000 + ) + assert len(cases) == 1 + assert cases[0].input_tokens == 9000 + + +def test_limit_caps_number_of_cases(tmp_path): + db = tmp_path / "state.db" + _make_db(db) + cases = load_trace_cases(db, since_hours=24, salt=SALT, limit=1, all_sessions=True) + assert len(cases) == 1 + + +def test_no_system_prompt_excludes_system(tmp_path): + db = tmp_path / "state.db" + _make_db(db) + cases = load_trace_cases( + db, + since_hours=24, + salt=SALT, + limit=10, + all_sessions=True, + include_system_prompt=False, + ) + heavy = next(c for c in cases if c.input_tokens == 9000) + assert all(m.role != "system" for m in heavy.messages) + assert SYS_LINE not in [m.content for m in heavy.messages] + + +def test_manifest_is_privacy_safe_no_raw_content(tmp_path): + db = tmp_path / "state.db" + _make_db(db) + cases = load_trace_cases(db, since_hours=24, salt=SALT, limit=10, all_sessions=True) + manifest = build_manifest( + cases, + date="2026-06-14", + salt=SALT, + since_hours=24, + all_sessions=True, + min_input_tokens=0, + include_system_prompt=True, + corpus_filename="corpus.jsonl", + ) + blob = json.dumps(manifest) + # No raw content anywhere in the manifest. + for needle in (SKILL_LINE, USER_LINE, TOOL_LINE, SYS_LINE): + assert needle not in blob + # Carries a salt fingerprint, never the raw salt. + assert manifest["salt_fingerprint"] == _salt_fingerprint(SALT) + assert SALT not in blob + assert manifest["case_count"] == len(cases) + assert manifest["messages_by_block_type"] # counters present + + +def test_write_validation_set_corpus_has_raw_manifest_does_not(tmp_path): + db = tmp_path / "state.db" + _make_db(db) + cases = load_trace_cases(db, since_hours=24, salt=SALT, limit=10, all_sessions=True) + manifest = build_manifest( + cases, + date="2026-06-14", + salt=SALT, + since_hours=24, + all_sessions=True, + min_input_tokens=0, + include_system_prompt=True, + corpus_filename="corpus.jsonl", + ) + corpus_path, manifest_path = write_validation_set( + cases, manifest, tmp_path / "out", "corpus.jsonl" + ) + corpus_text = corpus_path.read_text() + manifest_text = manifest_path.read_text() + # Raw content lives ONLY in the corpus, never the manifest. + assert USER_LINE in corpus_text + assert USER_LINE not in manifest_text + # Each corpus line is a well-formed case object. + for line in corpus_text.splitlines(): + obj = json.loads(line) + assert obj["schema_version"] == 1 + assert "case_id" in obj and "messages" in obj + + +def test_main_emits_privacy_safe_stdout(tmp_path, capsys): + db = tmp_path / "state.db" + _make_db(db) + out = tmp_path / "out" + rc = build_main( + [ + "--state-db", + str(db), + "--out", + str(out), + "--all-sessions", + "--salt", + SALT, + "--date", + "2026-06-14", + ] + ) + assert rc == 0 + printed = capsys.readouterr().out + payload = json.loads(printed) + assert payload["ok"] is True + assert payload["case_count"] == 2 + # Stdout carries paths + counters only, never raw content. + for needle in (SKILL_LINE, USER_LINE, TOOL_LINE, SYS_LINE): + assert needle not in printed + assert Path(payload["corpus"]).exists() + assert Path(payload["manifest"]).exists() + + +def test_main_missing_db_exits(tmp_path): + try: + build_main(["--state-db", str(tmp_path / "nope.db")]) + except SystemExit as exc: + assert exc.code != 0 + else: # pragma: no cover - should always raise + raise AssertionError("expected SystemExit for missing DB") diff --git a/tests/test_trace_validation_runner.py b/tests/test_trace_validation_runner.py new file mode 100644 index 0000000..4a26005 --- /dev/null +++ b/tests/test_trace_validation_runner.py @@ -0,0 +1,143 @@ +"""Tests for the trace-validation runner gate. + +The committed fixture is fully synthetic. The runner may read raw case content +from the local JSONL corpus, but its emitted report must remain privacy-safe and +must fail when mutations touch protected content or savings accounting lies. +""" + +import json +from pathlib import Path + +import pytest + +from contextpilot.hermes_opportunities.prompt_dedup_canary import PromptDedupCanaryResult +from contextpilot.trace_validation.runner import ( + assert_report_privacy_safe, + load_cases, + render_markdown, + report_to_dict, + run_validation, +) + +FIXTURE = Path("tests/fixtures/trace_validation/synthetic_cases.jsonl") +SALT = "test-trace-salt" + + +def test_canary_validation_passes_on_synthetic_fixture(): + cases = load_cases(FIXTURE) + report = run_validation( + cases, + baseline_mode="off", + candidate_mode="canary", + salt=SALT, + min_block_chars=40, + date="2026-06-14", + ) + assert report.passed is True + assert report.failed_cases == 0 + # The first synthetic skill case has safe duplicate lines; the third is + # denylisted by "must" and should remain unchanged. + assert report.total_blocks_replaced == 2 + assert report.total_chars_saved > 0 + assert any(c.mutated for c in report.cases) + assert report.tokenizer_status == "unavailable" + assert report.total_actual_tokens_saved is None + + +def test_shadow_validation_passes_without_realized_savings(): + cases = load_cases(FIXTURE) + report = run_validation( + cases, + baseline_mode="off", + candidate_mode="shadow", + salt=SALT, + min_block_chars=40, + date="2026-06-14", + ) + assert report.passed is True + assert report.total_blocks_replaced == 0 + assert report.total_chars_saved == 0 + assert all(not c.mutated for c in report.cases) + + +def test_report_is_privacy_safe_and_markdown_contains_no_raw_fixture_text(): + cases = load_cases(FIXTURE) + report = run_validation( + cases, + baseline_mode="off", + candidate_mode="canary", + salt=SALT, + min_block_chars=40, + date="2026-06-14", + ) + report_dict = report_to_dict(report) + raw_needles = [ + "Please summarize the synthetic quarterly figures attached above for the demo.", + "Synthetic reusable skill paragraph that explains how the demo helper", + "synthetic_weather=clear", + ] + assert_report_privacy_safe(report_dict, raw_needles) + blob = json.dumps(report_dict, ensure_ascii=False) + md = render_markdown(report) + for needle in raw_needles: + assert needle not in blob + assert needle not in md + + +def test_runner_fails_when_candidate_mutates_protected_user_content(): + cases = load_cases(FIXTURE) + + def bad_optimizer(messages, *, mode, salt, min_block_chars): + out = [dict(m) for m in messages] + if mode == "bad" and out: + for m in out: + if m["block_type"] == "user_prompt": + m["content"] = "[dropped]" + break + result = PromptDedupCanaryResult( + mode="canary", + prompt_dedup_class="same_type_skill_prompt_only", + mutated=True, + item_count=0, + skill_item_count=0, + candidate_block_count=0, + candidate_chars=0, + blocks_replaced=1, + chars_saved=1, + denylisted_block_count=0, + ) + return out, result + result = PromptDedupCanaryResult( + mode="off", + prompt_dedup_class="same_type_skill_prompt_only", + mutated=False, + item_count=0, + skill_item_count=0, + candidate_block_count=0, + candidate_chars=0, + blocks_replaced=0, + chars_saved=0, + denylisted_block_count=0, + ) + return out, result + + report = run_validation( + cases[:1], + baseline_mode="off", + candidate_mode="bad", + salt=SALT, + min_block_chars=40, + date="2026-06-14", + optimize_fn=bad_optimizer, + ) + assert report.passed is False + assert report.failed_cases == 1 + failed = report.cases[0].failed_invariants + assert "protected_content_preserved" in failed + assert "mutation_scope_allowed" in failed + assert "savings_accounting_consistent" in failed + + +def test_privacy_guard_rejects_raw_content_in_report(): + with pytest.raises(RuntimeError): + assert_report_privacy_safe({"ok": True, "note": "raw secret line"}, ["raw secret line"])