feat: consolidate Hermes context optimization and tokenizer telemetry

.gitignore

@@ -5,6 +5,17 @@ dist/
 *.log
 *.jsonl
 *.egg-info/
+
+# Trace-derived validation sets are local-only / private: generated corpora hold
+# raw conversation content and must never be committed. The in-repo convenience
+# dir is ignored; raw corpora also match the *.jsonl rule above.
+.contextpilot_validation/
+# ...but committed SYNTHETIC fixtures (no real trace data) are kept.
+!tests/fixtures/trace_validation/synthetic_cases.jsonl
+!datasets/
+!datasets/provenance_linking/
+!datasets/provenance_linking/synthetic_v1.jsonl
+!datasets/provenance_linking/synthetic_v1.jsonl.manifest.json
 */.DS_Store
 *.DS_Store
 

contextpilot/__init__.py

@@ -17,70 +17,87 @@
     >>> results = pipeline.run(queries=["What is AI?"])
 
 See docs/reference/api.md for detailed documentation.
-"""
-
-from .pipeline import (
-    RAGPipeline,
-    RetrieverConfig,
-    OptimizerConfig,
-    InferenceConfig,
-    PipelineConfig,
-)
-
-from .context_index import (
-    ContextIndex,
-    IndexResult,
-)
-
-from .context_ordering import (
-    IntraContextOrderer,
-)
 
-from .server.live_index import ContextPilot
-
-from .dedup import (
-    dedup_chat_completions,
-    dedup_responses_api,
-    DedupResult,
-)
-
-from .api import optimize, optimize_batch
+Imports are lazy (PEP 562): the heavy RAG stack (``pipeline`` -> ``context_index``
+-> ``scipy``) is only pulled in when one of its names is first accessed. This
+keeps lightweight, dependency-free consumers -- such as the standalone token
+monitor / provenance profiler in :mod:`contextpilot.hermes_opportunities` --
+importable inside minimal environments where SciPy and friends are absent.
+"""
+from __future__ import annotations
 
-from .retriever import (
-    BM25Retriever,
-    FAISSRetriever,
-    FAISS_AVAILABLE,
-    Mem0Retriever,
-    create_mem0_corpus_map,
-    MEM0_AVAILABLE,
-)
+import importlib
+from typing import TYPE_CHECKING
 
 __version__ = "0.4.1"
 
-__all__ = [
+# Map each public name to the submodule that defines it. Submodules are imported
+# on first attribute access, so importing ``contextpilot`` (or any lightweight
+# subpackage like ``hermes_opportunities``) never eagerly drags in SciPy/NumPy.
+_LAZY_EXPORTS = {
     # High-level pipeline API
-    "RAGPipeline",
-    "RetrieverConfig",
-    "OptimizerConfig",
-    "InferenceConfig",
-    "PipelineConfig",
+    "RAGPipeline": ".pipeline",
+    "RetrieverConfig": ".pipeline",
+    "OptimizerConfig": ".pipeline",
+    "InferenceConfig": ".pipeline",
+    "PipelineConfig": ".pipeline",
     # Core components
-    "ContextIndex",
-    "IndexResult",
-    "IntraContextOrderer",
-    "ContextPilot",
+    "ContextIndex": ".context_index",
+    "IndexResult": ".context_index",
+    "IntraContextOrderer": ".context_ordering",
+    "ContextPilot": ".server.live_index",
     # Deduplication
-    "dedup_chat_completions",
-    "dedup_responses_api",
-    "DedupResult",
+    "dedup_chat_completions": ".dedup",
+    "dedup_responses_api": ".dedup",
+    "DedupResult": ".dedup",
     # Convenience functions
-    "optimize",
-    "optimize_batch",
+    "optimize": ".api",
+    "optimize_batch": ".api",
     # Retrievers
-    "BM25Retriever",
-    "FAISSRetriever",
-    "FAISS_AVAILABLE",
-    "Mem0Retriever",
-    "create_mem0_corpus_map",
-    "MEM0_AVAILABLE",
-]
+    "BM25Retriever": ".retriever",
+    "FAISSRetriever": ".retriever",
+    "FAISS_AVAILABLE": ".retriever",
+    "Mem0Retriever": ".retriever",
+    "create_mem0_corpus_map": ".retriever",
+    "MEM0_AVAILABLE": ".retriever",
+}
+
+__all__ = list(_LAZY_EXPORTS)
+
+
+def __getattr__(name: str):
+    """Lazily resolve a public name to its (heavy) submodule on first access."""
+    module_name = _LAZY_EXPORTS.get(name)
+    if module_name is None:
+        raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
+    module = importlib.import_module(module_name, __name__)
+    value = getattr(module, name)
+    globals()[name] = value  # cache so subsequent lookups skip the import machinery
+    return value
+
+
+def __dir__():
+    return sorted(list(globals()) + __all__)
+
+
+if TYPE_CHECKING:  # pragma: no cover - import-time hints for type checkers only
+    from .api import optimize, optimize_batch
+    from .context_index import ContextIndex, IndexResult
+    from .context_ordering import IntraContextOrderer
+    from .dedup import DedupResult, dedup_chat_completions, dedup_responses_api
+    from .pipeline import (
+        InferenceConfig,
+        OptimizerConfig,
+        PipelineConfig,
+        RAGPipeline,
+        RetrieverConfig,
+    )
+    from .retriever import (
+        FAISS_AVAILABLE,
+        MEM0_AVAILABLE,
+        BM25Retriever,
+        FAISSRetriever,
+        Mem0Retriever,
+        create_mem0_corpus_map,
+    )
+    from .server.live_index import ContextPilot

contextpilot/_openai_hook.py

@@ -336,13 +336,12 @@ def _optimize_messages(kwargs):
     if chars_saved > 0 or docs_reordered > 0:
         logger.info(
             "[ContextPilot] Call #%d: %d chars saved, %d blocks deduped, "
-            "%d docs reordered (cumulative: %d chars ≈ %d tokens)",
+            "%d docs reordered (cumulative: %d chars; tokenizer accounting required for tokens)",
             _total_calls,
             chars_saved,
             dedup_result.blocks_deduped,
             docs_reordered,
             _total_chars_saved,
-            _total_chars_saved // 4,
         )
 
 
@@ -369,12 +368,11 @@ def _optimize_responses(kwargs):
     if dedup_result.chars_saved > 0:
         logger.info(
             "[ContextPilot] Responses call #%d: %d chars saved, %d blocks deduped "
-            "(cumulative: %d chars ≈ %d tokens)",
+            "(cumulative: %d chars; tokenizer accounting required for tokens)",
             _total_calls,
             dedup_result.chars_saved,
             dedup_result.blocks_deduped,
             _total_chars_saved,
-            _total_chars_saved // 4,
         )