diff --git a/packages/data-analytics-demo/Makefile b/packages/data-analytics-demo/Makefile index bae9fb2..a2f49e4 100644 --- a/packages/data-analytics-demo/Makefile +++ b/packages/data-analytics-demo/Makefile @@ -33,8 +33,7 @@ ml: $(PYTHON) -m data_analytics_demo.ml.upsell narrative: - @echo "[narrative] TODO T-08: Ollama narrative not yet implemented" - @exit 1 + $(PYTHON) -m data_analytics_demo.narrative.generate dashboard: @echo "[dashboard] TODO T-09: Evidence dashboard not yet implemented" diff --git a/packages/data-analytics-demo/src/data_analytics_demo/cli.py b/packages/data-analytics-demo/src/data_analytics_demo/cli.py index 71e850b..1596e6e 100644 --- a/packages/data-analytics-demo/src/data_analytics_demo/cli.py +++ b/packages/data-analytics-demo/src/data_analytics_demo/cli.py @@ -6,8 +6,6 @@ from __future__ import annotations -import sys - import typer app = typer.Typer( @@ -49,9 +47,11 @@ def ml() -> None: @app.command() def narrative() -> None: - """Generate LLM narrative via local Ollama (T-08, not yet implemented).""" - typer.echo("[narrative] TODO T-08: Ollama narrative not yet implemented", err=True) - sys.exit(1) + """Generate an executive narrative from SHAP via local Ollama.""" + from data_analytics_demo.narrative import generate as narrative_gen + + out = narrative_gen.main() + typer.echo(f"wrote {out}") if __name__ == "__main__": diff --git a/packages/data-analytics-demo/src/data_analytics_demo/narrative/__init__.py b/packages/data-analytics-demo/src/data_analytics_demo/narrative/__init__.py new file mode 100644 index 0000000..28c188d --- /dev/null +++ b/packages/data-analytics-demo/src/data_analytics_demo/narrative/__init__.py @@ -0,0 +1,11 @@ +"""LLM narrative layer for the customer-analytics demo. + +Public surface: + generate.main() Orchestrates SHAP -> Ollama -> markdown. + ollama_client.generate_narrative() Thin Ollama wrapper. + prompts.build_prompt() SHAP-summary -> prompt string. + +All inference runs locally via Ollama (no external LLM API). The module +asserts the absence of cloud-LLM credentials in `os.environ` at invocation +time (AC-4.3). +""" diff --git a/packages/data-analytics-demo/src/data_analytics_demo/narrative/generate.py b/packages/data-analytics-demo/src/data_analytics_demo/narrative/generate.py new file mode 100644 index 0000000..a888e03 --- /dev/null +++ b/packages/data-analytics-demo/src/data_analytics_demo/narrative/generate.py @@ -0,0 +1,107 @@ +"""Narrative generator entry point — reads SHAP summary, calls Ollama, writes markdown. + +Glue between the ML layer (SHAP JSON output) and the executive-facing +output. Implements AC-4.1 through AC-4.5 in a single linear flow. +""" + +from __future__ import annotations + +import json +import sys +from datetime import UTC, datetime +from pathlib import Path + +from .. import __version__ +from ..ml import _io +from . import ollama_client, prompts + + +def _emit(msg: str) -> None: + print(f"[narrative] {msg}", file=sys.stderr, flush=True) # noqa: T201 + + +def _default_shap_path() -> Path: + return _io.default_artifacts_dir() / "shap_summary.json" + + +def _default_output_path() -> Path: + return _io.package_root() / "narrative" / "output.md" + + +def _render( + *, + body: str, + model_id: str, + host: str, + shap_path: Path, + feature_count: int, +) -> str: + """Wrap the LLM body with provenance + reproducibility metadata. + + The metadata block satisfies AC-4.4 (citation back to SHAP JSON) and + AC-4.5 (model identifier present in the artifact). + """ + timestamp = datetime.now(UTC).isoformat(timespec="seconds") + return ( + "# Churn-Risk Narrative (Auto-generated)\n" + "\n" + "> **Model**: local Ollama — `" + f"{model_id}` via `{host}`.\n" + "> " + f"**Source**: SHAP feature importances at `{shap_path.as_posix()}` " + f"({feature_count} features ranked).\n" + f"> **Generated**: {timestamp} by `data-analytics-demo` v{__version__}.\n" + "> **External LLM calls**: 0 (assertion-enforced; see " + "`narrative/ollama_client.py::assert_no_external_api_envs`).\n" + "\n" + f"{body.strip()}\n" + "\n" + "---\n" + "*Auto-generated. Do not edit by hand — re-run `make narrative` to refresh.*\n" + ) + + +def main( + *, + shap_path: Path | None = None, + output_path: Path | None = None, +) -> Path: + """Run the full narrative pipeline and return the output file path.""" + # AC-4.3 — assert clean env before any LLM call. + ollama_client.assert_no_external_api_envs() + + shap_in = shap_path or _default_shap_path() + if not shap_in.exists(): + raise FileNotFoundError( + f"shap_summary.json not found at {shap_in}. " + "Run `make ml` to produce it before `make narrative`." + ) + + out_path = output_path or _default_output_path() + out_path.parent.mkdir(parents=True, exist_ok=True) + + _emit(f"loading SHAP summary from {shap_in.name}") + summary = json.loads(shap_in.read_text(encoding="utf-8")) + feature_count = len(summary.get("top_features", [])) + + prompt = prompts.build_prompt(summary) + _emit( + f"calling Ollama at {ollama_client.resolved_host()} " + f"(model={ollama_client.resolved_model()})" + ) + body = ollama_client.generate_narrative(prompt) + + rendered = _render( + body=body, + model_id=ollama_client.resolved_model(), + host=ollama_client.resolved_host(), + shap_path=shap_in, + feature_count=feature_count, + ) + out_path.write_text(rendered, encoding="utf-8") + _emit(f"wrote {out_path}") + return out_path + + +if __name__ == "__main__": + main() diff --git a/packages/data-analytics-demo/src/data_analytics_demo/narrative/ollama_client.py b/packages/data-analytics-demo/src/data_analytics_demo/narrative/ollama_client.py new file mode 100644 index 0000000..70609ae --- /dev/null +++ b/packages/data-analytics-demo/src/data_analytics_demo/narrative/ollama_client.py @@ -0,0 +1,94 @@ +"""Thin wrapper around the local Ollama HTTP API. + +Two responsibilities: +1. `assert_no_external_api_envs` — enforce AC-4.3 at runtime; raises if any + cloud-LLM credential is present in the environment. +2. `generate_narrative` — call the local Ollama daemon and return the text. + +The wrapper is intentionally small: the heavy lifting (transport, JSON +shape) is delegated to the `ollama` Python client, which is the official +maintainer's SDK. +""" + +from __future__ import annotations + +import os +from typing import Final + +import ollama + +DEFAULT_HOST: Final[str] = "http://localhost:11434" +DEFAULT_MODEL: Final[str] = "llama3.1:8b-instruct-q4_K_M" + +# Env vars that, if set, indicate the caller has set up a cloud LLM +# credential. Their mere presence triggers an AC-4.3 fail-stop. +EXTERNAL_API_ENV_VARS: Final[tuple[str, ...]] = ( + "ANTHROPIC_API_KEY", + "OPENAI_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "AZURE_OPENAI_API_KEY", + "COHERE_API_KEY", +) + + +def resolved_host() -> str: + return os.environ.get("OLLAMA_HOST", DEFAULT_HOST) + + +def resolved_model() -> str: + return os.environ.get("OLLAMA_MODEL", DEFAULT_MODEL) + + +def assert_no_external_api_envs() -> None: + """AC-4.3 — fail fast if any cloud-LLM credential leaked into env.""" + leaked = [k for k in EXTERNAL_API_ENV_VARS if os.environ.get(k)] + if leaked: + msg = ( + "External LLM API credentials detected in environment: " + f"{leaked}. This package routes all inference through local " + "Ollama. Unset these variables or use a clean shell before " + "running `make narrative`." + ) + raise RuntimeError(msg) + + +def generate_narrative( + prompt: str, + *, + model: str | None = None, + host: str | None = None, + temperature: float = 0.4, +) -> str: + """Send `prompt` to local Ollama and return the response text. + + Raises a clear RuntimeError (with remediation hint) if Ollama is not + reachable, satisfying AC-4.2. + """ + use_host = host or resolved_host() + use_model = model or resolved_model() + + try: + client = ollama.Client(host=use_host) + response = client.chat( + model=use_model, + messages=[{"role": "user", "content": prompt}], + options={"temperature": temperature}, + ) + except ConnectionError as exc: + raise RuntimeError( + f"Cannot reach Ollama at {use_host}. Run `ollama serve` and " + f"ensure model `{use_model}` is pulled (`ollama pull {use_model}`)." + ) from exc + except Exception as exc: # noqa: BLE001 + # ollama-py wraps transport errors in its own exception classes; + # surface them with the same remediation hint. + msg = str(exc).lower() + if "connect" in msg or "refused" in msg or "timeout" in msg: + raise RuntimeError( + f"Cannot reach Ollama at {use_host}. Run `ollama serve` and " + f"ensure model `{use_model}` is pulled (`ollama pull {use_model}`)." + ) from exc + raise + + return str(response["message"]["content"]) diff --git a/packages/data-analytics-demo/src/data_analytics_demo/narrative/prompts.py b/packages/data-analytics-demo/src/data_analytics_demo/narrative/prompts.py new file mode 100644 index 0000000..f509e76 --- /dev/null +++ b/packages/data-analytics-demo/src/data_analytics_demo/narrative/prompts.py @@ -0,0 +1,44 @@ +"""Prompt templates for the narrative layer. + +Kept here (not inline in `generate.py`) so the prompt is reviewable as a +single artifact and can be A/B-tested without touching the orchestration. +""" + +from __future__ import annotations + +from typing import Any + +NARRATIVE_TEMPLATE = """You are an analyst writing a brief for a SaaS executive team. \ +A machine-learning model has surfaced the following top drivers behind \ +customer-churn predictions (ranked by mean absolute SHAP value): + +{features} + +Write a concise 3-paragraph narrative for the executive audience: + +1. **What the model is telling us.** Summarise the dominant signal in plain \ +business language, without restating the SHAP numbers. +2. **Why it matters now.** Tie the signal to revenue impact and customer \ +experience; keep it actionable. +3. **Recommended next steps.** Propose 2-3 concrete experiments or playbooks \ +the customer-success team could run this quarter. + +Constraints: +- Avoid jargon (no "SHAP", "feature importance", "ROC-AUC" in the output). +- Do not invent metrics that the model did not surface. +- Keep total length under 350 words. +""" + + +def build_prompt(shap_summary: dict[str, Any]) -> str: + """Render the SHAP summary into the executive-narrative prompt.""" + rows = [] + for feat in shap_summary.get("top_features", []): + direction_label = ( + "raises the churn likelihood" + if feat.get("direction") == "increases_prediction" + else "lowers the churn likelihood" + ) + rows.append(f"- `{feat['name']}` ({direction_label}; magnitude {feat['mean_abs_shap']:.3f})") + features_block = "\n".join(rows) if rows else "- (no features available)" + return NARRATIVE_TEMPLATE.format(features=features_block) diff --git a/packages/data-analytics-demo/tests/test_narrative.py b/packages/data-analytics-demo/tests/test_narrative.py new file mode 100644 index 0000000..f4c5f17 --- /dev/null +++ b/packages/data-analytics-demo/tests/test_narrative.py @@ -0,0 +1,180 @@ +"""Tests for the narrative layer (T-08 / AC-4.1〜4.5).""" + +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +import pytest + +from data_analytics_demo.narrative import generate, ollama_client, prompts + + +@pytest.fixture() +def fake_shap_summary(tmp_path: Path) -> Path: + """Write a minimal SHAP summary JSON for the narrative pipeline to read.""" + p = tmp_path / "shap_summary.json" + p.write_text( + json.dumps( + { + "top_features": [ + { + "name": "recent_to_lifetime_ratio", + "mean_abs_shap": 0.42, + "mean_signed_shap": -0.37, + "direction": "decreases_prediction", + }, + { + "name": "failed_invoice_count", + "mean_abs_shap": 0.18, + "mean_signed_shap": 0.15, + "direction": "increases_prediction", + }, + ], + "summary": {"n_samples_explained": 200, "n_features": 18, "top_n_returned": 2}, + } + ), + encoding="utf-8", + ) + return p + + +class _FakeClient: + """Stand-in for `ollama.Client` — records the call, returns canned text.""" + + last_kwargs: dict[str, Any] = {} + + def __init__(self, host: str | None = None) -> None: + self.host = host + + def chat(self, **kwargs: Any) -> dict[str, Any]: + _FakeClient.last_kwargs = kwargs + return { + "message": { + "content": ( + "Customers who slow down their product usage in the trailing month " + "are the strongest churn risk. Acting on this signal early protects " + "expansion revenue and reduces incident load." + ) + } + } + + +# ---- AC-4.1: WHEN `make narrative` runs, the system produces output.md ------ + +def test_ac_4_1_produces_output_markdown( + fake_shap_summary: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + out = tmp_path / "output.md" + monkeypatch.setattr(ollama_client.ollama, "Client", _FakeClient) + # Make sure the env-var guard sees no external creds. + for key in ollama_client.EXTERNAL_API_ENV_VARS: + monkeypatch.delenv(key, raising=False) + + written = generate.main(shap_path=fake_shap_summary, output_path=out) + assert written == out + assert out.exists() + text = out.read_text(encoding="utf-8") + assert "Churn-Risk Narrative" in text + assert "expansion revenue" in text # body from _FakeClient + + +# ---- AC-4.2: IF Ollama is unreachable, fail with a remediation hint -------- + +def test_ac_4_2_unreachable_ollama_clear_error( + fake_shap_summary: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _BrokenClient: + def __init__(self, host: str | None = None) -> None: + self.host = host + + def chat(self, **kwargs: Any) -> Any: + raise ConnectionError("connection refused") + + monkeypatch.setattr(ollama_client.ollama, "Client", _BrokenClient) + for key in ollama_client.EXTERNAL_API_ENV_VARS: + monkeypatch.delenv(key, raising=False) + + with pytest.raises(RuntimeError, match="Cannot reach Ollama"): + generate.main(shap_path=fake_shap_summary, output_path=tmp_path / "out.md") + + +# ---- AC-4.3: WHERE LLM is invoked, system shall NOT call any external API -- + +def test_ac_4_3_external_api_env_blocks_invocation( + fake_shap_summary: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("ANTHROPIC_API_KEY", "leaked-fake-key") + # Even if the Ollama client would have worked, the guard must trip first. + monkeypatch.setattr(ollama_client.ollama, "Client", _FakeClient) + + with pytest.raises(RuntimeError, match="External LLM API credentials"): + generate.main(shap_path=fake_shap_summary, output_path=tmp_path / "out.md") + + +def test_ac_4_3_assert_no_external_api_envs_unit(monkeypatch: pytest.MonkeyPatch) -> None: + for key in ollama_client.EXTERNAL_API_ENV_VARS: + monkeypatch.delenv(key, raising=False) + # Should not raise. + ollama_client.assert_no_external_api_envs() + + +# ---- AC-4.4: output.md cites shap_summary.json + AC-4.5: model id present -- + +def test_ac_4_4_and_4_5_metadata_block( + fake_shap_summary: Path, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + out = tmp_path / "output.md" + monkeypatch.setattr(ollama_client.ollama, "Client", _FakeClient) + for key in ollama_client.EXTERNAL_API_ENV_VARS: + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("OLLAMA_MODEL", "llama3.1:8b-instruct-q4_K_M") + + generate.main(shap_path=fake_shap_summary, output_path=out) + text = out.read_text(encoding="utf-8") + + # AC-4.4: cites shap_summary source path + assert "shap_summary.json" in text + assert fake_shap_summary.name in text + # AC-4.5: model identifier present + assert "llama3.1:8b-instruct-q4_K_M" in text + # External-call assertion advertised in the metadata + assert "External LLM calls" in text + + +# ---- Missing-data path ----------------------------------------------------- + +def test_missing_shap_summary_raises(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + for key in ollama_client.EXTERNAL_API_ENV_VARS: + monkeypatch.delenv(key, raising=False) + with pytest.raises(FileNotFoundError, match="shap_summary.json"): + generate.main( + shap_path=tmp_path / "nope.json", + output_path=tmp_path / "out.md", + ) + + +# ---- Prompt builder -------------------------------------------------------- + +def test_build_prompt_includes_features() -> None: + summary = { + "top_features": [ + { + "name": "events_last_30d", + "mean_abs_shap": 0.55, + "direction": "increases_prediction", + } + ] + } + text = prompts.build_prompt(summary) + assert "events_last_30d" in text + assert "raises the churn likelihood" in text