gesh75 · gesh75 · Jul 13, 2026 · Jul 13, 2026 · Jul 13, 2026 · Jul 13, 2026
diff --git a/.env.example b/.env.example
@@ -4,14 +4,31 @@
 
 # ── Server ────────────────────────────────────────────────────────────────────
 ANALYZER_PORT=6060
+# Bind address. Default 127.0.0.1 (localhost only). Set 0.0.0.0 to expose —
+# the server warns if you do that without an API token.
+#ANALYZER_HOST=127.0.0.1
+
+# ── Security ──────────────────────────────────────────────────────────────────
+# When set, every mutating/data API route requires the token, supplied as
+# either `X-API-Token: <token>` or `Authorization: Bearer <token>`.
+#AI_LOG_ANALYZER_API_TOKEN=
+# CORS allow-list for /api/* (comma-separated origins).
+#AI_LOG_ANALYZER_CORS_ORIGINS=http://localhost:6060,http://127.0.0.1:6060
+# Colon-separated roots POST /api/analyze {source:"file"} may read from.
+#AI_LOG_ANALYZER_FILE_ROOTS=~:/var/log
 
 # ── LLM provider ──────────────────────────────────────────────────────────────
+# ollama      : Ollama native API on :11434 (default provider)
 # local       : Docker Model Runner (TCP 12434 or auto-detected Unix socket)
 # claude      : Anthropic Claude first, fall back to local if Claude is unreachable
 # claude-only : Anthropic Claude only — never call local
-LLM_PROVIDER=local
+LLM_PROVIDER=ollama
 LLM_ENABLED=true
-LLM_TIMEOUT=60
+LLM_TIMEOUT=120
+
+# ── Ollama (native API) ───────────────────────────────────────────────────────
+#OLLAMA_URL=http://localhost:11434
+#OLLAMA_MODEL=qwen2.5-coder:latest
 
 # ── Local LLM (Docker Model Runner) ───────────────────────────────────────────
 MODEL_RUNNER_URL=http://localhost:12434
@@ -22,6 +39,18 @@ LLM_MODEL=ai/qwen3:latest
 ANTHROPIC_API_KEY=
 ANTHROPIC_MODEL=claude-haiku-4-5-20251001
 
+# ── Cross-run template memory (optional) ──────────────────────────────────────
+# Path to a JSON store remembering every mined template shape across runs.
+# When set, templates never seen in any prior run are flagged 🆕 in the
+# Unknown Patterns panel and counted as new_template_count in /api/analyze.
+#AI_LOG_ANALYZER_TEMPLATE_STORE=~/.cache/netlog-ai/templates.json
+
+# ── Custom classification rules (optional) ────────────────────────────────────
+# JSON file with a list of {"pattern", "severity", "category", "description"}
+# objects. Loaded at startup; rules take precedence over the built-in KB.
+# Add more at runtime via POST /api/rules.
+#AI_LOG_ANALYZER_CUSTOM_RULES=~/.netlog-ai/rules.json
+
 # ── Alert webhooks (optional) ────────────────────────────────────────────────
 # When set, every analysis that finds events at/above the threshold severity
 # fires a webhook. Delivery is best-effort (5s timeout, failures logged).

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,67 @@ All notable changes to **netlog-ai** are documented here.
 Format follows [Keep a Changelog](https://keepachangelog.com/); this project uses
 loose semantic versioning.
 
+## [Unreleased]
+
+### Added
+
+- **Unknown-pattern template mining** (`patterns.py`) — every line the regex KB
+  can't match is mined into templates with a dependency-free Drain-style
+  streaming clusterer (variables masked as `<ip>/<mac>/<hex>/<if>/<n>/<ts>`,
+  similar shapes merged into `<*>` wildcards, LRU-bounded memory). The
+  analyzer surfaces the top templates — error-smelling shapes ranked first —
+  as `unknown_patterns` in `/api/analyze`, the MCP `analyze_logs` tool, and a
+  new **🔭 Unknown Patterns** panel in the UI. Novel failure modes no longer
+  vanish as `info` noise.
+- **Custom classification rules** — operators can extend/override the built-in
+  KB: a JSON rules file loaded at startup (`AI_LOG_ANALYZER_CUSTOM_RULES`) and
+  a runtime API (`GET/POST /api/rules`). Custom rules are checked before the
+  built-in patterns, so a known-noise event can be demoted or a site-specific
+  failure promoted.
+- **Cross-run template persistence** — set `AI_LOG_ANALYZER_TEMPLATE_STORE`
+  to a path and the miner remembers every template shape across runs
+  (FIFO-bounded JSON store, atomic writes, corrupt-file tolerant). Templates
+  never seen in *any* prior run are flagged `is_new` (🆕 in the UI panel,
+  `new_template_count` in the API) — the classic AIOps early-warning signal.
+- **Coverage wave for phases 0–12 leftovers** — `reports.py` (MD/CSV/HTML
+  exporters), `runbook.py` (Ansible/netmiko generation + command extraction),
+  `topology.py` (build/exports/finding overlay), and the previously untested
+  web routes `/api/report`, `/api/runbook`, `/api/topology`, plus the
+  validation paths of `/api/diff` and `/api/copilot`. Suite 294 → 325 tests.
+
+### Security
+
+- **Log lines now pass the sanitize gate before LLM calls.** `deep_analyze()`
+  sent raw sample log lines (and severity-promoted raw descriptions) to the
+  LLM unsanitized — configs were always scrubbed but logs were not,
+  contradicting the sanitize-before-LLM guarantee. Both are now scrubbed, as
+  is the executive-summary item list.
+- **`/api/sources/<id>/test` and `/api/sources/<id>/fetch` now require the API
+  token** — they were the only POST data routes missing the auth gate.
+- **`Authorization: Bearer <token>` accepted** everywhere `X-API-Token` is
+  (the README documented Bearer; the code only accepted the custom header).
+
+### Fixed
+
+- **MCP server finds bundled sites on pip installs** — `list_sites` /
+  `analyze_site` resolved only the repo-root `sites/` symlink and returned
+  empty from a `pip install netlog-ai[mcp]` wheel; now uses the same
+  env-override → packaged-data → repo-root resolution as the web app.
+- **`/api/optimize/site` no longer mislabels device platforms on mixed-vendor
+  sites** — it stamped every device with the manifest-level vendor string
+  (e.g. `multi (Nokia SRL + Arista cEOS + FRR)`); it now uses the shared
+  loader that keeps per-device `platform`.
+- **Syslog connector `fetch()` is idempotent** — it used to drain the ring
+  buffer, so a correlate → triage sequence on the same source saw zero events
+  on the second call. It now snapshots; the deque's `maxlen` bounds memory.
+- **Default Ollama model corrected** to `qwen2.5-coder:latest` (was
+  `gemma4:latest`, which doesn't exist, breaking the out-of-box default
+  provider).
+- Env-configured sources no longer leak `verify_tls`/`timeout_seconds` into
+  `extra`; `.env.example` and README now document the real security env vars
+  (`AI_LOG_ANALYZER_API_TOKEN`, `AI_LOG_ANALYZER_CORS_ORIGINS`,
+  `ANALYZER_HOST`, `AI_LOG_ANALYZER_FILE_ROOTS`) and the `ollama` provider.
+
 ## [0.3.1] - 2026-06-10
 
 - UI header version badge is now populated live from `/api/health` (the 0.3.0

diff --git a/README.md b/README.md
@@ -15,7 +15,7 @@
 
 > **Network logs in. Ranked actions out.** A local, dark-themed dashboard that classifies syslog events from any vendor (Junos, Arista EOS, FRR), builds a prioritized action list, and lets an LLM write the root-cause analysis with copy-pastable CLI fixes.
 
-[![CI](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml/badge.svg)](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml) ![Tests](https://img.shields.io/badge/tests-294%20passing-brightgreen) ![License](https://img.shields.io/badge/license-MIT-blue) ![Python](https://img.shields.io/badge/python-3.10%2B-blue) ![Stack](https://img.shields.io/badge/stack-Flask%20%2B%20vanilla%20JS-1f6feb)
+[![CI](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml/badge.svg)](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml) ![Tests](https://img.shields.io/badge/tests-325%20passing-brightgreen) ![License](https://img.shields.io/badge/license-MIT-blue) ![Python](https://img.shields.io/badge/python-3.10%2B-blue) ![Stack](https://img.shields.io/badge/stack-Flask%20%2B%20vanilla%20JS-1f6feb)
 
 > 📓 Recent changes — cross-source correlation + per-device triage (MCP tools **and** Device-tab UI) — are in [`CHANGELOG.md`](CHANGELOG.md).
 
@@ -66,7 +66,8 @@ Tools exposed: `list_sources`, `add_source`, `fetch_logs`, `search_logs`,
 | 🤖 **MCP server mode** | Claude Code / Cursor / Continue can call the analyzer directly as agent tools |
 | 🔗 **Cross-source correlation** | **Device tab** → *Correlate Sources*: scans every registered source and tags each host `confirmed` (flagged by ≥ 2 sources) or `suspected` (1) in a sortable, severity-coded table |
 | 🔬 **Per-device triage** | **Device tab** → *Triage Device*: one host's verdict + 0–100 health score, severity histogram, top processes, and deduped error patterns in a single panel |
-| 🔎 **Classify** | 50+ regex patterns across Junos, EOS, FRR, IOS, RFC-3164/5424 |
+| 🔎 **Classify** | 50+ regex patterns across Junos, EOS, FRR, IOS, RFC-3164/5424 — plus your own rules via `AI_LOG_ANALYZER_CUSTOM_RULES` / `POST /api/rules` |
+| 🔭 **Unknown patterns** | Lines no rule matched are template-mined (Drain-style, zero deps): variables masked, shapes clustered, error-smelling templates ranked first — novel failure modes surface instead of vanishing as `info` noise. Set `AI_LOG_ANALYZER_TEMPLATE_STORE` for cross-run memory: shapes never seen in any prior run get flagged 🆕 |
 | 🧭 **Prioritize** | Deduped action items, ranked by severity × count, recovery events excluded |
 | 🧠 **Deep analyze** | Top-N items get an LLM-written root-cause + risk + remediation playbook |
 | 🛡️ **Sanitize-first** | Every config/log payload is scrubbed (`$6$`, `$9$`, SSH keys, SNMP, RADIUS, public IPs) before LLM call |
@@ -111,10 +112,11 @@ pytest --cov=src --cov-report=term-missing
 
 ## Configure the LLM
 
-Three providers, switchable at runtime from the UI dropdown — or via env / API:
+Four providers, switchable at runtime from the UI dropdown — or via env / API:
 
 | Mode          | Order |
 |---------------|-------|
+| `ollama`      | Ollama native API on `:11434` (default; `OLLAMA_URL` / `OLLAMA_MODEL`) |
 | `local`       | Local Docker Model Runner → falls back to Claude if `ANTHROPIC_API_KEY` is set |
 | `claude`      | Claude first → falls back to local |
 | `claude-only` | Claude only, no fallback |
@@ -315,7 +317,8 @@ flowchart TB
 
 ```
 src/ai_log_analyzer/
-  classifier.py        50+ regex patterns + severity/category lookup
+  classifier.py        50+ regex patterns + severity/category lookup + custom rules
+  patterns.py          Drain-lite template miner for lines no rule matched
   kb.py                Rule-based deep-analysis KB (fallback when LLM is off)
   llm.py               Docker Model Runner (TCP + UDS) + Anthropic Claude
   analyzer.py          End-to-end pipeline: classify → actions → score → summary
@@ -346,7 +349,9 @@ src/ai_log_analyzer/
 | `POST` | `/api/llm/toggle` | `{"enabled": true\|false}` |
 | `GET`  | `/api/lab/containers` | Running FRR-lab container names |
 | `GET`  | `/api/sites` | List bundled site bundles |
-| `POST` | `/api/analyze` | Full pipeline — see request shapes below |
+| `POST` | `/api/analyze` | Full pipeline — see request shapes below (response includes `unknown_patterns`: mined templates of unmatched lines) |
+| `GET`  | `/api/rules` | Built-in rule count + registered custom classification rules |
+| `POST` | `/api/rules` | Add custom rules at runtime — `{"rules": [{"pattern", "severity", "category", "description"}]}` |
 | `POST` | `/api/optimize` | Device-level config audit + patches |
 | `POST` | `/api/optimize/site` | Cross-device site analysis |
 | `POST` | `/api/optimize/site-wide/<id>` | Strategic maturity scoring + phased roadmap |
@@ -385,14 +390,15 @@ src/ai_log_analyzer/
   - SNMP communities, RADIUS / TACACS+ keys
   - IPsec pre-shared keys
   - Public IPv4 addresses (mapped to RFC-5737 doc prefixes for the LLM context)
-- **Localhost bind by default** — set `ANALYZER_HOST=0.0.0.0` to expose; the server warns if you bind publicly without an `API_TOKEN`.
-- **API-token gate** — set `API_TOKEN=...` to require `Authorization: Bearer ...` on every POST.
-- **CORS allow-list** — `ANALYZER_CORS_ORIGINS=https://a.com,https://b.com`.
+- **Localhost bind by default** — set `ANALYZER_HOST=0.0.0.0` to expose; the server warns if you bind publicly without an API token.
+- **API-token gate** — set `AI_LOG_ANALYZER_API_TOKEN=...` to require the token on every POST, supplied as either `X-API-Token: <token>` or `Authorization: Bearer <token>`.
+- **CORS allow-list** — `AI_LOG_ANALYZER_CORS_ORIGINS=https://a.com,https://b.com`.
+- **File-ingest confinement** — `AI_LOG_ANALYZER_FILE_ROOTS=~:/var/log` bounds what `{source:"file"}` may read.
 - **No telemetry** — outbound calls go only to (a) the LLM provider you select and (b) the local Docker socket if you analyze FRR-lab containers.
 
 ## Tested & accessible
 
-- **294 unit + integration tests** (pytest)
+- **325 unit + integration tests** (pytest)
 - Frontend audited across 8 review rounds:
   - WCAG-AA: `:focus-visible` rings, `aria-live` regions, `role=tablist/tab/tabpanel`, skip-to-main link, `prefers-reduced-motion` fallback
   - Responsive ≤ 1100px, PWA-ready (`theme-color`, `mobile-web-app-capable`, SVG favicon)
@@ -407,7 +413,10 @@ src/ai_log_analyzer/
 - [x] Slack / generic-JSON alert webhooks per severity threshold (`AI_LOG_ANALYZER_WEBHOOK_URL` — see `.env.example`)
 - [ ] More vendor adapters (Nokia SR Linux, Cisco IOS-XE, Cumulus NCLU)
 - [ ] Snapshot / replay analysis runs for regression testing
-- [ ] Custom rule editor in the UI
+- [x] Custom classification rules — JSON file (`AI_LOG_ANALYZER_CUSTOM_RULES`) + runtime `POST /api/rules`; UI editor still open
+- [x] Unknown-pattern template mining — Drain-style clustering of unmatched lines, surfaced in UI/API/MCP
+- [x] Cross-run template persistence — `AI_LOG_ANALYZER_TEMPLATE_STORE` flags never-before-seen shapes (🆕)
+- [ ] Per-template volume anomaly detection (sudden rate spike/drop per device)
 
 ## Contributing
 

diff --git a/src/ai_log_analyzer/adapters/tfsm_auto.py b/src/ai_log_analyzer/adapters/tfsm_auto.py
@@ -37,9 +37,11 @@
 logger = logging.getLogger(__name__)
 
 # Public-facing URL of the upstream template database. Pinned to the main branch.
-# If upstream restructures, change this constant or set TFSM_DB_PATH locally.
-_UPSTREAM_DB_URL = (
-    "https://github.com/scottpeterman/tfsm_fire/raw/main/tfire/tfsm_templates.db"
+# Upstream has restructured before (the raw URL 404s when it does) — override
+# with TFSM_DB_URL to point at a mirror, or TFSM_DB_PATH at a local copy.
+_UPSTREAM_DB_URL = os.environ.get(
+    "TFSM_DB_URL",
+    "https://github.com/scottpeterman/tfsm_fire/raw/main/tfire/tfsm_templates.db",
 )
 _DEFAULT_DB_PATH = Path(
     os.environ.get(