From 0eaaa3ef836241e3172fd75eb4a7be3fff741302 Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 13 Jul 2026 15:44:59 +0000 Subject: [PATCH] feat: fabric stability engine + LLM-as-Judge playbook scoring (v0.4.0) Builds the real version of the 'Phase 12' feature set (previously described only in an unexported chat session), natively on main and verified: - stability.py: per-device flap detection (interface/BGP/OSPF/LAG/VPN down-up oscillation), event-rate burst detection against each device's own baseline, rising/stable/falling trend, deterministic 24h risk band. Wired into analyze()'s single streaming pass (O(devices x entities) memory, minute-prefix bucketing tolerant of unreliable timestamps), exposed as 'stability' in /api/analyze + MCP, rendered as a Fabric Stability panel in the UI with recommendations. - judge.py + 'ai-log-analyzer eval': scores playbooks 0-10 on actionability/safety/grounding/completeness. Heuristic core (empty playbooks can't ace safety by absence; disruptive commands without context penalized; placeholder hostnames hurt grounding); optional --use-llm blend with silent fallback; --min-score CI gate; KB self-test by default (scores all 41 actionable rule playbooks). Suite 325 -> 338 tests. Version 0.3.1 -> 0.4.0. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_015XM7j9iDXYwSvjnxZYjzaq --- CHANGELOG.md | 21 +- README.md | 16 +- pyproject.toml | 2 +- src/ai_log_analyzer/analyzer.py | 12 +- src/ai_log_analyzer/cli.py | 50 +++++ src/ai_log_analyzer/judge.py | 239 ++++++++++++++++++++++ src/ai_log_analyzer/stability.py | 225 ++++++++++++++++++++ src/ai_log_analyzer/web/static/app.js | 33 +++ src/ai_log_analyzer/web/static/index.html | 13 ++ tests/test_stability_judge.py | 197 ++++++++++++++++++ tests/test_streaming_analyze.py | 6 +- 11 files changed, 805 insertions(+), 9 deletions(-) create mode 100644 src/ai_log_analyzer/judge.py create mode 100644 src/ai_log_analyzer/stability.py create mode 100644 tests/test_stability_judge.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 1e615ff..82eecd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,10 +4,29 @@ All notable changes to **netlog-ai** are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/); this project uses loose semantic versioning. -## [Unreleased] +## [0.4.0] - 2026-07-13 ### Added +- **Fabric stability engine** (`stability.py`) โ€” per-device flap detection + (downโ†”up oscillation of interfaces / BGP peers / OSPF neighbors / LAG + members / VPN tunnels), event-rate burst detection against each device's + own baseline, rising/stable/falling trend, and a deterministic 24h risk + band. Surfaces as `stability` in `/api/analyze` + MCP and a **๐Ÿ“ถ Fabric + Stability** panel in the UI (worst devices first, actionable + recommendations). Streaming-safe: O(devices ร— entity classes) memory, + minute-prefix bucketing with arrival-order fallback so unreliable syslog + timestamps can't break it. +- **LLM-as-Judge playbook scoring** (`judge.py` + `ai-log-analyzer eval`) โ€” + scores generated playbooks 0โ€“10 on actionability / safety / grounding / + completeness with a deterministic heuristic core (empty playbooks can't + ace safety "by absence"; disruptive commands without context are + penalized; R1/SW2-style placeholder names hurt grounding). Optional + `--use-llm` blends a real LLM judge via the configured provider, falling + back silently. `eval` self-tests every rule-based KB playbook by default, + scores a saved analyze result with `--file`, and `--min-score` makes it a + CI quality gate. + - **Unknown-pattern template mining** (`patterns.py`) โ€” every line the regex KB can't match is mined into templates with a dependency-free Drain-style streaming clusterer (variables masked as `/////`, diff --git a/README.md b/README.md index 571a0eb..812266d 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ > **Network logs in. Ranked actions out.** A local, dark-themed dashboard that classifies syslog events from any vendor (Junos, Arista EOS, FRR), builds a prioritized action list, and lets an LLM write the root-cause analysis with copy-pastable CLI fixes. -[![CI](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml/badge.svg)](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml) ![Tests](https://img.shields.io/badge/tests-325%20passing-brightgreen) ![License](https://img.shields.io/badge/license-MIT-blue) ![Python](https://img.shields.io/badge/python-3.10%2B-blue) ![Stack](https://img.shields.io/badge/stack-Flask%20%2B%20vanilla%20JS-1f6feb) +[![CI](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml/badge.svg)](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml) ![Tests](https://img.shields.io/badge/tests-338%20passing-brightgreen) ![License](https://img.shields.io/badge/license-MIT-blue) ![Python](https://img.shields.io/badge/python-3.10%2B-blue) ![Stack](https://img.shields.io/badge/stack-Flask%20%2B%20vanilla%20JS-1f6feb) > ๐Ÿ““ Recent changes โ€” cross-source correlation + per-device triage (MCP tools **and** Device-tab UI) โ€” are in [`CHANGELOG.md`](CHANGELOG.md). @@ -69,6 +69,8 @@ Tools exposed: `list_sources`, `add_source`, `fetch_logs`, `search_logs`, | ๐Ÿ”Ž **Classify** | 50+ regex patterns across Junos, EOS, FRR, IOS, RFC-3164/5424 โ€” plus your own rules via `AI_LOG_ANALYZER_CUSTOM_RULES` / `POST /api/rules` | | ๐Ÿ”ญ **Unknown patterns** | Lines no rule matched are template-mined (Drain-style, zero deps): variables masked, shapes clustered, error-smelling templates ranked first โ€” novel failure modes surface instead of vanishing as `info` noise. Set `AI_LOG_ANALYZER_TEMPLATE_STORE` for cross-run memory: shapes never seen in any prior run get flagged ๐Ÿ†• | | ๐Ÿงญ **Prioritize** | Deduped action items, ranked by severity ร— count, recovery events excluded | +| ๐Ÿ“ถ **Fabric stability** | Per-device flap detection (interface/BGP/OSPF/LAG/VPN downโ†”up oscillation), event-rate bursts vs the device's own baseline, trend + heuristic 24h risk band โ€” in `/api/analyze` and its own UI panel | +| โš–๏ธ **LLM-as-Judge** | `ai-log-analyzer eval` scores playbooks 0โ€“10 on actionability/safety/grounding/completeness โ€” heuristic core, optional real LLM judge, `--min-score` CI gate | | ๐Ÿง  **Deep analyze** | Top-N items get an LLM-written root-cause + risk + remediation playbook | | ๐Ÿ›ก๏ธ **Sanitize-first** | Every config/log payload is scrubbed (`$6$`, `$9$`, SSH keys, SNMP, RADIUS, public IPs) before LLM call | | ๐Ÿ“ˆ **Health score** | Weighted formula โ†’ 0โ€“100 + A/B/C/D/F + sparkline trend | @@ -106,6 +108,11 @@ ai-log-analyzer analyze --frr r1 r2 --no-llm | jq .score ai-log-analyzer analyze --file /var/log/syslog docker logs my-router 2>&1 | ai-log-analyzer analyze --stdin +# Score playbook quality (LLM-as-Judge; KB self-test by default) +ai-log-analyzer eval # heuristic, offline +ai-log-analyzer eval --use-llm # blend a real LLM judge +ai-log-analyzer eval --file result.json --min-score 7 # CI gate + # Run the full test suite pytest --cov=src --cov-report=term-missing ``` @@ -319,6 +326,8 @@ flowchart TB src/ai_log_analyzer/ classifier.py 50+ regex patterns + severity/category lookup + custom rules patterns.py Drain-lite template miner for lines no rule matched + stability.py Flap/burst/trend scoring + 24h risk per device + judge.py LLM-as-Judge playbook quality scoring (eval CLI) kb.py Rule-based deep-analysis KB (fallback when LLM is off) llm.py Docker Model Runner (TCP + UDS) + Anthropic Claude analyzer.py End-to-end pipeline: classify โ†’ actions โ†’ score โ†’ summary @@ -398,7 +407,7 @@ src/ai_log_analyzer/ ## Tested & accessible -- **325 unit + integration tests** (pytest) +- **338 unit + integration tests** (pytest) - Frontend audited across 8 review rounds: - WCAG-AA: `:focus-visible` rings, `aria-live` regions, `role=tablist/tab/tabpanel`, skip-to-main link, `prefers-reduced-motion` fallback - Responsive โ‰ค 1100px, PWA-ready (`theme-color`, `mobile-web-app-capable`, SVG favicon) @@ -416,7 +425,8 @@ src/ai_log_analyzer/ - [x] Custom classification rules โ€” JSON file (`AI_LOG_ANALYZER_CUSTOM_RULES`) + runtime `POST /api/rules`; UI editor still open - [x] Unknown-pattern template mining โ€” Drain-style clustering of unmatched lines, surfaced in UI/API/MCP - [x] Cross-run template persistence โ€” `AI_LOG_ANALYZER_TEMPLATE_STORE` flags never-before-seen shapes (๐Ÿ†•) -- [ ] Per-template volume anomaly detection (sudden rate spike/drop per device) +- [x] Fabric stability scoring โ€” flap/burst/trend detection + 24h risk band per device +- [x] LLM-as-Judge playbook quality scoring (`ai-log-analyzer eval`, CI-gate ready) ## Contributing diff --git a/pyproject.toml b/pyproject.toml index 53ad707..d8e2784 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "netlog-ai" -version = "0.3.1" +version = "0.4.0" description = "AI-powered network syslog analyzer with pluggable LLM providers (local Docker Model Runner / Anthropic Claude) and lab adapters." readme = "README.md" license = { text = "MIT" } diff --git a/src/ai_log_analyzer/analyzer.py b/src/ai_log_analyzer/analyzer.py index a77c30f..15b659f 100644 --- a/src/ai_log_analyzer/analyzer.py +++ b/src/ai_log_analyzer/analyzer.py @@ -17,6 +17,7 @@ ) from ai_log_analyzer.patterns import TemplateMiner, apply_template_store from ai_log_analyzer.sanitize import sanitize +from ai_log_analyzer.stability import StabilityTracker # Recovery events are classified as medium because they're useful in the @@ -91,6 +92,7 @@ class AnalysisResult: llm_powered: bool generated_at: str unknown_patterns: dict = field(default_factory=dict) + stability: dict = field(default_factory=dict) def to_dict(self) -> dict: return { @@ -106,6 +108,7 @@ def to_dict(self) -> dict: "llm_powered": self.llm_powered, "generated_at": self.generated_at, "unknown_patterns": self.unknown_patterns, + "stability": self.stability, } @@ -515,7 +518,7 @@ def _executive_summary( def _aggregate_stream( classified: Iterable[ClassifiedEvent], top_k: int = 300, -) -> tuple[list[ClassifiedEvent], dict[str, int], dict[str, int], dict, dict, TemplateMiner]: +) -> tuple[list[ClassifiedEvent], dict[str, int], dict[str, int], dict, dict, TemplateMiner, StabilityTracker]: """Single bounded pass over a classified stream. Memory is O(top_k + KB rules + devices + template clusters) regardless of @@ -532,12 +535,14 @@ def _aggregate_stream( groups: dict[tuple[str, str], dict] = {} by_host: dict[str, dict] = {} miner = TemplateMiner() + tracker = StabilityTracker() for seq, e in enumerate(classified): sev_counts[e.severity] = sev_counts.get(e.severity, 0) + 1 cat_counts[e.category] = cat_counts.get(e.category, 0) + 1 _group_actionable(e, groups, seq) _count_device(e, by_host) + tracker.add(e) if e.category == "other" and e.message: miner.add(e.message, e.hostname) heap = buckets.setdefault(e.severity, []) @@ -554,7 +559,7 @@ def _aggregate_stream( ) if len(top_events) >= top_k: break - return top_events[:top_k], sev_counts, cat_counts, groups, by_host, miner + return top_events[:top_k], sev_counts, cat_counts, groups, by_host, miner, tracker def analyze(events: Iterable[LogEvent], use_llm: bool = True, llm_top_n: int = 3) -> AnalysisResult: @@ -565,7 +570,7 @@ def analyze(events: Iterable[LogEvent], use_llm: bool = True, llm_top_n: int = 3 event list. Thread-safe: never mutates global llm state. `use_llm=False` is enforced by forcing `llm_top_n=0` and disabling the LLM exec summary. """ - top_events, sev_counts, cat_counts, groups, by_host, miner = _aggregate_stream( + top_events, sev_counts, cat_counts, groups, by_host, miner, tracker = _aggregate_stream( iter_classify(events) ) # Opt-in cross-run persistence: flags templates never seen in ANY prior @@ -595,6 +600,7 @@ def analyze(events: Iterable[LogEvent], use_llm: bool = True, llm_top_n: int = 3 llm_powered=any_llm, generated_at=datetime.now(timezone.utc).isoformat(timespec="seconds"), unknown_patterns=miner.to_dict(top_n=10), + stability=tracker.report(), ) diff --git a/src/ai_log_analyzer/cli.py b/src/ai_log_analyzer/cli.py index decd0da..458284a 100644 --- a/src/ai_log_analyzer/cli.py +++ b/src/ai_log_analyzer/cli.py @@ -48,6 +48,48 @@ def cmd_containers(_args: argparse.Namespace) -> int: return 0 +def cmd_eval(args: argparse.Namespace) -> int: + """Score playbook quality with the LLM-as-Judge harness. + + Default: self-test every rule-based KB playbook (the quality floor with + the LLM off). --file scores a saved /api/analyze JSON result instead. + --min-score turns the run into a CI gate (exit 1 below threshold). + """ + from ai_log_analyzer import judge + + if args.file: + try: + with open(args.file, encoding="utf-8") as fh: + result = json.load(fh) + except (OSError, json.JSONDecodeError) as exc: + print(f"error: cannot read {args.file}: {exc}", file=sys.stderr) + return 2 + report = judge.judge_analysis_result(result, use_llm=args.use_llm) + else: + report = judge.judge_kb(use_llm=args.use_llm) + + if args.json: + print(json.dumps(report, indent=2)) + else: + print(f"Playbooks scored : {report['playbooks_scored']}") + print(f"Overall quality : {report['overall']}/10") + print() + for v in report["verdicts"][: args.show]: + s = v["scores"] + label = v.get("description") or v.get("category", "") + print(f" {s['overall']:>4}/10 [{v.get('judge')}] {label[:70]}") + print(f" action={s['actionability']} safety={s['safety']} " + f"grounding={s['grounding']} complete={s['completeness']}") + for note in v.get("notes", [])[:2]: + print(f" ! {note}") + + if args.min_score and report["overall"] < args.min_score: + print(f"\nFAIL: overall {report['overall']} < required {args.min_score}", + file=sys.stderr) + return 1 + return 0 + + def cmd_mcp(args: argparse.Namespace) -> int: """Start the MCP server (stdio by default) so Claude Code / Cursor / Continue can call netlog-ai tools directly.""" @@ -83,6 +125,14 @@ def main() -> None: cp = sub.add_parser("containers", help="List running FRR lab containers") cp.set_defaults(func=cmd_containers) + ep = sub.add_parser("eval", help="Score playbook quality (LLM-as-Judge; KB self-test by default)") + ep.add_argument("--file", help="Saved /api/analyze JSON result to score (default: score the built-in KB)") + ep.add_argument("--use-llm", action="store_true", help="Blend in a real LLM judge (falls back to heuristic)") + ep.add_argument("--json", action="store_true", help="Machine-readable output") + ep.add_argument("--show", type=int, default=10, help="Verdicts to print, worst first (default 10)") + ep.add_argument("--min-score", type=float, default=0.0, help="Exit 1 if overall quality is below this (CI gate)") + ep.set_defaults(func=cmd_eval) + mp = sub.add_parser("mcp", help="Run as an MCP server (for Claude Code / Cursor / Continue)") mp.add_argument("--transport", default="stdio", choices=["stdio", "streamable-http"], help="MCP transport (default: stdio)") diff --git a/src/ai_log_analyzer/judge.py b/src/ai_log_analyzer/judge.py new file mode 100644 index 0000000..85c1412 --- /dev/null +++ b/src/ai_log_analyzer/judge.py @@ -0,0 +1,239 @@ +"""LLM-as-Judge โ€” score generated playbooks so quality regressions get caught. + +The product's core output is a remediation playbook (root cause, phased +actions, CLI, monitoring). Nothing previously measured whether those +playbooks are any good โ€” an LLM regression or a KB edit could silently gut +them. This module scores a playbook 0โ€“10 on four dimensions: + + * **actionability** โ€” are there phases with real, copy-pasteable CLI, + expected outputs, preventive config? + * **safety** โ€” do disruptive commands (reload / clear / shutdown) come with + context, and are secrets/placeholders handled properly? + * **grounding** โ€” does it reference the real devices it was asked about, + not invented textbook names (R1, SW2, CR-01)? + * **completeness** โ€” root cause, risk, verify phase, monitoring rules with + actual thresholds. + +The core is a deterministic heuristic (zero dependencies, CI-safe). Pass +`use_llm=True` to blend in a real LLM judge via the configured provider โ€” +it degrades silently back to the heuristic when no LLM is reachable. + +CLI: `ai-log-analyzer eval` scores every playbook in the rule-based KB (a +built-in quality self-test); `--file result.json` scores a saved +/api/analyze result; `--min-score N` turns it into a CI gate. +""" +from __future__ import annotations + +import json +import re +from typing import Any + +from ai_log_analyzer import llm + +# Same placeholder pattern the analyzer scrubs โ€” a playbook naming R1/SW2 +# instead of real inventory is hallucinating. +_PLACEHOLDER_RE = re.compile( + r"\b(?:R[1-9]\d?|SW[1-9]\d?|CR-?\d+|BR-?\d+|" + r"spine-?[XYN]|leaf-?[XYN]|router-?[XYN]|switch-?[XYN])\b", + re.IGNORECASE, +) + +# Disruptive commands that deserve surrounding context (a note, an expected +# outcome, or a rollback) before an operator pastes them at 3am. +_DISRUPTIVE_RE = re.compile( + r"\b(?:reload|reboot|restart\s+system|request\s+system\s+(?:reboot|halt)|" + r"clear\s+(?:bgp|ospf|ip\s+route)|shutdown|write\s+erase|format|" + r"delete\s+system)\b", + re.IGNORECASE, +) + +_EXPECTED_PHASES = ("diagnose", "mitigate", "remediate", "verify", "optimize") + + +def _clamp(x: float) -> float: + return round(max(0.0, min(10.0, x)), 1) + + +def judge_playbook(deep: dict, devices: list[str] | None = None) -> dict[str, Any]: + """Score one deep-analysis playbook. Deterministic, no network.""" + notes: list[str] = [] + phases = deep.get("phases") or [] + actions = [a for p in phases if isinstance(p, dict) + for a in (p.get("actions") or []) if isinstance(a, dict)] + clis = [c for a in actions for c in (a.get("cli") or {}).values() if c] + + # actionability โ”€ phases + CLI density + preventive config + score = 0.0 + phase_names = {str(p.get("name", "")).lower() for p in phases if isinstance(p, dict)} + score += 4.0 * (len(phase_names & set(_EXPECTED_PHASES)) / len(_EXPECTED_PHASES)) + if clis: + score += 3.0 + multi_platform = any(len(a.get("cli") or {}) >= 2 for a in actions) + if multi_platform: + score += 1.0 + else: + notes.append("no CLI commands in any phase") + if deep.get("preventive_config"): + score += 1.0 + if any(a.get("expected") for a in actions): + score += 1.0 + actionability = _clamp(score) + + # An empty playbook must not ace safety/grounding "by absence" โ€” with no + # content to assess those dimensions are neutral, not perfect. + has_content = bool(clis or (deep.get("root_cause") or "").strip()) + + # safety โ”€ disruptive commands need context; secrets must not appear + score = 10.0 if has_content else 5.0 + for a in actions: + for cli in (a.get("cli") or {}).values(): + if cli and _DISRUPTIVE_RE.search(str(cli)): + if not (a.get("note") or a.get("expected")): + score -= 3.0 + notes.append(f"disruptive command without context: {str(cli)[:60]}") + else: + score -= 0.5 + blob = json.dumps(deep) + if re.search(r"\$[169]\$[\w./$]{8,}", blob): + score -= 5.0 + notes.append("playbook contains what looks like a password hash") + safety = _clamp(score) + + # grounding โ”€ real hostnames in, placeholders out + score = 10.0 if has_content else 5.0 + placeholder_hits = len(_PLACEHOLDER_RE.findall(blob)) + if placeholder_hits: + real = {d.lower() for d in (devices or [])} + mentions_real = any(d and d in blob.lower() for d in real) + score -= min(placeholder_hits * 2.0, 6.0) * (0.5 if mentions_real else 1.0) + notes.append(f"{placeholder_hits} textbook placeholder name(s) (R1/SW2-style)") + if devices: + blob_lc = blob.lower() + if not any(d.lower() in blob_lc for d in devices): + score -= 2.0 + notes.append("does not mention any affected device by name") + grounding = _clamp(score) + + # completeness โ”€ root cause, risk, verify, monitoring with thresholds + score = 0.0 + if (deep.get("root_cause") or "").strip(): + score += 3.0 + else: + notes.append("missing root_cause") + if (deep.get("risk") or "").strip(): + score += 2.0 + if "verify" in phase_names: + score += 2.0 + monitoring = deep.get("monitoring") or [] + if monitoring: + score += 1.5 + if any(re.search(r"\d", str(m)) for m in monitoring): + score += 1.5 # thresholds are numbers, not vibes + else: + notes.append("monitoring rules carry no numeric thresholds") + completeness = _clamp(score) + + overall = _clamp( + actionability * 0.35 + safety * 0.25 + grounding * 0.2 + completeness * 0.2 + ) + return { + "scores": { + "actionability": actionability, + "safety": safety, + "grounding": grounding, + "completeness": completeness, + "overall": overall, + }, + "notes": notes[:8], + "judge": "heuristic", + } + + +_LLM_JUDGE_PROMPT = """You are a strict senior NOC engineer reviewing an incident playbook. +Score it 0-10 on: actionability (concrete runnable CLI), safety (disruptive +steps have context/rollback), grounding (real device names, no invented ones), +completeness (root cause, risk, verify steps, monitoring thresholds). +Output STRICT JSON only: {"actionability": N, "safety": N, "grounding": N, "completeness": N}""" + + +def judge_playbook_llm(deep: dict, devices: list[str] | None = None) -> dict[str, Any] | None: + """LLM pass โ€” returns None when no provider is reachable or output is junk.""" + if not llm.is_enabled(): + return None + prompt = (f"AFFECTED DEVICES: {devices or ['unknown']}\n\n" + f"PLAYBOOK JSON:\n{json.dumps(deep, indent=1)[:6000]}\n\nScore it now.") + text = llm.query(_LLM_JUDGE_PROMPT, prompt, max_tokens=200) + if not text: + return None + try: + raw = json.loads(re.sub(r"^```[a-z]*\n?|\n?```$", "", text.strip())) + except json.JSONDecodeError: + return None + if not isinstance(raw, dict): + return None + dims = ("actionability", "safety", "grounding", "completeness") + try: + scores = {d: _clamp(float(raw[d])) for d in dims} + except (KeyError, TypeError, ValueError): + return None + scores["overall"] = _clamp(sum(scores[d] for d in dims) / 4) + return {"scores": scores, "notes": [], "judge": "llm"} + + +def judge(deep: dict, devices: list[str] | None = None, use_llm: bool = False) -> dict[str, Any]: + """Heuristic verdict, blended 50/50 with the LLM's when one is available.""" + heuristic = judge_playbook(deep, devices) + if not use_llm: + return heuristic + llm_verdict = judge_playbook_llm(deep, devices) + if llm_verdict is None: + return heuristic + blended = { + k: _clamp((heuristic["scores"][k] + llm_verdict["scores"][k]) / 2) + for k in heuristic["scores"] + } + return {"scores": blended, "notes": heuristic["notes"], "judge": "heuristic+llm"} + + +def judge_analysis_result(result: dict, use_llm: bool = False) -> dict[str, Any]: + """Score every action item's playbook in a saved /api/analyze result.""" + items = result.get("action_items") or [] + verdicts = [] + for item in items: + deep = item.get("deep_analysis") or {} + if not deep: + continue + v = judge(deep, devices=item.get("devices"), use_llm=use_llm) + verdicts.append({ + "description": item.get("description", ""), + "severity": item.get("severity", ""), + **v, + }) + overall = (round(sum(v["scores"]["overall"] for v in verdicts) / len(verdicts), 1) + if verdicts else 0.0) + return {"overall": overall, "playbooks_scored": len(verdicts), "verdicts": verdicts} + + +def judge_kb(use_llm: bool = False) -> dict[str, Any]: + """Built-in self-test: score the rule-based KB's playbook for every + (category, description) the classifier can emit. This is the floor of + quality users get with the LLM off.""" + from ai_log_analyzer import kb + from ai_log_analyzer.classifier import _KB_PATTERNS + + seen: set[tuple[str, str]] = set() + verdicts = [] + for _, severity, category, description in _KB_PATTERNS: + if severity not in ("critical", "high", "medium"): + continue + key = (category, description) + if key in seen: + continue + seen.add(key) + deep = kb.lookup(category, description) + v = judge(deep, use_llm=use_llm) + verdicts.append({"category": category, "description": description, **v}) + verdicts.sort(key=lambda v: v["scores"]["overall"]) + overall = (round(sum(v["scores"]["overall"] for v in verdicts) / len(verdicts), 1) + if verdicts else 0.0) + return {"overall": overall, "playbooks_scored": len(verdicts), "verdicts": verdicts} diff --git a/src/ai_log_analyzer/stability.py b/src/ai_log_analyzer/stability.py new file mode 100644 index 0000000..1e097e7 --- /dev/null +++ b/src/ai_log_analyzer/stability.py @@ -0,0 +1,225 @@ +"""Fabric stability scoring โ€” flap detection, volume bursts, 24h risk forecast. + +Classified events tell you what broke; this module tells you what keeps +breaking. It tracks, per device: + + * **Flapping** โ€” downโ†’upโ†’down oscillation of the same entity class + (interface, BGP peer, OSPF neighbor, LAG member, VPN tunnel). A link that + bounces six times an hour is a worse signal than one clean failure, but a + severity-ranked action list shows them the same way. + * **Volume bursts** โ€” a device whose event rate spikes far above its own + baseline is misbehaving even when every individual event looks routine. + * **Trend** โ€” whether a device's error activity is rising or falling across + the analysis window, projected into a deterministic 24h risk band. + +Everything is heuristic, streaming-safe (O(devices ร— entity classes) memory), +and dependency-free. Timestamps in network logs are unreliable โ€” mixed +formats, missing entirely โ€” so bucketing uses the minute prefix of whatever +timestamp string exists and falls back to arrival order. +""" +from __future__ import annotations + +import re +from collections import OrderedDict +from dataclasses import dataclass, field + +from ai_log_analyzer.classifier import ClassifiedEvent + +# Descriptions that represent an entity going DOWN / coming back UP. +# Keyed by KB description (stable strings from the classifier), grouped into +# an entity class so "BGP peer down" and "BGP peer established" pair up. +_DOWN_DESCRIPTIONS: dict[str, str] = { + "Interface link down": "interface", + "BGP peer down / connect failure": "bgp", + "BGP peer idle / notification": "bgp", + "BGP notification / adjacency down (FRR/IOS)": "bgp", + "OSPF neighbor state change": "ospf", + "OSPF adjacency loss (FRR/IOS)": "ospf", + "BFD session down": "bfd", + "LACP/LAG member down": "lag", + "LAG member leaving bundle": "lag", + "VPN/IPsec tunnel failure": "vpn", + "VRRP/gateway failover": "redundancy", +} +_UP_DESCRIPTIONS: dict[str, str] = { + "Interface link up": "interface", + "BGP peer established": "bgp", + "OSPF neighbor established": "ospf", + "LAG member joining bundle": "lag", +} + +_ENTITY_LABELS: dict[str, str] = { + "interface": "interface", "bgp": "BGP peer", "ospf": "OSPF neighbor", + "bfd": "BFD session", "lag": "LAG member", "vpn": "VPN tunnel", + "redundancy": "gateway redundancy", +} + +# Strip trailing ":SS" / ":SS.mmm" so bucket keys have minute granularity. +_SECONDS_RE = re.compile(r":\d{2}(?:[.,]\d+)?\s*(?:Z|[+-]\d{2}:?\d{2})?$") + +_MAX_BUCKETS_PER_HOST = 512 +_ARRIVAL_BUCKET_SIZE = 200 # events per synthetic bucket when no timestamp + + +@dataclass +class _DeviceState: + transitions: dict[str, int] = field(default_factory=dict) # entity โ†’ downโ†”up flips + last_state: dict[str, str] = field(default_factory=dict) # entity โ†’ "down"|"up" + downs: dict[str, int] = field(default_factory=dict) # entity โ†’ down events + criticals: int = 0 + highs: int = 0 + total: int = 0 + buckets: OrderedDict[str, int] = field(default_factory=OrderedDict) # minute โ†’ count + + +class StabilityTracker: + """Single-pass stability aggregator โ€” feed every classified event.""" + + def __init__(self) -> None: + self._devices: dict[str, _DeviceState] = {} + self._seq = 0 + + def add(self, e: ClassifiedEvent) -> None: + if not e.hostname: + return + st = self._devices.setdefault(e.hostname, _DeviceState()) + st.total += 1 + if e.severity == "critical": + st.criticals += 1 + elif e.severity == "high": + st.highs += 1 + + entity = _DOWN_DESCRIPTIONS.get(e.description) + if entity is not None: + st.downs[entity] = st.downs.get(entity, 0) + 1 + if st.last_state.get(entity) == "up": + st.transitions[entity] = st.transitions.get(entity, 0) + 1 + st.last_state[entity] = "down" + else: + entity = _UP_DESCRIPTIONS.get(e.description) + if entity is not None: + if st.last_state.get(entity) == "down": + st.transitions[entity] = st.transitions.get(entity, 0) + 1 + st.last_state[entity] = "up" + + # Minute-granularity bucket; arrival-order fallback for blank stamps. + ts = e.timestamp or "" + key = _SECONDS_RE.sub("", ts) if ts else f"seq-{self._seq // _ARRIVAL_BUCKET_SIZE}" + st.buckets[key] = st.buckets.get(key, 0) + 1 + st.buckets.move_to_end(key) + while len(st.buckets) > _MAX_BUCKETS_PER_HOST: + st.buckets.popitem(last=False) + self._seq += 1 + + # โ”€โ”€ report โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ + + @staticmethod + def _flap_count(st: _DeviceState) -> int: + """Full flaps = completed downโ†”up round trips across all entities.""" + return sum(t // 2 for t in st.transitions.values()) + + @staticmethod + def _burst_ratio(st: _DeviceState) -> float: + """Peak minute vs the device's own median minute (โ‰ฅ1.0).""" + counts = sorted(st.buckets.values()) + if len(counts) < 3: + return 1.0 + median = counts[len(counts) // 2] or 1 + return round(counts[-1] / median, 1) + + @staticmethod + def _trend(st: _DeviceState) -> str: + """Rising/stable/falling error activity across the window halves.""" + counts = list(st.buckets.values()) + if len(counts) < 4: + return "stable" + half = len(counts) // 2 + first, second = sum(counts[:half]) or 1, sum(counts[half:]) + ratio = second / first + if ratio >= 1.5: + return "rising" + if ratio <= 0.67: + return "falling" + return "stable" + + @staticmethod + def _score(st: _DeviceState, flaps: int, burst: float) -> int: + score = 100 + score -= min(flaps * 6, 35) + score -= min(st.criticals * 5, 25) + score -= min(st.highs * 1, 15) + if burst >= 10: + score -= 20 + elif burst >= 5: + score -= 12 + elif burst >= 3: + score -= 6 + return max(0, score) + + @staticmethod + def _risk(score: int, trend: str) -> str: + if score < 50 or (score < 70 and trend == "rising"): + return "high" + if score < 70 or trend == "rising": + return "medium" + return "low" + + def report(self, top_n: int = 20) -> dict: + """JSON-ready stability report: per-device scores, worst first.""" + devices: list[dict] = [] + for host, st in self._devices.items(): + flaps = self._flap_count(st) + burst = self._burst_ratio(st) + trend = self._trend(st) + score = self._score(st, flaps, burst) + worst_entity = max(st.transitions, key=st.transitions.get) if st.transitions else "" + devices.append({ + "hostname": host, + "score": score, + "flaps": flaps, + "flapping_entity": _ENTITY_LABELS.get(worst_entity, worst_entity), + "criticals": st.criticals, + "highs": st.highs, + "events": st.total, + "burst_ratio": burst, + "trend": trend, + "risk_24h": self._risk(score, trend), + }) + devices.sort(key=lambda d: (d["score"], -d["flaps"])) + + if devices: + # Weight by event volume so one noisy core drags more than a quiet leaf. + total_events = sum(d["events"] for d in devices) or 1 + fabric = round(sum(d["score"] * d["events"] for d in devices) / total_events) + else: + fabric = 100 + + return { + "fabric_score": fabric, + "device_count": len(devices), + "devices": devices[:top_n], + "recommendations": _recommendations(devices), + } + + +def _recommendations(devices: list[dict]) -> list[str]: + recs: list[str] = [] + for d in devices: + if d["flaps"] >= 2 and d["flapping_entity"]: + recs.append( + f"{d['hostname']}: {d['flapping_entity']} flapped {d['flaps']}ร— โ€” " + f"dampen or fix the underlying link before it pages again" + ) + if d["risk_24h"] == "high": + recs.append( + f"{d['hostname']}: high 24h risk (score {d['score']}, {d['trend']} trend) โ€” " + f"prioritize in the current shift" + ) + if d["burst_ratio"] >= 5: + recs.append( + f"{d['hostname']}: event rate spiked {d['burst_ratio']}ร— above its median " + f"minute โ€” check for a rolling condition" + ) + if len(recs) >= 8: + break + return recs[:8] diff --git a/src/ai_log_analyzer/web/static/app.js b/src/ai_log_analyzer/web/static/app.js index 26a8cda..bb99309 100644 --- a/src/ai_log_analyzer/web/static/app.js +++ b/src/ai_log_analyzer/web/static/app.js @@ -770,6 +770,39 @@ function render(r) { aTbody.appendChild(tr); }); + // Fabric stability (flaps, bursts, 24h risk) + const stab = r.stability || {}; + const stabPanel = $("stability-panel"); + if (stabPanel) { + const devs = stab.devices || []; + if (devs.length) { + stabPanel.style.display = "block"; + $("stability-score").textContent = `(fabric ${stab.fabric_score}/100 ยท ${stab.device_count} devices)`; + const sTbody = $("stability-table").querySelector("tbody"); + clear(sTbody); + devs.slice(0, 15).forEach((d) => { + const riskClass = d.risk_24h === "high" ? "sev-row-critical" + : d.risk_24h === "medium" ? "sev-row-medium" : ""; + const flapLabel = d.flaps + ? `${d.flaps}ร— ${d.flapping_entity || ""}`.trim() : "โ€”"; + const trendIcon = d.trend === "rising" ? "โ†—" : d.trend === "falling" ? "โ†˜" : "โ†’"; + sTbody.appendChild(el("tr", { className: riskClass }, + el("td", { text: d.hostname }), + el("td", { text: String(d.score) }), + el("td", { text: flapLabel }), + el("td", { text: d.burst_ratio > 1 ? `${d.burst_ratio}ร—` : "โ€”" }), + el("td", { text: `${trendIcon} ${d.trend}` }), + el("td", { text: d.risk_24h.toUpperCase() }), + )); + }); + const recs = $("stability-recs"); + clear(recs); + (stab.recommendations || []).forEach((t) => recs.appendChild(el("li", { text: t }))); + } else { + stabPanel.style.display = "none"; + } + } + // Unknown patterns (mined templates of lines no KB rule matched) const up = r.unknown_patterns || {}; const upTemplates = up.top_templates || []; diff --git a/src/ai_log_analyzer/web/static/index.html b/src/ai_log_analyzer/web/static/index.html index 780907a..6c14c21 100644 --- a/src/ai_log_analyzer/web/static/index.html +++ b/src/ai_log_analyzer/web/static/index.html @@ -232,6 +232,7 @@ #summary-panel { contain-intrinsic-size: 0 260px; } #actions-panel { contain-intrinsic-size: 0 380px; } #unknown-panel { contain-intrinsic-size: 0 300px; } + #stability-panel { contain-intrinsic-size: 0 340px; } #events-panel { contain-intrinsic-size: 0 540px; } #optimize-panel { contain-intrinsic-size: 0 800px; } #site-panel { contain-intrinsic-size: 0 720px; } @@ -1437,6 +1438,18 @@

Action Items

+ +