diff --git a/.env.example b/.env.example index 51b6737..69dc43b 100644 --- a/.env.example +++ b/.env.example @@ -4,14 +4,31 @@ # ── Server ──────────────────────────────────────────────────────────────────── ANALYZER_PORT=6060 +# Bind address. Default 127.0.0.1 (localhost only). Set 0.0.0.0 to expose — +# the server warns if you do that without an API token. +#ANALYZER_HOST=127.0.0.1 + +# ── Security ────────────────────────────────────────────────────────────────── +# When set, every mutating/data API route requires the token, supplied as +# either `X-API-Token: ` or `Authorization: Bearer `. +#AI_LOG_ANALYZER_API_TOKEN= +# CORS allow-list for /api/* (comma-separated origins). +#AI_LOG_ANALYZER_CORS_ORIGINS=http://localhost:6060,http://127.0.0.1:6060 +# Colon-separated roots POST /api/analyze {source:"file"} may read from. +#AI_LOG_ANALYZER_FILE_ROOTS=~:/var/log # ── LLM provider ────────────────────────────────────────────────────────────── +# ollama : Ollama native API on :11434 (default provider) # local : Docker Model Runner (TCP 12434 or auto-detected Unix socket) # claude : Anthropic Claude first, fall back to local if Claude is unreachable # claude-only : Anthropic Claude only — never call local -LLM_PROVIDER=local +LLM_PROVIDER=ollama LLM_ENABLED=true -LLM_TIMEOUT=60 +LLM_TIMEOUT=120 + +# ── Ollama (native API) ─────────────────────────────────────────────────────── +#OLLAMA_URL=http://localhost:11434 +#OLLAMA_MODEL=qwen2.5-coder:latest # ── Local LLM (Docker Model Runner) ─────────────────────────────────────────── MODEL_RUNNER_URL=http://localhost:12434 @@ -22,6 +39,18 @@ LLM_MODEL=ai/qwen3:latest ANTHROPIC_API_KEY= ANTHROPIC_MODEL=claude-haiku-4-5-20251001 +# ── Cross-run template memory (optional) ────────────────────────────────────── +# Path to a JSON store remembering every mined template shape across runs. +# When set, templates never seen in any prior run are flagged 🆕 in the +# Unknown Patterns panel and counted as new_template_count in /api/analyze. +#AI_LOG_ANALYZER_TEMPLATE_STORE=~/.cache/netlog-ai/templates.json + +# ── Custom classification rules (optional) ──────────────────────────────────── +# JSON file with a list of {"pattern", "severity", "category", "description"} +# objects. Loaded at startup; rules take precedence over the built-in KB. +# Add more at runtime via POST /api/rules. +#AI_LOG_ANALYZER_CUSTOM_RULES=~/.netlog-ai/rules.json + # ── Alert webhooks (optional) ──────────────────────────────────────────────── # When set, every analysis that finds events at/above the threshold severity # fires a webhook. Delivery is best-effort (5s timeout, failures logged). diff --git a/CHANGELOG.md b/CHANGELOG.md index 450d63c..1e615ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,67 @@ All notable changes to **netlog-ai** are documented here. Format follows [Keep a Changelog](https://keepachangelog.com/); this project uses loose semantic versioning. +## [Unreleased] + +### Added + +- **Unknown-pattern template mining** (`patterns.py`) — every line the regex KB + can't match is mined into templates with a dependency-free Drain-style + streaming clusterer (variables masked as `/////`, + similar shapes merged into `<*>` wildcards, LRU-bounded memory). The + analyzer surfaces the top templates — error-smelling shapes ranked first — + as `unknown_patterns` in `/api/analyze`, the MCP `analyze_logs` tool, and a + new **🔭 Unknown Patterns** panel in the UI. Novel failure modes no longer + vanish as `info` noise. +- **Custom classification rules** — operators can extend/override the built-in + KB: a JSON rules file loaded at startup (`AI_LOG_ANALYZER_CUSTOM_RULES`) and + a runtime API (`GET/POST /api/rules`). Custom rules are checked before the + built-in patterns, so a known-noise event can be demoted or a site-specific + failure promoted. +- **Cross-run template persistence** — set `AI_LOG_ANALYZER_TEMPLATE_STORE` + to a path and the miner remembers every template shape across runs + (FIFO-bounded JSON store, atomic writes, corrupt-file tolerant). Templates + never seen in *any* prior run are flagged `is_new` (🆕 in the UI panel, + `new_template_count` in the API) — the classic AIOps early-warning signal. +- **Coverage wave for phases 0–12 leftovers** — `reports.py` (MD/CSV/HTML + exporters), `runbook.py` (Ansible/netmiko generation + command extraction), + `topology.py` (build/exports/finding overlay), and the previously untested + web routes `/api/report`, `/api/runbook`, `/api/topology`, plus the + validation paths of `/api/diff` and `/api/copilot`. Suite 294 → 325 tests. + +### Security + +- **Log lines now pass the sanitize gate before LLM calls.** `deep_analyze()` + sent raw sample log lines (and severity-promoted raw descriptions) to the + LLM unsanitized — configs were always scrubbed but logs were not, + contradicting the sanitize-before-LLM guarantee. Both are now scrubbed, as + is the executive-summary item list. +- **`/api/sources//test` and `/api/sources//fetch` now require the API + token** — they were the only POST data routes missing the auth gate. +- **`Authorization: Bearer ` accepted** everywhere `X-API-Token` is + (the README documented Bearer; the code only accepted the custom header). + +### Fixed + +- **MCP server finds bundled sites on pip installs** — `list_sites` / + `analyze_site` resolved only the repo-root `sites/` symlink and returned + empty from a `pip install netlog-ai[mcp]` wheel; now uses the same + env-override → packaged-data → repo-root resolution as the web app. +- **`/api/optimize/site` no longer mislabels device platforms on mixed-vendor + sites** — it stamped every device with the manifest-level vendor string + (e.g. `multi (Nokia SRL + Arista cEOS + FRR)`); it now uses the shared + loader that keeps per-device `platform`. +- **Syslog connector `fetch()` is idempotent** — it used to drain the ring + buffer, so a correlate → triage sequence on the same source saw zero events + on the second call. It now snapshots; the deque's `maxlen` bounds memory. +- **Default Ollama model corrected** to `qwen2.5-coder:latest` (was + `gemma4:latest`, which doesn't exist, breaking the out-of-box default + provider). +- Env-configured sources no longer leak `verify_tls`/`timeout_seconds` into + `extra`; `.env.example` and README now document the real security env vars + (`AI_LOG_ANALYZER_API_TOKEN`, `AI_LOG_ANALYZER_CORS_ORIGINS`, + `ANALYZER_HOST`, `AI_LOG_ANALYZER_FILE_ROOTS`) and the `ollama` provider. + ## [0.3.1] - 2026-06-10 - UI header version badge is now populated live from `/api/health` (the 0.3.0 diff --git a/README.md b/README.md index 1fb790b..571a0eb 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ > **Network logs in. Ranked actions out.** A local, dark-themed dashboard that classifies syslog events from any vendor (Junos, Arista EOS, FRR), builds a prioritized action list, and lets an LLM write the root-cause analysis with copy-pastable CLI fixes. -[![CI](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml/badge.svg)](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml) ![Tests](https://img.shields.io/badge/tests-294%20passing-brightgreen) ![License](https://img.shields.io/badge/license-MIT-blue) ![Python](https://img.shields.io/badge/python-3.10%2B-blue) ![Stack](https://img.shields.io/badge/stack-Flask%20%2B%20vanilla%20JS-1f6feb) +[![CI](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml/badge.svg)](https://github.com/gesh75/netlog-ai/actions/workflows/ci.yml) ![Tests](https://img.shields.io/badge/tests-325%20passing-brightgreen) ![License](https://img.shields.io/badge/license-MIT-blue) ![Python](https://img.shields.io/badge/python-3.10%2B-blue) ![Stack](https://img.shields.io/badge/stack-Flask%20%2B%20vanilla%20JS-1f6feb) > 📓 Recent changes — cross-source correlation + per-device triage (MCP tools **and** Device-tab UI) — are in [`CHANGELOG.md`](CHANGELOG.md). @@ -66,7 +66,8 @@ Tools exposed: `list_sources`, `add_source`, `fetch_logs`, `search_logs`, | 🤖 **MCP server mode** | Claude Code / Cursor / Continue can call the analyzer directly as agent tools | | 🔗 **Cross-source correlation** | **Device tab** → *Correlate Sources*: scans every registered source and tags each host `confirmed` (flagged by ≥ 2 sources) or `suspected` (1) in a sortable, severity-coded table | | 🔬 **Per-device triage** | **Device tab** → *Triage Device*: one host's verdict + 0–100 health score, severity histogram, top processes, and deduped error patterns in a single panel | -| 🔎 **Classify** | 50+ regex patterns across Junos, EOS, FRR, IOS, RFC-3164/5424 | +| 🔎 **Classify** | 50+ regex patterns across Junos, EOS, FRR, IOS, RFC-3164/5424 — plus your own rules via `AI_LOG_ANALYZER_CUSTOM_RULES` / `POST /api/rules` | +| 🔭 **Unknown patterns** | Lines no rule matched are template-mined (Drain-style, zero deps): variables masked, shapes clustered, error-smelling templates ranked first — novel failure modes surface instead of vanishing as `info` noise. Set `AI_LOG_ANALYZER_TEMPLATE_STORE` for cross-run memory: shapes never seen in any prior run get flagged 🆕 | | 🧭 **Prioritize** | Deduped action items, ranked by severity × count, recovery events excluded | | 🧠 **Deep analyze** | Top-N items get an LLM-written root-cause + risk + remediation playbook | | 🛡️ **Sanitize-first** | Every config/log payload is scrubbed (`$6$`, `$9$`, SSH keys, SNMP, RADIUS, public IPs) before LLM call | @@ -111,10 +112,11 @@ pytest --cov=src --cov-report=term-missing ## Configure the LLM -Three providers, switchable at runtime from the UI dropdown — or via env / API: +Four providers, switchable at runtime from the UI dropdown — or via env / API: | Mode | Order | |---------------|-------| +| `ollama` | Ollama native API on `:11434` (default; `OLLAMA_URL` / `OLLAMA_MODEL`) | | `local` | Local Docker Model Runner → falls back to Claude if `ANTHROPIC_API_KEY` is set | | `claude` | Claude first → falls back to local | | `claude-only` | Claude only, no fallback | @@ -315,7 +317,8 @@ flowchart TB ``` src/ai_log_analyzer/ - classifier.py 50+ regex patterns + severity/category lookup + classifier.py 50+ regex patterns + severity/category lookup + custom rules + patterns.py Drain-lite template miner for lines no rule matched kb.py Rule-based deep-analysis KB (fallback when LLM is off) llm.py Docker Model Runner (TCP + UDS) + Anthropic Claude analyzer.py End-to-end pipeline: classify → actions → score → summary @@ -346,7 +349,9 @@ src/ai_log_analyzer/ | `POST` | `/api/llm/toggle` | `{"enabled": true\|false}` | | `GET` | `/api/lab/containers` | Running FRR-lab container names | | `GET` | `/api/sites` | List bundled site bundles | -| `POST` | `/api/analyze` | Full pipeline — see request shapes below | +| `POST` | `/api/analyze` | Full pipeline — see request shapes below (response includes `unknown_patterns`: mined templates of unmatched lines) | +| `GET` | `/api/rules` | Built-in rule count + registered custom classification rules | +| `POST` | `/api/rules` | Add custom rules at runtime — `{"rules": [{"pattern", "severity", "category", "description"}]}` | | `POST` | `/api/optimize` | Device-level config audit + patches | | `POST` | `/api/optimize/site` | Cross-device site analysis | | `POST` | `/api/optimize/site-wide/` | Strategic maturity scoring + phased roadmap | @@ -385,14 +390,15 @@ src/ai_log_analyzer/ - SNMP communities, RADIUS / TACACS+ keys - IPsec pre-shared keys - Public IPv4 addresses (mapped to RFC-5737 doc prefixes for the LLM context) -- **Localhost bind by default** — set `ANALYZER_HOST=0.0.0.0` to expose; the server warns if you bind publicly without an `API_TOKEN`. -- **API-token gate** — set `API_TOKEN=...` to require `Authorization: Bearer ...` on every POST. -- **CORS allow-list** — `ANALYZER_CORS_ORIGINS=https://a.com,https://b.com`. +- **Localhost bind by default** — set `ANALYZER_HOST=0.0.0.0` to expose; the server warns if you bind publicly without an API token. +- **API-token gate** — set `AI_LOG_ANALYZER_API_TOKEN=...` to require the token on every POST, supplied as either `X-API-Token: ` or `Authorization: Bearer `. +- **CORS allow-list** — `AI_LOG_ANALYZER_CORS_ORIGINS=https://a.com,https://b.com`. +- **File-ingest confinement** — `AI_LOG_ANALYZER_FILE_ROOTS=~:/var/log` bounds what `{source:"file"}` may read. - **No telemetry** — outbound calls go only to (a) the LLM provider you select and (b) the local Docker socket if you analyze FRR-lab containers. ## Tested & accessible -- **294 unit + integration tests** (pytest) +- **325 unit + integration tests** (pytest) - Frontend audited across 8 review rounds: - WCAG-AA: `:focus-visible` rings, `aria-live` regions, `role=tablist/tab/tabpanel`, skip-to-main link, `prefers-reduced-motion` fallback - Responsive ≤ 1100px, PWA-ready (`theme-color`, `mobile-web-app-capable`, SVG favicon) @@ -407,7 +413,10 @@ src/ai_log_analyzer/ - [x] Slack / generic-JSON alert webhooks per severity threshold (`AI_LOG_ANALYZER_WEBHOOK_URL` — see `.env.example`) - [ ] More vendor adapters (Nokia SR Linux, Cisco IOS-XE, Cumulus NCLU) - [ ] Snapshot / replay analysis runs for regression testing -- [ ] Custom rule editor in the UI +- [x] Custom classification rules — JSON file (`AI_LOG_ANALYZER_CUSTOM_RULES`) + runtime `POST /api/rules`; UI editor still open +- [x] Unknown-pattern template mining — Drain-style clustering of unmatched lines, surfaced in UI/API/MCP +- [x] Cross-run template persistence — `AI_LOG_ANALYZER_TEMPLATE_STORE` flags never-before-seen shapes (🆕) +- [ ] Per-template volume anomaly detection (sudden rate spike/drop per device) ## Contributing diff --git a/src/ai_log_analyzer/adapters/tfsm_auto.py b/src/ai_log_analyzer/adapters/tfsm_auto.py index c7904b8..95e15e9 100644 --- a/src/ai_log_analyzer/adapters/tfsm_auto.py +++ b/src/ai_log_analyzer/adapters/tfsm_auto.py @@ -37,9 +37,11 @@ logger = logging.getLogger(__name__) # Public-facing URL of the upstream template database. Pinned to the main branch. -# If upstream restructures, change this constant or set TFSM_DB_PATH locally. -_UPSTREAM_DB_URL = ( - "https://github.com/scottpeterman/tfsm_fire/raw/main/tfire/tfsm_templates.db" +# Upstream has restructured before (the raw URL 404s when it does) — override +# with TFSM_DB_URL to point at a mirror, or TFSM_DB_PATH at a local copy. +_UPSTREAM_DB_URL = os.environ.get( + "TFSM_DB_URL", + "https://github.com/scottpeterman/tfsm_fire/raw/main/tfire/tfsm_templates.db", ) _DEFAULT_DB_PATH = Path( os.environ.get( diff --git a/src/ai_log_analyzer/analyzer.py b/src/ai_log_analyzer/analyzer.py index e0abb68..a77c30f 100644 --- a/src/ai_log_analyzer/analyzer.py +++ b/src/ai_log_analyzer/analyzer.py @@ -15,6 +15,7 @@ LogEvent, iter_classify, ) +from ai_log_analyzer.patterns import TemplateMiner, apply_template_store from ai_log_analyzer.sanitize import sanitize @@ -89,6 +90,7 @@ class AnalysisResult: executive_summary: list[str] llm_powered: bool generated_at: str + unknown_patterns: dict = field(default_factory=dict) def to_dict(self) -> dict: return { @@ -103,6 +105,7 @@ def to_dict(self) -> dict: "executive_summary": self.executive_summary, "llm_powered": self.llm_powered, "generated_at": self.generated_at, + "unknown_patterns": self.unknown_patterns, } @@ -209,11 +212,19 @@ def deep_analyze( llm_result: dict | None = None if not skip_llm: - samples = "\n".join(f" - {m}" for m in sample_messages[:5]) or " (no samples)" + # CRITICAL: raw log lines can carry secrets (auth failures echo + # usernames, SNMP traps echo communities, RADIUS/TACACS lines echo + # server keys). Same sanitize gate as the config paths — nothing + # reaches an external LLM unscrubbed. + safe_samples = [sanitize(m, mask_pii=True)[0] for m in sample_messages[:5]] + samples = "\n".join(f" - {m}" for m in safe_samples) or " (no samples)" + # Descriptions are usually KB constants, but severity-promoted events + # (raw crit/emerg/alert with no KB match) carry a raw message snippet. + safe_description, _ = sanitize(description, mask_pii=True) user_prompt = ( f"INCIDENT CONTEXT\n" f"Category: {category}\n" - f"Description: {description}\n" + f"Description: {safe_description}\n" f"Occurrences: {count}\n" f"Affected devices ({len(devices)}): {', '.join(devices[:5]) if devices else 'unknown'}\n" f"Platform hint: {platform_hint or 'mixed (frr/junos/eos)'}\n" @@ -462,8 +473,8 @@ def _executive_summary( "- Match hostnames verbatim — no abbreviations, no shortenings." ) top = [ - f"{a.severity.upper()} | {a.description} ({a.count}× on " - f"{', '.join(a.devices[:5]) if a.devices else 'unknown'})" + f"{a.severity.upper()} | {sanitize(a.description, mask_pii=True)[0]} " + f"({a.count}× on {', '.join(a.devices[:5]) if a.devices else 'unknown'})" for a in items[:5] ] user_prompt = ( @@ -504,26 +515,31 @@ def _executive_summary( def _aggregate_stream( classified: Iterable[ClassifiedEvent], top_k: int = 300, -) -> tuple[list[ClassifiedEvent], dict[str, int], dict[str, int], dict, dict]: +) -> tuple[list[ClassifiedEvent], dict[str, int], dict[str, int], dict, dict, TemplateMiner]: """Single bounded pass over a classified stream. - Memory is O(top_k + KB rules + devices) regardless of input size — this - is what lets analyze() handle multi-GB syslog without materializing it. - Per-severity heaps keyed on (timestamp, -seq) keep the newest top_k - events of each severity; finalization reproduces classify_events()' - ordering exactly (severity rank, then timestamp desc, then arrival). + Memory is O(top_k + KB rules + devices + template clusters) regardless of + input size — this is what lets analyze() handle multi-GB syslog without + materializing it. Per-severity heaps keyed on (timestamp, -seq) keep the + newest top_k events of each severity; finalization reproduces + classify_events()' ordering exactly (severity rank, then timestamp desc, + then arrival). Events no KB rule matched are additionally mined into + templates so novel message shapes surface instead of vanishing as noise. """ sev_counts: dict[str, int] = {"critical": 0, "high": 0, "medium": 0, "low": 0, "info": 0} cat_counts: dict[str, int] = {} buckets: dict[str, list] = {sev: [] for sev in SEV_ORDER} groups: dict[tuple[str, str], dict] = {} by_host: dict[str, dict] = {} + miner = TemplateMiner() for seq, e in enumerate(classified): sev_counts[e.severity] = sev_counts.get(e.severity, 0) + 1 cat_counts[e.category] = cat_counts.get(e.category, 0) + 1 _group_actionable(e, groups, seq) _count_device(e, by_host) + if e.category == "other" and e.message: + miner.add(e.message, e.hostname) heap = buckets.setdefault(e.severity, []) entry = (e.timestamp, -seq, e) if len(heap) < top_k: @@ -538,7 +554,7 @@ def _aggregate_stream( ) if len(top_events) >= top_k: break - return top_events[:top_k], sev_counts, cat_counts, groups, by_host + return top_events[:top_k], sev_counts, cat_counts, groups, by_host, miner def analyze(events: Iterable[LogEvent], use_llm: bool = True, llm_top_n: int = 3) -> AnalysisResult: @@ -549,9 +565,12 @@ def analyze(events: Iterable[LogEvent], use_llm: bool = True, llm_top_n: int = 3 event list. Thread-safe: never mutates global llm state. `use_llm=False` is enforced by forcing `llm_top_n=0` and disabling the LLM exec summary. """ - top_events, sev_counts, cat_counts, groups, by_host = _aggregate_stream( + top_events, sev_counts, cat_counts, groups, by_host, miner = _aggregate_stream( iter_classify(events) ) + # Opt-in cross-run persistence: flags templates never seen in ANY prior + # run (AI_LOG_ANALYZER_TEMPLATE_STORE) — the classic AIOps early-warning. + apply_template_store(miner) effective_llm = use_llm and llm.is_enabled() action_items = _finalize_action_items( @@ -575,6 +594,7 @@ def analyze(events: Iterable[LogEvent], use_llm: bool = True, llm_top_n: int = 3 executive_summary=summary, llm_powered=any_llm, generated_at=datetime.now(timezone.utc).isoformat(timespec="seconds"), + unknown_patterns=miner.to_dict(top_n=10), ) diff --git a/src/ai_log_analyzer/classifier.py b/src/ai_log_analyzer/classifier.py index a335c19..19aa0e1 100644 --- a/src/ai_log_analyzer/classifier.py +++ b/src/ai_log_analyzer/classifier.py @@ -5,6 +5,7 @@ """ from __future__ import annotations +import os import re from dataclasses import dataclass, field from typing import Iterable, Iterator @@ -162,6 +163,71 @@ def _prefer(cur: set[str] | None, new: set[str] | None) -> set[str] | None: if not any(other != k and other in k for other in _kw) )) +# ── Custom user rules ──────────────────────────────────────────────────── +# Loaded from a JSON file (env AI_LOG_ANALYZER_CUSTOM_RULES or add_custom_rules()). +# Custom rules are checked BEFORE the built-in KB so operators can override +# any built-in verdict. They bypass the literal gate — rule counts are small, +# so the extra scans are negligible. +_CUSTOM_RULES: list[tuple[re.Pattern[str], str, str, str]] = [] + + +def add_custom_rules(rules: list[dict]) -> tuple[int, list[str]]: + """Register user-defined classification rules at runtime. + + Each rule: {"pattern": regex, "severity": critical|high|medium|low|info, + "category": str, "description": str}. Returns (added_count, errors); + invalid rules are skipped, never fatal. + """ + added, errors = 0, [] + for i, rule in enumerate(rules): + if not isinstance(rule, dict): + errors.append(f"rule {i}: not an object") + continue + pattern = rule.get("pattern", "") + severity = str(rule.get("severity", "")).lower() + category = str(rule.get("category", "custom")) or "custom" + description = str(rule.get("description", "")) or "Custom rule match" + if severity not in SEV_ORDER: + errors.append(f"rule {i}: invalid severity {severity!r}") + continue + try: + compiled = re.compile(pattern, re.IGNORECASE) + except re.error as exc: + errors.append(f"rule {i}: bad regex: {exc}") + continue + _CUSTOM_RULES.append((compiled, severity, category, description)) + added += 1 + return added, errors + + +def load_custom_rules_file(path: str) -> tuple[int, list[str]]: + """Load custom rules from a JSON file: a list of rule objects.""" + import json + from pathlib import Path + p = Path(path).expanduser() + if not p.is_file(): + return 0, [f"custom rules file not found: {p}"] + try: + data = json.loads(p.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return 0, [f"custom rules file unreadable: {exc}"] + if not isinstance(data, list): + return 0, ["custom rules file must contain a JSON list"] + return add_custom_rules(data) + + +def custom_rules() -> list[dict]: + """JSON-ready view of the currently registered custom rules.""" + return [ + {"pattern": p.pattern, "severity": s, "category": c, "description": d} + for (p, s, c, d) in _CUSTOM_RULES + ] + + +_rules_path = os.environ.get("AI_LOG_ANALYZER_CUSTOM_RULES", "") +if _rules_path: + load_custom_rules_file(_rules_path) + # ANSI/VT100 terminal escape sequences — systemd, journald, FRR vtysh, and # any source piping colored output leaks these into the message field. # Pattern covers CSI (\x1b[...m), OSC (\x1b]...\x07), and SGR-style codes. @@ -259,17 +325,24 @@ def iter_classify(events: Iterable[LogEvent]) -> Iterator[ClassifiedEvent]: combined = f"{ev.appname} {clean_message}".lower() sev, cat, desc = "info", "other", "" - # Literal gate: no keyword → only the unproven patterns can possibly - # match (their relative order is preserved, so precedence holds). - if any(k in combined for k in _KEYWORDS): - candidates = _COMPILED - else: - candidates = _UNGUARDED - for pattern, p_sev, p_cat, p_desc in candidates: + # Custom user rules take precedence over the built-in KB. + for pattern, p_sev, p_cat, p_desc in _CUSTOM_RULES: if pattern.search(combined): sev, cat, desc = p_sev, p_cat, p_desc break + if not desc: + # Literal gate: no keyword → only the unproven patterns can possibly + # match (their relative order is preserved, so precedence holds). + if any(k in combined for k in _KEYWORDS): + candidates = _COMPILED + else: + candidates = _UNGUARDED + for pattern, p_sev, p_cat, p_desc in candidates: + if pattern.search(combined): + sev, cat, desc = p_sev, p_cat, p_desc + break + if not desc: snippet = clean_message[:120].strip() or ev.appname or "General log message" desc = snippet diff --git a/src/ai_log_analyzer/llm.py b/src/ai_log_analyzer/llm.py index f8efa26..c6bc66d 100644 --- a/src/ai_log_analyzer/llm.py +++ b/src/ai_log_analyzer/llm.py @@ -1,4 +1,4 @@ -"""Pluggable LLM client with three providers: +"""Pluggable LLM client with four providers: - "ollama" : Ollama native API (gemma3 / qwen2.5-coder / llama3.2) — preferred local - "local" : Docker Model Runner (OpenAI-compatible) via TCP or Unix socket @@ -45,7 +45,7 @@ def _cached_probe(name: str, fn: Callable[[], bool]) -> bool: "enabled": os.environ.get("LLM_ENABLED", "true").lower() == "true", # Ollama (native API) "ollama_url": os.environ.get("OLLAMA_URL", "http://localhost:11434"), - "ollama_model": os.environ.get("OLLAMA_MODEL", "gemma4:latest"), + "ollama_model": os.environ.get("OLLAMA_MODEL", "qwen2.5-coder:latest"), # Docker Model Runner (OpenAI-compatible) "local_url": os.environ.get("MODEL_RUNNER_URL", "http://localhost:12434"), "local_model": os.environ.get("LLM_MODEL", "ai/qwen3:latest"), @@ -152,7 +152,7 @@ def query(system_prompt: str, user_prompt: str, max_tokens: int = 800) -> Option # ── Ollama native API ──────────────────────────────────────────────────────── def _query_ollama(system_prompt: str, user_prompt: str, max_tokens: int) -> Optional[str]: - """Ollama native /api/chat — supports `think:false` for gemma4/qwen3.""" + """Ollama native /api/chat — supports `think:false` for thinking models (qwen3 etc.).""" url = f"{str(_state['ollama_url']).rstrip('/')}/api/chat" payload = { "model": _state["ollama_model"], diff --git a/src/ai_log_analyzer/mcp_server/server.py b/src/ai_log_analyzer/mcp_server/server.py index 9f72a6d..b917ad2 100644 --- a/src/ai_log_analyzer/mcp_server/server.py +++ b/src/ai_log_analyzer/mcp_server/server.py @@ -35,7 +35,21 @@ log = logging.getLogger(__name__) -_SITES_DIR = Path(__file__).resolve().parents[3] / "sites" +def _resolve_sites_dir() -> Path: + """Same resolution order as the web app: env override → packaged data + (present in pip/Docker installs) → legacy repo-root layout. The old + repo-root-only lookup made list_sites/analyze_site return empty on any + `pip install netlog-ai[mcp]` wheel.""" + override = os.environ.get("AI_LOG_ANALYZER_SITES_DIR", "") + if override: + return Path(override).expanduser() + packaged = Path(__file__).resolve().parents[1] / "data" / "sites" + if packaged.is_dir(): + return packaged + return Path(__file__).resolve().parents[3] / "sites" + + +_SITES_DIR = _resolve_sites_dir() def _build_server(): @@ -277,7 +291,7 @@ def analyze_site(site_id: str) -> dict[str, Any]: cross-device findings (BGP, OSPF, MTU, BFD, LLDP gaps + topology hints). Note: LLM usage is governed by the server-side LLM toggle, not a per-call - parameter — same behaviour as the web UI's /api/site/analyze endpoint. + parameter — same behaviour as the web UI's /api/optimize/site endpoint. """ from ai_log_analyzer.analyzer import analyze_site as _analyze_site safe_id = re.sub(r"[^a-zA-Z0-9_\-]", "", site_id) diff --git a/src/ai_log_analyzer/patterns.py b/src/ai_log_analyzer/patterns.py new file mode 100644 index 0000000..589c201 --- /dev/null +++ b/src/ai_log_analyzer/patterns.py @@ -0,0 +1,276 @@ +"""Streaming template miner for *unclassified* log events (Drain-lite). + +Every event the regex KB can't match is silently classified `info`/`other` +and vanishes from the action list — yet "a message shape we've never seen +before" is one of the strongest early-warning signals in network operations +(new error paths show up as new templates before they show up as outages). + +This module mines those unmatched lines into templates with zero external +dependencies, so the analyzer can surface a ranked "unknown patterns" panel +next to the KB-classified action items: + + 1. **Mask** variable parts (IPs, MACs, hex, numbers, interface names) so + `bgp peer 10.0.0.1 reset` and `bgp peer 10.0.0.9 reset` share a shape. + 2. **Cluster** masked token streams with a fixed-depth parse tree keyed by + (token count, first token) and a similarity threshold — the same idea + as the Drain algorithm, trimmed to what syslog needs. + 3. **Merge** diverging tokens into `<*>` wildcards as a cluster grows. + +Memory is bounded by `max_clusters` (LRU eviction), so it is safe inside +analyze()'s single streaming pass at any input size. +""" +from __future__ import annotations + +import json +import logging +import os +import re +from collections import OrderedDict +from dataclasses import dataclass, field +from pathlib import Path + +logger = logging.getLogger(__name__) + +# Masking rules applied before tokenization, most specific first. +# Every mask becomes a fixed token so it can't split template shapes. +_MASKS: list[tuple[re.Pattern[str], str]] = [ + (re.compile(r"\b\d{1,3}(?:\.\d{1,3}){3}(?:/\d{1,2})?(?::\d+)?\b"), ""), + (re.compile(r"\b(?:[0-9a-f]{2}[:.-]){5}[0-9a-f]{2}\b", re.I), ""), + (re.compile(r"\b(?:[0-9a-f]{4}\.){2}[0-9a-f]{4}\b", re.I), ""), + (re.compile(r"\b[0-9a-f]{2}(?::[0-9a-f]{2}){3,}\b", re.I), ""), + (re.compile(r"\b0x[0-9a-f]+\b", re.I), ""), + # interface names: ge-0/0/1.0, xe-1/2/3, et-0/0/0, Ethernet49/1, eth1, ae0.100 + (re.compile(r"\b(?:[gxem]t?e|et|ae|irb|lo|vlan|fxp|em|mge)-?\d+(?:[/.:]\d+)*\b", re.I), ""), + (re.compile(r"\b(?:hundredgige|fortygige|twentyfivegige|tengige|gigabitethernet|fastethernet|ethernet|port-channel|tunnel|loopback|management)\d+(?:[/.:]\d+)*\b", re.I), ""), + # ISO / syslog timestamps embedded mid-message + (re.compile(r"\b\d{4}-\d{2}-\d{2}[t ]\d{2}:\d{2}:\d{2}(?:[.,]\d+)?(?:z|[+-]\d{2}:?\d{2})?\b", re.I), ""), + (re.compile(r"\b\d{2}:\d{2}:\d{2}(?:\.\d+)?\b"), ""), + # any remaining integers / decimals (counters, PIDs, ports, durations) + (re.compile(r"(?"), +] + +_WILDCARD = "<*>" + + +def mask_message(message: str) -> str: + """Replace variable fields (IPs, MACs, hex, numbers, interfaces) with tokens.""" + out = message + for pattern, token in _MASKS: + out = pattern.sub(token, out) + return out + + +def _similarity(a: list[str], b: list[str]) -> float: + """Positional token similarity of two equal-length token lists (0..1).""" + if not a: + return 1.0 + same = sum(1 for x, y in zip(a, b) if x == y or x == _WILDCARD or y == _WILDCARD) + return same / len(a) + + +@dataclass +class TemplateCluster: + """One mined template with its live statistics.""" + cluster_id: int + tokens: list[str] + group_key: tuple[int, str] = (0, "") + count: int = 0 + hosts: set[str] = field(default_factory=set) + sample: str = "" # first raw (masked) example, for the UI + severity_hint: str = "info" # promoted if the shape smells like an error + is_new: bool = False # never seen in any prior run (needs a store) + + @property + def template(self) -> str: + return " ".join(self.tokens) + + def to_dict(self) -> dict: + return { + "template": self.template, + "count": self.count, + "hosts": sorted(self.hosts)[:10], + "host_count": len(self.hosts), + "sample": self.sample, + "severity_hint": self.severity_hint, + "is_new": self.is_new, + } + + +# Words that mark an unknown template as probably-bad even though no KB rule +# matched it. Deliberately conservative: presence promotes the hint only. +_ERRORISH_RE = re.compile( + r"\b(?:error|err|fail|failed|failure|crit|critical|fatal|panic|deny|denied|" + r"drop|dropped|timeout|unreachable|refused|invalid|corrupt|exceed|exceeded|" + r"mismatch|abort|aborted|lost|expired|violation)\b", + re.IGNORECASE, +) + + +class TemplateMiner: + """Fixed-depth streaming template miner (Drain-lite), bounded memory. + + Clusters are grouped by (token_count, first_token); within a group a new + line joins the most similar cluster above `sim_threshold`, merging + diverging positions into `<*>`, otherwise it starts a new cluster. + """ + + def __init__(self, sim_threshold: float = 0.55, max_clusters: int = 512, + max_tokens: int = 40) -> None: + self.sim_threshold = sim_threshold + self.max_clusters = max_clusters + self.max_tokens = max_tokens + self._clusters: OrderedDict[int, TemplateCluster] = OrderedDict() + self._groups: dict[tuple[int, str], list[int]] = {} + self._next_id = 1 + + def add(self, message: str, hostname: str = "") -> TemplateCluster: + """Feed one raw message; returns the cluster it joined or created.""" + masked = mask_message(message.strip()) + tokens = masked.split() + if len(tokens) > self.max_tokens: + tokens = tokens[: self.max_tokens] + # Group key: length + first token — wildcard first tokens bucket together. + first = tokens[0] if tokens else "" + if first.startswith("<"): + first = _WILDCARD + key = (len(tokens), first) + + best: TemplateCluster | None = None + best_sim = 0.0 + for cid in self._groups.get(key, ()): + cluster = self._clusters.get(cid) + if cluster is None: + continue + sim = _similarity(cluster.tokens, tokens) + if sim > best_sim: + best, best_sim = cluster, sim + + if best is not None and best_sim >= self.sim_threshold: + # Merge diverging positions into wildcards. + best.tokens = [ + t if t == u else _WILDCARD for t, u in zip(best.tokens, tokens) + ] + cluster = best + else: + cluster = TemplateCluster( + cluster_id=self._next_id, tokens=tokens, group_key=key, + sample=masked[:200], + ) + self._next_id += 1 + self._clusters[cluster.cluster_id] = cluster + self._groups.setdefault(key, []).append(cluster.cluster_id) + self._evict_if_needed() + + cluster.count += 1 + if hostname: + cluster.hosts.add(hostname) + if cluster.severity_hint == "info" and _ERRORISH_RE.search(masked): + cluster.severity_hint = "warning" + # LRU touch + self._clusters.move_to_end(cluster.cluster_id) + return cluster + + def _evict_if_needed(self) -> None: + while len(self._clusters) > self.max_clusters: + old_id, old = self._clusters.popitem(last=False) + ids = self._groups.get(old.group_key) + if ids and old_id in ids: + ids.remove(old_id) + + @property + def cluster_count(self) -> int: + return len(self._clusters) + + def top(self, n: int = 10, errorish_first: bool = True) -> list[TemplateCluster]: + """Highest-signal clusters: error-smelling shapes first, never-seen + shapes next within each band, then by volume.""" + clusters = list(self._clusters.values()) + if errorish_first: + clusters.sort(key=lambda c: (c.severity_hint == "info", not c.is_new, -c.count)) + else: + clusters.sort(key=lambda c: -c.count) + return clusters[:n] + + def to_dict(self, top_n: int = 10) -> dict: + """JSON-ready summary for AnalysisResult / API / MCP.""" + top = self.top(top_n) + return { + "total_unclassified": sum(c.count for c in self._clusters.values()), + "template_count": self.cluster_count, + "new_template_count": sum(1 for c in self._clusters.values() if c.is_new), + "top_templates": [c.to_dict() for c in top], + } + + +class TemplateStore: + """On-disk memory of every template shape seen in prior runs. + + Backs the highest-signal AIOps event — "this message shape has NEVER been + seen before" — across analysis runs. The store is a flat JSON list of + template strings, FIFO-bounded so it can't grow without limit. Wildcard + merging can rename a template between runs, so matching is exact-string + and deliberately conservative: a renamed shape reports as new once, then + settles. + """ + + def __init__(self, path: str | Path, max_entries: int = 20_000) -> None: + self.path = Path(path).expanduser() + self.max_entries = max_entries + self._templates: OrderedDict[str, None] = OrderedDict() + self._load() + + def _load(self) -> None: + if not self.path.is_file(): + return + try: + data = json.loads(self.path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + logger.warning("template store unreadable (%s) — starting fresh", exc) + return + if isinstance(data, list): + for t in data[-self.max_entries:]: + if isinstance(t, str): + self._templates[t] = None + + def __contains__(self, template: str) -> bool: + return template in self._templates + + def __len__(self) -> int: + return len(self._templates) + + def mark_and_update(self, miner: TemplateMiner) -> int: + """Flag never-before-seen clusters as `is_new`, absorb this run's + templates into the store, and return the new-template count.""" + new_count = 0 + for cluster in miner._clusters.values(): + t = cluster.template + if t not in self._templates: + cluster.is_new = True + new_count += 1 + self._templates[t] = None + while len(self._templates) > self.max_entries: + self._templates.popitem(last=False) + return new_count + + def save(self) -> None: + """Persist atomically; failures are logged, never raised — a broken + store must not break an analysis run.""" + try: + self.path.parent.mkdir(parents=True, exist_ok=True) + tmp = self.path.with_suffix(self.path.suffix + ".tmp") + tmp.write_text(json.dumps(list(self._templates)), encoding="utf-8") + tmp.replace(self.path) + except OSError as exc: + logger.warning("could not persist template store to %s: %s", self.path, exc) + + +def apply_template_store(miner: TemplateMiner) -> None: + """Opt-in cross-run persistence: when AI_LOG_ANALYZER_TEMPLATE_STORE + names a path, mark this run's never-seen-before templates and update the + store. No env var → no-op (single-run behaviour unchanged).""" + store_path = os.environ.get("AI_LOG_ANALYZER_TEMPLATE_STORE", "") + if not store_path or miner.cluster_count == 0: + return + store = TemplateStore(store_path) + store.mark_and_update(miner) + store.save() diff --git a/src/ai_log_analyzer/sources/manager.py b/src/ai_log_analyzer/sources/manager.py index bc9c2d9..a0c7662 100644 --- a/src/ai_log_analyzer/sources/manager.py +++ b/src/ai_log_analyzer/sources/manager.py @@ -151,9 +151,9 @@ def load_from_env(self) -> list[str]: username=fields.pop("username", ""), password=fields.pop("password", ""), cookies=cookies, + verify_tls=fields.pop("verify_tls", "true").lower() != "false", + timeout_seconds=float(fields.pop("timeout_seconds", "30")), extra={k: v for k, v in fields.items() if v}, - verify_tls=fields.get("verify_tls", "true").lower() != "false", - timeout_seconds=float(fields.get("timeout_seconds", "30")), ) try: self.add(cfg) diff --git a/src/ai_log_analyzer/sources/syslog.py b/src/ai_log_analyzer/sources/syslog.py index d0342ce..b250edb 100644 --- a/src/ai_log_analyzer/sources/syslog.py +++ b/src/ai_log_analyzer/sources/syslog.py @@ -59,10 +59,12 @@ def healthcheck(self) -> bool: def fetch(self, *, since_seconds: int = 3600, limit: int = 10_000, host_filter: str = "") -> Iterable[LogEvent]: - # Drain up to `limit` lines from the buffer (newest-last order preserved). + # Snapshot up to `limit` lines (newest-last order preserved) WITHOUT + # draining. Every other connector is idempotent; clearing here meant a + # correlate → triage sequence on the same source saw zero events on + # the second call. The deque's maxlen already bounds memory. with self._lock: lines = list(self._buffer) - self._buffer.clear() if limit and len(lines) > limit: lines = lines[-int(limit):] for ev in parse_lines(lines, default_host=host_filter or "syslog"): diff --git a/src/ai_log_analyzer/web/app.py b/src/ai_log_analyzer/web/app.py index eeddb82..c9ceb26 100644 --- a/src/ai_log_analyzer/web/app.py +++ b/src/ai_log_analyzer/web/app.py @@ -98,9 +98,20 @@ class BadCommand(ValueError): """Raised when /api/run receives a command we won't translate to argv.""" +def _token_supplied() -> str: + """The API token from the request, from either accepted header form.""" + supplied = request.headers.get("X-API-Token", "") + if not supplied: + auth = request.headers.get("Authorization", "") + if auth.startswith("Bearer "): + supplied = auth[len("Bearer "):].strip() + return supplied + + def require_api_token(fn): - """Require X-API-Token header when AI_LOG_ANALYZER_API_TOKEN is set. + """Require the API token when AI_LOG_ANALYZER_API_TOKEN is set. + Accepted as `X-API-Token: ` or `Authorization: Bearer `. When the env var is empty (dev mode + bound to 127.0.0.1) this decorator is a no-op. Once set, every mutating route must present the token. """ @@ -108,8 +119,7 @@ def require_api_token(fn): def wrapper(*args, **kwargs): if not API_TOKEN: return fn(*args, **kwargs) - supplied = request.headers.get("X-API-Token", "") - if supplied != API_TOKEN: + if _token_supplied() != API_TOKEN: abort(401) return fn(*args, **kwargs) return wrapper @@ -240,7 +250,7 @@ def llm_status(): # last_errors carries upstream HTTP error bodies — useful in the UI, # but on a token-protected deployment don't hand it to anonymous # callers. Dev mode (no token) keeps full output for the dashboard. - if API_TOKEN and request.headers.get("X-API-Token", "") != API_TOKEN: + if API_TOKEN and _token_supplied() != API_TOKEN: state = {k: v for k, v in state.items() if k != "last_errors"} return jsonify(state) @@ -260,6 +270,36 @@ def llm_toggle(): enabled = bool(body.get("enabled", True)) return jsonify({"ok": True, "enabled": llm.set_enabled(enabled)}) + # ── Classification rules ────────────────────────────────────────────── + @app.route("/api/rules", methods=["GET"]) + def api_rules_list(): + """Built-in rule count + the operator's custom classification rules.""" + from ai_log_analyzer import classifier as _clf + return jsonify({ + "builtin_count": len(_clf._KB_PATTERNS), + "custom": _clf.custom_rules(), + }) + + @app.route("/api/rules", methods=["POST"]) + @require_api_token + def api_rules_add(): + """Register custom classification rules at runtime (non-persistent). + + Body: {"rules": [{"pattern": "...", "severity": "high", + "category": "...", "description": "..."}]} + Persist across restarts by putting the same list in a JSON file and + pointing AI_LOG_ANALYZER_CUSTOM_RULES at it. + """ + from ai_log_analyzer import classifier as _clf + body = request.get_json(silent=True) or {} + rules = body.get("rules") + if not isinstance(rules, list) or not rules: + return jsonify({"error": "body must be {\"rules\": [...]} with ≥1 rule"}), 400 + added, errors = _clf.add_custom_rules(rules) + status = 200 if added else 400 + return jsonify({"added": added, "errors": errors, + "custom_total": len(_clf.custom_rules())}), status + # ── Inventory ───────────────────────────────────────────────────────── @app.route("/api/lab/containers", methods=["GET"]) def lab_containers(): @@ -561,26 +601,15 @@ def api_optimize_site(): safe = "".join(c for c in site_id if c.isalnum() or c in "-_").lower() if safe != site_id.lower(): return jsonify({"error": "invalid site id"}), 400 - site_dir = SITES_DIR / safe - manifest_path = site_dir / "manifest.json" - if not manifest_path.is_file(): - return jsonify({"error": f"site bundle not found: {safe}"}), 404 if not llm.get_state()["enabled"]: return jsonify({"error": "LLM is disabled — site analysis needs LLM"}), 400 - import json as _json - manifest = _json.loads(manifest_path.read_text(encoding="utf-8")) - devices: list[dict] = [] - for d in manifest.get("devices", []): - fpath = site_dir / d["file"] - if not fpath.is_file(): - continue - devices.append({ - "hostname": d.get("hostname", fpath.stem), - "function": d.get("function", "unknown"), - "platform": manifest.get("vendor", "junos").lower(), - "config_text": fpath.read_text(encoding="utf-8", errors="replace"), - }) + # Shared loader keeps per-device platform intact — the old inline loop + # stamped every device with the manifest-level vendor string, which is + # wrong on mixed-vendor sites (e.g. "multi (Nokia SRL + Arista cEOS…)"). + devices, _manifest = _load_site_devices(safe) + if devices is None: + return jsonify({"error": f"site bundle not found: {safe}"}), 404 if not devices: return jsonify({"error": "no device configs found in bundle"}), 404 @@ -851,10 +880,12 @@ def api_sources_remove(source_id: str): return jsonify({"ok": removed}) @app.route("/api/sources//test", methods=["POST"]) + @require_api_token def api_sources_test(source_id: str): return jsonify(source_manager.healthcheck(source_id)) @app.route("/api/sources//fetch", methods=["POST"]) + @require_api_token def api_sources_fetch(source_id: str): """Pull a batch of events from a registered source. Body: {since_seconds, limit, host_filter}.""" diff --git a/src/ai_log_analyzer/web/static/app.js b/src/ai_log_analyzer/web/static/app.js index f9b4174..26a8cda 100644 --- a/src/ai_log_analyzer/web/static/app.js +++ b/src/ai_log_analyzer/web/static/app.js @@ -770,6 +770,35 @@ function render(r) { aTbody.appendChild(tr); }); + // Unknown patterns (mined templates of lines no KB rule matched) + const up = r.unknown_patterns || {}; + const upTemplates = up.top_templates || []; + const upPanel = $("unknown-panel"); + if (upPanel) { + if (upTemplates.length) { + upPanel.style.display = "block"; + $("unknown-count").textContent = + `(${up.template_count} templates · ${up.total_unclassified} lines)`; + const uTbody = $("unknown-table").querySelector("tbody"); + clear(uTbody); + upTemplates.forEach((t) => { + const hostLabel = t.host_count > 3 + ? `${t.host_count}: ${t.hosts.slice(0, 3).join(", ")}…` + : t.hosts.join(", "); + const flags = (t.severity_hint !== "info" ? "⚠" : "") + (t.is_new ? " 🆕" : ""); + const tr = el("tr", { className: t.severity_hint !== "info" ? "sev-row-medium" : "" }, + el("td", { text: flags.trim() }), + el("td", {}, el("span", { className: "row-msg", text: t.template })), + el("td", { text: String(t.count) }), + el("td", { text: hostLabel }), + ); + uTbody.appendChild(tr); + }); + } else { + upPanel.style.display = "none"; + } + } + // Events $("events-panel").style.display = "block"; const totalEvents = (r.classified_events || []).length; diff --git a/src/ai_log_analyzer/web/static/index.html b/src/ai_log_analyzer/web/static/index.html index b140852..780907a 100644 --- a/src/ai_log_analyzer/web/static/index.html +++ b/src/ai_log_analyzer/web/static/index.html @@ -231,6 +231,7 @@ #health-panel { contain-intrinsic-size: 0 320px; } #summary-panel { contain-intrinsic-size: 0 260px; } #actions-panel { contain-intrinsic-size: 0 380px; } + #unknown-panel { contain-intrinsic-size: 0 300px; } #events-panel { contain-intrinsic-size: 0 540px; } #optimize-panel { contain-intrinsic-size: 0 800px; } #site-panel { contain-intrinsic-size: 0 720px; } @@ -1436,6 +1437,17 @@

Action Items

+ +