fix: community rules follow-up — JA4 rules, tests, shadow logging

cmd/ja4monitor/main.go

@@ -492,6 +492,7 @@ func reloadEvaluator(configFile string, evaluator *anomaly.Evaluator) {
 		log.Printf("SIGHUP: community rules load failed, keeping old rules: %v", err)
 		return
 	}
+	logShadowedRules(communityRuleCfgs, customRuleCfgs)
 	merged := config.MergeRules(communityRuleCfgs, customRuleCfgs)
 	if err := evaluator.ReloadCustomRules(merged); err != nil {
 		log.Printf("SIGHUP: custom rules reload failed, keeping old rules: %v", err)
@@ -512,6 +513,25 @@ func reloadEvaluator(configFile string, evaluator *anomaly.Evaluator) {
 	}
 }
 
+// logShadowedRules logs a warning for each community rule whose name matches
+// a user-defined custom rule. The custom rule takes priority (shadow-merge),
+// so this is purely informational — it helps operators debug unexpected
+// detection gaps caused by accidental name collisions.
+func logShadowedRules(community, custom []config.CustomRuleConfig) {
+	if len(custom) == 0 {
+		return
+	}
+	customNames := make(map[string]struct{}, len(custom))
+	for _, r := range custom {
+		customNames[r.Name] = struct{}{}
+	}
+	for _, r := range community {
+		if _, shadowed := customNames[r.Name]; shadowed {
+			log.Printf("rules: user rule %q shadows community rule of same name", r.Name)
+		}
+	}
+}
+
 // initPipeline wires up the persistent state: SQLite store, first-seen
 // learning map, and anomaly evaluator. The engine stats counter gets plumbed
 // into the store (for flush failures) and the evaluator (for alert drops)
@@ -536,6 +556,7 @@ func initPipeline(cfg config.Config, stats *engine.EngineStats) (*storage.Store,
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("load community rules: %w", err)
 	}
+	logShadowedRules(communityRuleCfgs, customRuleCfgs)
 	allRules := config.MergeRules(communityRuleCfgs, customRuleCfgs)
 	log.Printf("rules: %d community + %d custom = %d active",
 		len(communityRuleCfgs), len(customRuleCfgs), len(allRules))

internal/anomaly/community_rules_integration_test.go

@@ -0,0 +1,74 @@
+package anomaly
+
+import (
+	"testing"
+
+	"github.com/Crank-Git/ja4monitor/internal/config"
+)
+
+// TestCommunityRules_FireThroughEvaluator is an end-to-end integration test
+// that verifies community rules are correctly wired from LoadCommunityRules
+// through NewCustomRules and into Evaluate — the same path used by
+// initPipeline in main.go.
+//
+// If the LoadCommunityRules → MergeRules → NewCustomRules chain in main.go is
+// ever accidentally broken, this test will catch it before the binary ships.
+//
+// Rule under test: port_scan_detector
+//   aggregation = "count_distinct", field = "dst_port", group_by = "src_ip"
+//   threshold = 20, window = "60s", condition = "gt"
+//
+// To trigger it: send 21 connections from one src_ip to 21 distinct dst_ports.
+func TestCommunityRules_FireThroughEvaluator(t *testing.T) {
+	cfgs, err := config.LoadCommunityRules()
+	if err != nil {
+		t.Fatalf("LoadCommunityRules: %v", err)
+	}
+	if len(cfgs) == 0 {
+		t.Fatal("no community rules loaded; expected at least one")
+	}
+
+	rules, err := NewCustomRules(cfgs)
+	if err != nil {
+		t.Fatalf("NewCustomRules: %v", err)
+	}
+
+	// Locate port_scan_detector so we only exercise it and avoid noise from
+	// other rules that might fire on unrelated fields.
+	var portScanRule Rule
+	for _, r := range rules {
+		if r.Name() == "port_scan_detector" {
+			portScanRule = r
+			break
+		}
+	}
+	if portScanRule == nil {
+		t.Fatal("port_scan_detector not found in community rules; was it renamed or removed?")
+	}
+
+	// Threshold is gt=20 on count_distinct(dst_port) grouped by src_ip.
+	// Send 21 connections from the same src_ip, each on a distinct port.
+	// The rule must not fire on calls 1-20 and must fire on call 21.
+	srcIP := "192.168.100.1"
+	var firedAlert *Alert
+	for port := uint16(1); port <= 21; port++ {
+		conn := makeConn(srcIP, "10.0.0.1", port)
+		alerts := portScanRule.Evaluate(conn)
+		if port < 21 && len(alerts) != 0 {
+			t.Errorf("port_scan_detector fired early at %d distinct ports (threshold is gt=20)", port)
+		}
+		if port == 21 && len(alerts) == 1 {
+			firedAlert = &alerts[0]
+		}
+	}
+
+	if firedAlert == nil {
+		t.Fatal("port_scan_detector did not fire after 21 distinct dst_ports from one src_ip")
+	}
+	if firedAlert.Rule != "port_scan_detector" {
+		t.Errorf("alert.Rule = %q, want %q", firedAlert.Rule, "port_scan_detector")
+	}
+	if firedAlert.SrcIP != srcIP {
+		t.Errorf("alert.SrcIP = %q, want %q", firedAlert.SrcIP, srcIP)
+	}
+}

internal/config/community_rules_test.go

@@ -112,6 +112,17 @@ func TestMergeRules_EmptyCommunity(t *testing.T) {
 	}
 }
 
+// TestMergeRules_BothNil verifies that MergeRules(nil, nil) returns nil
+// without panicking. This is an edge case that cannot occur in normal
+// production operation (LoadCommunityRules always returns at least one rule),
+// but guards against panics if the wiring in main.go ever changes.
+func TestMergeRules_BothNil(t *testing.T) {
+	result := MergeRules(nil, nil)
+	if result != nil {
+		t.Errorf("MergeRules(nil, nil) = %v, want nil", result)
+	}
+}
+
 // TestMergeRules_OrderPreserved verifies that community rules come before
 // custom rules in the merged output (for stable ordering in logs).
 func TestMergeRules_OrderPreserved(t *testing.T) {

internal/config/rules.d/c2_malware.toml

@@ -8,6 +8,8 @@
 #   tls_fingerprint_flood  — lower threshold to 20 for very quiet networks
 #   rapid_beaconing        — raise to 240+ for high-throughput servers
 #   slow_beaconing         — lower window to 30m for tighter detection
+#   new_ja4_burst          — raise to 30+ on developer workstation segments
+#   dga_cert_diversity     — enable only after confirming JA4x capture is active
 
 # ---------------------------------------------------------------------------
 # TLS fingerprint flood
@@ -71,3 +73,55 @@ group_by    = "src_ip+dst_ip"
 threshold   = 60
 window      = "1h"
 condition   = "gt"
+
+# ---------------------------------------------------------------------------
+# JA4 fingerprint burst (sliding window)
+# Fires when a single source IP presents more than 20 distinct JA4 TLS client
+# fingerprints within any rolling 5-minute window. A legitimate workstation
+# running a browser and common background tools produces 8–15 distinct JA4
+# values at startup. Exceeding 20 indicates a scanner, C2 implant cycling TLS
+# configs, or automated tooling that iterates through TLS configurations.
+#
+# Note: only fires on connections with completed TLS handshakes (JA4 captured).
+# TUNE: raise to 30+ on developer workstation segments; lower to 15 on
+#       server/IoT segments where TLS client diversity should be minimal.
+# ---------------------------------------------------------------------------
+[[rule]]
+name        = "new_ja4_burst"
+description = "Source IP presented more than 20 distinct JA4 fingerprints in any 5-minute window"
+enabled     = true
+severity    = "medium"
+type        = "threshold"
+aggregation = "count_distinct"
+field       = "ja4"
+group_by    = "src_ip"
+threshold   = 20
+window      = "5m"
+window_type = "sliding"
+condition   = "gt"
+
+# ---------------------------------------------------------------------------
+# DGA certificate diversity (sliding window)
+# Fires when a single source IP sees more than 40 distinct JA4x server
+# certificate fingerprints within any rolling 5-minute window. DGA and
+# fast-flux C2 infrastructure rotates certificates constantly — a host hitting
+# 40+ different server certs in 5 minutes is resolving and connecting to
+# DGA backends or fast-flux infrastructure, not browsing.
+#
+# Note: only fires on connections with completed TLS handshakes (JA4x captured).
+# Enable only after confirming that JA4x server-side fingerprinting is active
+# (ja4x fingerprints visible in the TUI or alert output).
+# ---------------------------------------------------------------------------
+[[rule]]
+name        = "dga_cert_diversity"
+description = "Source IP saw more than 40 distinct server certificate fingerprints in any 5-minute window"
+enabled     = true
+severity    = "high"
+type        = "threshold"
+aggregation = "count_distinct"
+field       = "ja4x"
+group_by    = "src_ip"
+threshold   = 40
+window      = "5m"
+window_type = "sliding"
+condition   = "gt"

internal/config/rules.d/lateral_movement.toml

@@ -42,7 +42,7 @@ condition   = "gt"
 # ---------------------------------------------------------------------------
 [[rule]]
 name        = "ssh_connection_burst"
-description = "Source opened more than 20 SSH connections in any 60s window"
+description = "Source connected to more than 20 distinct hosts via SSH in any 60s window"
 enabled     = true
 severity    = "high"
 type        = "threshold"