feat: community detection rule library — 9 curated rules embedded in binary

cmd/ja4monitor/main.go

@@ -481,16 +481,24 @@ func reloadEvaluator(configFile string, evaluator *anomaly.Evaluator) {
 
 	// Reload custom rules (independent of allowlist). Note: window state for
 	// reloaded rules resets to zero — any in-progress windows are discarded.
+	// Community rules are static (embedded in binary); only custom rules reload.
 	customRuleCfgs, err := config.LoadCustomRules(cfg.CustomRulesFile)
 	if err != nil {
 		log.Printf("SIGHUP: custom rules reload failed, keeping old rules: %v", err)
 		return
 	}
-	if err := evaluator.ReloadCustomRules(customRuleCfgs); err != nil {
+	communityRuleCfgs, err := config.LoadCommunityRules()
+	if err != nil {
+		log.Printf("SIGHUP: community rules load failed, keeping old rules: %v", err)
+		return
+	}
+	merged := config.MergeRules(communityRuleCfgs, customRuleCfgs)
+	if err := evaluator.ReloadCustomRules(merged); err != nil {
 		log.Printf("SIGHUP: custom rules reload failed, keeping old rules: %v", err)
 		return
 	}
-	log.Printf("SIGHUP: custom rules reloaded — %d enabled rules", len(customRuleCfgs))
+	log.Printf("SIGHUP: rules reloaded — %d community + %d custom = %d active",
+		len(communityRuleCfgs), len(customRuleCfgs), len(merged))
 
 	// Reload behavioral profiler. Learned profiles are reset on reload (same
 	// semantics as custom rules resetting window state). The new profiler picks
@@ -524,8 +532,15 @@ func initPipeline(cfg config.Config, stats *engine.EngineStats) (*storage.Store,
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("load custom rules: %w", err)
 	}
+	communityRuleCfgs, err := config.LoadCommunityRules()
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("load community rules: %w", err)
+	}
+	allRules := config.MergeRules(communityRuleCfgs, customRuleCfgs)
+	log.Printf("rules: %d community + %d custom = %d active",
+		len(communityRuleCfgs), len(customRuleCfgs), len(allRules))
 
-	evaluator, err := anomaly.NewEvaluator(cfg.Rules, cfg.Alerting, cfg.AllowlistFile, customRuleCfgs, cfg.Behavior)
+	evaluator, err := anomaly.NewEvaluator(cfg.Rules, cfg.Alerting, cfg.AllowlistFile, allRules, cfg.Behavior)
 	if err != nil {
 		return nil, nil, nil, fmt.Errorf("init evaluator: %w", err)
 	}

internal/config/community_rules.go

@@ -0,0 +1,77 @@
+package config
+
+import (
+	"embed"
+	"fmt"
+	"io/fs"
+	"path/filepath"
+
+	"github.com/BurntSushi/toml"
+)
+
+//go:embed rules.d/*.toml
+var communityRulesFS embed.FS
+
+// LoadCommunityRules returns all rules embedded in the binary from rules.d/.
+// These rules ship with every release and are not reloaded on SIGHUP.
+//
+// If the caller also has user-defined custom rules, merge them with
+// MergeRules so that user rules take priority when names collide.
+func LoadCommunityRules() ([]CustomRuleConfig, error) {
+	entries, err := fs.ReadDir(communityRulesFS, "rules.d")
+	if err != nil {
+		return nil, fmt.Errorf("read embedded rules.d: %w", err)
+	}
+
+	var all []CustomRuleConfig
+	for _, e := range entries {
+		if e.IsDir() || filepath.Ext(e.Name()) != ".toml" {
+			continue
+		}
+		data, err := communityRulesFS.ReadFile("rules.d/" + e.Name())
+		if err != nil {
+			return nil, fmt.Errorf("read %s: %w", e.Name(), err)
+		}
+		var f customRulesFile
+		if err := toml.Unmarshal(data, &f); err != nil {
+			return nil, fmt.Errorf("parse %s: %w", e.Name(), err)
+		}
+		if err := validateCustomRules(f.Rule); err != nil {
+			return nil, fmt.Errorf("validate %s: %w", e.Name(), err)
+		}
+		all = append(all, f.Rule...)
+	}
+	return all, nil
+}
+
+// MergeRules combines community rules and user custom rules into a single
+// slice. User rules take priority: any community rule whose name matches a
+// user rule is silently dropped. This lets operators disable or override a
+// community rule by adding a rule with the same name to their custom file
+// (set enabled=false to disable without writing a replacement).
+//
+// The returned slice has community rules first, then user rules — consistent
+// ordering for stable logging and deterministic test assertions.
+func MergeRules(community, custom []CustomRuleConfig) []CustomRuleConfig {
+	if len(community) == 0 {
+		return custom
+	}
+	if len(custom) == 0 {
+		return community
+	}
+
+	// Build a set of user rule names for O(1) lookup.
+	userNames := make(map[string]struct{}, len(custom))
+	for _, r := range custom {
+		userNames[r.Name] = struct{}{}
+	}
+
+	merged := make([]CustomRuleConfig, 0, len(community)+len(custom))
+	for _, r := range community {
+		if _, shadowed := userNames[r.Name]; !shadowed {
+			merged = append(merged, r)
+		}
+	}
+	merged = append(merged, custom...)
+	return merged
+}

internal/config/community_rules_test.go

@@ -0,0 +1,138 @@
+package config
+
+import (
+	"testing"
+)
+
+// TestLoadCommunityRules_Valid verifies that all embedded community rule
+// files parse and validate without error, and that at least one rule is
+// returned (guards against accidentally empty rule sets after a refactor).
+func TestLoadCommunityRules_Valid(t *testing.T) {
+	rules, err := LoadCommunityRules()
+	if err != nil {
+		t.Fatalf("LoadCommunityRules: %v", err)
+	}
+	if len(rules) == 0 {
+		t.Fatal("LoadCommunityRules returned 0 rules; expected at least one")
+	}
+	t.Logf("loaded %d community rules", len(rules))
+
+	// Spot-check that each rule has required fields filled in.
+	for _, r := range rules {
+		if r.Name == "" {
+			t.Errorf("community rule has empty name: %+v", r)
+		}
+		if r.Type == "" {
+			t.Errorf("rule %q has empty type", r.Name)
+		}
+		if r.Severity == "" {
+			t.Errorf("rule %q has empty severity", r.Name)
+		}
+	}
+}
+
+// TestLoadCommunityRules_NoDuplicateNames verifies that no two community
+// rules share a name (which would cause a validation error when loaded
+// alongside custom rules).
+func TestLoadCommunityRules_NoDuplicateNames(t *testing.T) {
+	rules, err := LoadCommunityRules()
+	if err != nil {
+		t.Fatalf("LoadCommunityRules: %v", err)
+	}
+
+	seen := make(map[string]struct{}, len(rules))
+	for _, r := range rules {
+		if _, dup := seen[r.Name]; dup {
+			t.Errorf("duplicate community rule name: %q", r.Name)
+		}
+		seen[r.Name] = struct{}{}
+	}
+}
+
+// TestMergeRules_UserShadowsCommunity verifies that when a user rule has
+// the same name as a community rule, the community rule is dropped and the
+// user rule is kept.
+func TestMergeRules_UserShadowsCommunity(t *testing.T) {
+	community := []CustomRuleConfig{
+		{Name: "port_scan_detector", Severity: "high"},
+		{Name: "host_sweep_detector", Severity: "high"},
+	}
+	custom := []CustomRuleConfig{
+		// User overrides port_scan_detector with a stricter threshold.
+		{Name: "port_scan_detector", Severity: "critical"},
+	}
+
+	merged := MergeRules(community, custom)
+
+	if len(merged) != 2 {
+		t.Fatalf("expected 2 merged rules, got %d", len(merged))
+	}
+
+	var portScan *CustomRuleConfig
+	for i := range merged {
+		if merged[i].Name == "port_scan_detector" {
+			portScan = &merged[i]
+		}
+	}
+	if portScan == nil {
+		t.Fatal("port_scan_detector not found in merged rules")
+	}
+	if portScan.Severity != "critical" {
+		t.Errorf("expected user rule to shadow community rule; got severity %q", portScan.Severity)
+	}
+}
+
+// TestMergeRules_EmptyCustom verifies that when custom is empty,
+// MergeRules returns the community rules unchanged.
+func TestMergeRules_EmptyCustom(t *testing.T) {
+	community := []CustomRuleConfig{
+		{Name: "rule_a", Severity: "low"},
+		{Name: "rule_b", Severity: "medium"},
+	}
+
+	merged := MergeRules(community, nil)
+	if len(merged) != 2 {
+		t.Fatalf("expected 2 rules, got %d", len(merged))
+	}
+}
+
+// TestMergeRules_EmptyCommunity verifies that when community is empty,
+// MergeRules returns the custom rules unchanged.
+func TestMergeRules_EmptyCommunity(t *testing.T) {
+	custom := []CustomRuleConfig{
+		{Name: "user_rule", Severity: "high"},
+	}
+
+	merged := MergeRules(nil, custom)
+	if len(merged) != 1 {
+		t.Fatalf("expected 1 rule, got %d", len(merged))
+	}
+	if merged[0].Name != "user_rule" {
+		t.Errorf("unexpected rule name: %q", merged[0].Name)
+	}
+}
+
+// TestMergeRules_OrderPreserved verifies that community rules come before
+// custom rules in the merged output (for stable ordering in logs).
+func TestMergeRules_OrderPreserved(t *testing.T) {
+	community := []CustomRuleConfig{
+		{Name: "community_a"},
+		{Name: "community_b"},
+	}
+	custom := []CustomRuleConfig{
+		{Name: "custom_x"},
+		{Name: "custom_y"},
+	}
+
+	merged := MergeRules(community, custom)
+	if len(merged) != 4 {
+		t.Fatalf("expected 4 rules, got %d", len(merged))
+	}
+	// Community rules must come before custom rules.
+	if merged[0].Name != "community_a" || merged[1].Name != "community_b" {
+		t.Errorf("community rules not at front: %v", merged)
+	}
+	if merged[2].Name != "custom_x" || merged[3].Name != "custom_y" {
+		t.Errorf("custom rules not at back: %v", merged)
+	}
+}

internal/config/rules.d/c2_malware.toml

@@ -0,0 +1,73 @@
+# ja4monitor community rules — C2 and Malware Patterns
+#
+# Detect command-and-control beaconing, malware spawning many identical TLS
+# connections, and other high-frequency outbound patterns that suggest
+# automated tooling rather than organic user traffic.
+#
+# Tuning guidance:
+#   tls_fingerprint_flood  — lower threshold to 20 for very quiet networks
+#   rapid_beaconing        — raise to 240+ for high-throughput servers
+#   slow_beaconing         — lower window to 30m for tighter detection
+
+# ---------------------------------------------------------------------------
+# TLS fingerprint flood
+# Fires when more than 100 connections share the same JA4 fingerprint from
+# the same source IP within 30 seconds. Identical client hello fingerprints
+# at high frequency indicate automated tooling or malware spawning many
+# parallel TLS connections (e.g. botnet callback, scanner, or credential
+# stuffer reusing the same TLS library configuration).
+# ---------------------------------------------------------------------------
+[[rule]]
+name        = "tls_fingerprint_flood"
+description = "More than 100 connections with identical JA4 from same source in 30s"
+enabled     = true
+severity    = "high"
+type        = "threshold"
+aggregation = "count"
+field       = "connection"
+group_by    = "src_ip+ja4"
+threshold   = 100
+window      = "30s"
+condition   = "gt"
+
+# ---------------------------------------------------------------------------
+# Rapid beaconing (sliding window)
+# Fires when a single source IP makes more than 120 outbound connections in
+# any rolling 60-second window (2/sec average). Consistent high-frequency
+# outbound contact to any destination suggests C2 beaconing or exfiltration
+# that is rate-limiting itself just below the connection_flood threshold.
+# Use alongside behavioral profiler anomalies for higher-confidence detections.
+# ---------------------------------------------------------------------------
+[[rule]]
+name        = "rapid_beaconing"
+description = "Source IP made more than 120 outbound connections in any 60s window"
+enabled     = true
+severity    = "medium"
+type        = "threshold"
+aggregation = "count"
+field       = "connection"
+group_by    = "src_ip"
+threshold   = 120
+window      = "60s"
+window_type = "sliding"
+condition   = "gt"
+
+# ---------------------------------------------------------------------------
+# Slow beaconing — persistent low-frequency C2
+# Fires when a single source-to-destination pair accumulates more than 60
+# connections in a 1-hour tumbling window (~1/minute). Slow beaconing is a
+# common C2 evasion technique: low enough not to alarm rate-based detections,
+# but persistent enough to maintain reliable channel availability.
+# ---------------------------------------------------------------------------
+[[rule]]
+name        = "slow_beaconing"
+description = "Source-destination pair connected more than 60 times in 1 hour"
+enabled     = true
+severity    = "medium"
+type        = "threshold"
+aggregation = "count"
+field       = "connection"
+group_by    = "src_ip+dst_ip"
+threshold   = 60
+window      = "1h"
+condition   = "gt"