feat/uniform detectors

.gitignore

@@ -25,3 +25,6 @@ clickguard-*.tar
 # dialyzer
 /priv/plts/*.plt
 /priv/plts/*.plt.hash
+
+# misc
+.DS_Store

lib/clickguard/detector/referer.ex

@@ -1,15 +1,15 @@
 defmodule Clickguard.Detector.Referer do
   @moduledoc """
-  Surfaces requests with empty or known-spam referers
+  Flags requests with empty or known-spam referers, keyed by IP.
 
-  Rules:
+  Rules (Subject = IP):
     * :empty_referer - referer is nil/blank. Weak signal (privacy browsers,
       Referrer-Policy, HTTPS->HTTP downgrade all strip it legitimately).
     * :spam_referer  - referer host matches a known referer-spam domain.
-      Subject is the matched domain.
 
-  NOTE: subject semantics differ by rule (sentinel vs domain). Never reason
-  about a referer finding's subject without branching on :rule.
+  Subject is uniformly the IP (symmetric with UserAgent). The domain-level
+  view ("domain.com across N IPs") is recoverable vie a later group_by on
+  evidence.matched_referers - deferred, not lost.
 
   Deferred (need deployment config): publisher mismatch, self-referencing loops.
 
@@ -20,13 +20,9 @@ defmodule Clickguard.Detector.Referer do
 
   @type evidence :: %{
           event_count: pos_integer(),
-          ips: [String.t()],
-          ips_truncated: boolean(),
-          matched_value: String.t() | nil
+          matched_referers: [String.t()]
         }
 
-  @empty_subject "(none)"
-
   @default_spam_domains ~w(
     brandedleadgeneration.com addshoppers.com 7minuteworkout.com
   )
@@ -39,70 +35,53 @@ defmodule Clickguard.Detector.Referer do
     spam = Keyword.get(opts, :spam_domains, @default_spam_domains) |> MapSet.new()
     detected_at = DateTime.now!("Etc/UTC")
 
-    empty_findings(events, detected_at) ++ spam_findings(events, spam, detected_at)
+    events
+    |> Enum.flat_map(fn e ->
+      case classify(e, spam) do
+        nil -> []
+        {rule, value} -> [{{Event.ip_string(e), rule}, {e, value}}]
+      end
+    end)
+    |> Enum.group_by(fn {key, _} -> key end, fn {_, pair} -> pair end)
+    |> Enum.map(fn {{ip, rule}, pairs} -> build_finding(rule, ip, pairs, detected_at) end)
   end
 
-  defp empty_findings(events, detected_at) do
-    case Enum.filter(events, &blank_referer?/1) do
-      [] -> []
-      evts -> [build_finding(:empty_referer, @empty_subject, nil, evts, detected_at)]
-    end
+  defp classify(%Event{referer: r}, spam) do
+    if blank?(r), do: {:empty_referer, nil}, else: spam_match(r, spam)
   end
 
-  defp spam_findings(events, spam, detected_at) do
-    events
-    |> Enum.flat_map(fn e ->
-      case spam_host(e, spam) do
-        nil ->
-          []
+  defp spam_match(referer, spam) do
+    case URI.parse(String.trim(referer)).host do
+      nil ->
+        nil
 
-        host ->
-          [{host, e}]
-      end
-    end)
-    |> Enum.group_by(fn {host, _} -> host end, fn {_, e} -> e end)
-    |> Enum.map(fn {host, evts} ->
-      build_finding(:spam_referer, host, host, evts, detected_at)
-    end)
+      host ->
+        normalized = host |> String.downcase() |> String.replace_prefix("www.", "")
+        if MapSet.member?(spam, normalized), do: {:spam_referer, normalized}, else: nil
+    end
   end
 
-  defp build_finding(rule, subject, matched, evts, detected_at) do
-    ips = evts |> Enum.map(&Event.ip_string/1) |> Enum.uniq() |> Enum.reject(&is_nil/1)
+  defp build_finding(rule, ip, pairs, detected_at) do
+    events = Enum.map(pairs, fn {e, _} -> e end)
 
     %Finding{
       rule: rule,
       severity: :low,
-      subject: subject,
-      evidence: %{
-        event_count: length(evts),
-        ips: ips |> Enum.take(50),
-        ips_truncated: length(ips) > 50,
-        matched_value: matched
-      },
-      sample_events: Event.sample(evts),
+      subject: ip,
+      evidence: build_evidence(pairs),
+      sample_events: Event.sample(events),
       detected_at: detected_at
     }
   end
 
-  defp blank_referer?(%Event{referer: nil}), do: true
-  defp blank_referer?(%Event{referer: r}), do: String.trim(r) == ""
-
-  defp spam_host(%Event{referer: r}, _spam) when is_nil(r), do: nil
-
-  defp spam_host(%Event{referer: r}, spam) do
-    case String.trim(r) do
-      "" -> nil
-      trimmed -> match_host(URI.parse(trimmed).host, spam)
-    end
+  defp build_evidence(pairs) do
+    %{
+      event_count: length(pairs),
+      matched_referers:
+        pairs |> Enum.map(fn {_, v} -> v end) |> Enum.reject(&is_nil/1) |> Enum.uniq()
+    }
   end
 
-  defp match_host(nil, _spam), do: nil
-
-  defp match_host(host, spam) do
-    normalized =
-      String.downcase(host)
-      |> String.replace_prefix("www.", "")
-
-    if MapSet.member?(spam, normalized), do: normalized, else: nil
-  end
+  defp blank?(nil), do: true
+  defp blank?(r), do: String.trim(r) == ""
 end

lib/clickguard/detector/user_agent.ex

@@ -2,7 +2,7 @@ defmodule Clickguard.Detector.UserAgent do
   @moduledoc """
   Flags requests by automated/headless user agents, keyed by IP.
 
-  Rules (Subject IP):
+  Rules (Subject = IP):
     * :empty_ua - UA nil/blank
     * :automation_tool - python-requests, curl, wget, Go-http-client, Scrapy
     * :headless_browser - HeadlessChrome, PhantomJS

lib/clickguard/fixtures.ex

@@ -58,7 +58,7 @@ defmodule Clickguard.Fixtures do
 
     output =
       (freqip_lines ++ good_lines ++ bad_ua_lines ++ bad_referer_lines)
-      # |> Enum.shuffle()
+      |> Enum.shuffle()
       |> Enum.join("\n")
 
     File.mkdir_p!(Path.dirname(out))
@@ -89,26 +89,28 @@ defmodule Clickguard.Fixtures do
     [first, second | rest] = @bad_user_agents
 
     single_ua_per_ip = [
-      clf_line(ip: "10.0.0.1", ts: ts(0), ua: first),
-      clf_line(ip: "10.0.0.2", ts: ts(10), ua: second)
+      clf_line(ip: "10.0.0.1", ts: ts(0), user_agent: first),
+      clf_line(ip: "10.0.0.2", ts: ts(10), user_agent: second)
     ]
 
     multiple_ua_per_ip =
-      for ua <- rest, do: clf_line(ip: "10.0.0.10", ts: ts(10), ua: ua)
+      for ua <- rest, do: clf_line(ip: "10.0.0.10", ts: ts(10), user_agent: ua)
 
     single_ua_per_ip ++ multiple_ua_per_ip
   end
 
   defp generate_bad_referer_lines do
-    [first | rest] = @bad_referers
+    [first, second | rest] = @bad_referers
 
-    one_domain_across_many_ips =
-      for n <- 0..4, do: clf_line(ip: "192.168.0.#{n + 1}", ts: ts(n), ref: first)
+    single_ref_per_ip = [
+      clf_line(ip: "192.168.1.1", ts: ts(0), referer: first),
+      clf_line(ip: "192.168.1.2", ts: ts(10), referer: second)
+    ]
 
-    other_events =
-      for ref <- rest, do: clf_line(ip: "192.168.0.10", ts: ts(10), ref: ref)
+    multiple_ref_per_ip =
+      for ref <- rest, do: clf_line(ip: "192.168.1.10", ts: ts(10), referer: ref)
 
-    one_domain_across_many_ips ++ other_events
+    single_ref_per_ip ++ multiple_ref_per_ip
   end
 
   defp clf_line(fields) do
@@ -121,8 +123,8 @@ defmodule Clickguard.Fixtures do
     http_version = Keyword.get_lazy(fields, :http_version, &http_version/0)
     response_code = Keyword.get_lazy(fields, :response_code, &response_code/0)
     size = Keyword.get_lazy(fields, :size, &size/0)
-    ua = Keyword.get_lazy(fields, :ua, &user_agent/0)
-    ref = Keyword.get_lazy(fields, :ref, &referer/0)
+    ua = Keyword.get_lazy(fields, :user_agent, &user_agent/0)
+    ref = Keyword.get_lazy(fields, :referer, &referer/0)
 
     "#{ip} #{identity} #{username} #{ts} \"#{method} #{target} HTTP/#{http_version}\" #{response_code} #{size} \"#{ref}\" \"#{ua}\""
   end