diff --git a/.gitignore b/.gitignore index 0e40326..db77a9c 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ clickguard-*.tar # dialyzer /priv/plts/*.plt /priv/plts/*.plt.hash + +# misc +.DS_Store diff --git a/lib/clickguard/detector/referer.ex b/lib/clickguard/detector/referer.ex index bb47f68..63beeaf 100644 --- a/lib/clickguard/detector/referer.ex +++ b/lib/clickguard/detector/referer.ex @@ -1,15 +1,15 @@ defmodule Clickguard.Detector.Referer do @moduledoc """ - Surfaces requests with empty or known-spam referers + Flags requests with empty or known-spam referers, keyed by IP. - Rules: + Rules (Subject = IP): * :empty_referer - referer is nil/blank. Weak signal (privacy browsers, Referrer-Policy, HTTPS->HTTP downgrade all strip it legitimately). * :spam_referer - referer host matches a known referer-spam domain. - Subject is the matched domain. - NOTE: subject semantics differ by rule (sentinel vs domain). Never reason - about a referer finding's subject without branching on :rule. + Subject is uniformly the IP (symmetric with UserAgent). The domain-level + view ("domain.com across N IPs") is recoverable vie a later group_by on + evidence.matched_referers - deferred, not lost. Deferred (need deployment config): publisher mismatch, self-referencing loops. @@ -20,13 +20,9 @@ defmodule Clickguard.Detector.Referer do @type evidence :: %{ event_count: pos_integer(), - ips: [String.t()], - ips_truncated: boolean(), - matched_value: String.t() | nil + matched_referers: [String.t()] } - @empty_subject "(none)" - @default_spam_domains ~w( brandedleadgeneration.com addshoppers.com 7minuteworkout.com ) @@ -39,70 +35,53 @@ defmodule Clickguard.Detector.Referer do spam = Keyword.get(opts, :spam_domains, @default_spam_domains) |> MapSet.new() detected_at = DateTime.now!("Etc/UTC") - empty_findings(events, detected_at) ++ spam_findings(events, spam, detected_at) + events + |> Enum.flat_map(fn e -> + case classify(e, spam) do + nil -> [] + {rule, value} -> [{{Event.ip_string(e), rule}, {e, value}}] + end + end) + |> Enum.group_by(fn {key, _} -> key end, fn {_, pair} -> pair end) + |> Enum.map(fn {{ip, rule}, pairs} -> build_finding(rule, ip, pairs, detected_at) end) end - defp empty_findings(events, detected_at) do - case Enum.filter(events, &blank_referer?/1) do - [] -> [] - evts -> [build_finding(:empty_referer, @empty_subject, nil, evts, detected_at)] - end + defp classify(%Event{referer: r}, spam) do + if blank?(r), do: {:empty_referer, nil}, else: spam_match(r, spam) end - defp spam_findings(events, spam, detected_at) do - events - |> Enum.flat_map(fn e -> - case spam_host(e, spam) do - nil -> - [] + defp spam_match(referer, spam) do + case URI.parse(String.trim(referer)).host do + nil -> + nil - host -> - [{host, e}] - end - end) - |> Enum.group_by(fn {host, _} -> host end, fn {_, e} -> e end) - |> Enum.map(fn {host, evts} -> - build_finding(:spam_referer, host, host, evts, detected_at) - end) + host -> + normalized = host |> String.downcase() |> String.replace_prefix("www.", "") + if MapSet.member?(spam, normalized), do: {:spam_referer, normalized}, else: nil + end end - defp build_finding(rule, subject, matched, evts, detected_at) do - ips = evts |> Enum.map(&Event.ip_string/1) |> Enum.uniq() |> Enum.reject(&is_nil/1) + defp build_finding(rule, ip, pairs, detected_at) do + events = Enum.map(pairs, fn {e, _} -> e end) %Finding{ rule: rule, severity: :low, - subject: subject, - evidence: %{ - event_count: length(evts), - ips: ips |> Enum.take(50), - ips_truncated: length(ips) > 50, - matched_value: matched - }, - sample_events: Event.sample(evts), + subject: ip, + evidence: build_evidence(pairs), + sample_events: Event.sample(events), detected_at: detected_at } end - defp blank_referer?(%Event{referer: nil}), do: true - defp blank_referer?(%Event{referer: r}), do: String.trim(r) == "" - - defp spam_host(%Event{referer: r}, _spam) when is_nil(r), do: nil - - defp spam_host(%Event{referer: r}, spam) do - case String.trim(r) do - "" -> nil - trimmed -> match_host(URI.parse(trimmed).host, spam) - end + defp build_evidence(pairs) do + %{ + event_count: length(pairs), + matched_referers: + pairs |> Enum.map(fn {_, v} -> v end) |> Enum.reject(&is_nil/1) |> Enum.uniq() + } end - defp match_host(nil, _spam), do: nil - - defp match_host(host, spam) do - normalized = - String.downcase(host) - |> String.replace_prefix("www.", "") - - if MapSet.member?(spam, normalized), do: normalized, else: nil - end + defp blank?(nil), do: true + defp blank?(r), do: String.trim(r) == "" end diff --git a/lib/clickguard/detector/user_agent.ex b/lib/clickguard/detector/user_agent.ex index ee61762..14230b8 100644 --- a/lib/clickguard/detector/user_agent.ex +++ b/lib/clickguard/detector/user_agent.ex @@ -2,7 +2,7 @@ defmodule Clickguard.Detector.UserAgent do @moduledoc """ Flags requests by automated/headless user agents, keyed by IP. - Rules (Subject IP): + Rules (Subject = IP): * :empty_ua - UA nil/blank * :automation_tool - python-requests, curl, wget, Go-http-client, Scrapy * :headless_browser - HeadlessChrome, PhantomJS diff --git a/lib/clickguard/fixtures.ex b/lib/clickguard/fixtures.ex index 4704b59..597953c 100644 --- a/lib/clickguard/fixtures.ex +++ b/lib/clickguard/fixtures.ex @@ -58,7 +58,7 @@ defmodule Clickguard.Fixtures do output = (freqip_lines ++ good_lines ++ bad_ua_lines ++ bad_referer_lines) - # |> Enum.shuffle() + |> Enum.shuffle() |> Enum.join("\n") File.mkdir_p!(Path.dirname(out)) @@ -89,26 +89,28 @@ defmodule Clickguard.Fixtures do [first, second | rest] = @bad_user_agents single_ua_per_ip = [ - clf_line(ip: "10.0.0.1", ts: ts(0), ua: first), - clf_line(ip: "10.0.0.2", ts: ts(10), ua: second) + clf_line(ip: "10.0.0.1", ts: ts(0), user_agent: first), + clf_line(ip: "10.0.0.2", ts: ts(10), user_agent: second) ] multiple_ua_per_ip = - for ua <- rest, do: clf_line(ip: "10.0.0.10", ts: ts(10), ua: ua) + for ua <- rest, do: clf_line(ip: "10.0.0.10", ts: ts(10), user_agent: ua) single_ua_per_ip ++ multiple_ua_per_ip end defp generate_bad_referer_lines do - [first | rest] = @bad_referers + [first, second | rest] = @bad_referers - one_domain_across_many_ips = - for n <- 0..4, do: clf_line(ip: "192.168.0.#{n + 1}", ts: ts(n), ref: first) + single_ref_per_ip = [ + clf_line(ip: "192.168.1.1", ts: ts(0), referer: first), + clf_line(ip: "192.168.1.2", ts: ts(10), referer: second) + ] - other_events = - for ref <- rest, do: clf_line(ip: "192.168.0.10", ts: ts(10), ref: ref) + multiple_ref_per_ip = + for ref <- rest, do: clf_line(ip: "192.168.1.10", ts: ts(10), referer: ref) - one_domain_across_many_ips ++ other_events + single_ref_per_ip ++ multiple_ref_per_ip end defp clf_line(fields) do @@ -121,8 +123,8 @@ defmodule Clickguard.Fixtures do http_version = Keyword.get_lazy(fields, :http_version, &http_version/0) response_code = Keyword.get_lazy(fields, :response_code, &response_code/0) size = Keyword.get_lazy(fields, :size, &size/0) - ua = Keyword.get_lazy(fields, :ua, &user_agent/0) - ref = Keyword.get_lazy(fields, :ref, &referer/0) + ua = Keyword.get_lazy(fields, :user_agent, &user_agent/0) + ref = Keyword.get_lazy(fields, :referer, &referer/0) "#{ip} #{identity} #{username} #{ts} \"#{method} #{target} HTTP/#{http_version}\" #{response_code} #{size} \"#{ref}\" \"#{ua}\"" end diff --git a/test/clickguard/detector/referer_test.exs b/test/clickguard/detector/referer_test.exs index b09c960..c3efdaf 100644 --- a/test/clickguard/detector/referer_test.exs +++ b/test/clickguard/detector/referer_test.exs @@ -7,70 +7,80 @@ defmodule Clickguard.Detector.RefererTest do @base_ts ~U[2016-05-24 13:26:08.003Z] @custom_spam_domains ["semalt.com", "badtestdomain.com"] - describe "detect/2 - :empty_referer rule" do + describe "detect/2 - common tests for all rules" do test "event with valid referer produces no findings" do event = EB.event({127, 0, 0, 1}, @base_ts, referer: "http://gooddomain.com") - assert Referer.detect([event], []) == [] + assert Referer.detect([event], spam_domains: @custom_spam_domains) == [] end - test "event without referer produces (none) finding" do - event = EB.event({127, 0, 0, 1}, @base_ts, referer: nil) - assert [f] = Referer.detect([event], []) - assert f.subject == "(none)" + test "multiple rules per IP produce separate findings" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, referer: nil), + EB.event({127, 0, 0, 1}, @base_ts, referer: "http://semalt.com") + ] + + assert [f1, f2] = Referer.detect(events, spam_domains: @custom_spam_domains) + rules = [f1, f2] |> Enum.map(& &1.rule) |> Enum.sort() + assert rules == [:empty_referer, :spam_referer] end - test "finding shape" do - event = EB.event({127, 0, 0, 1}, @base_ts, referer: nil) + test "one rule per IP produces separate findings" do + events = [ + EB.event({127, 0, 0, 1}, @base_ts, referer: nil), + EB.event({127, 0, 0, 2}, @base_ts, referer: "http://semalt.com") + ] + + assert [f1, f2] = Referer.detect(events, spam_domains: @custom_spam_domains) + rules = [f1, f2] |> Enum.map(& &1.rule) |> Enum.sort() + subjects = [f1, f2] |> Enum.map(& &1.subject) |> Enum.sort() + matched = [f1, f2] |> Enum.flat_map(& &1.evidence.matched_referers) + assert rules == [:empty_referer, :spam_referer] + assert subjects == ["127.0.0.1", "127.0.0.2"] + assert matched == ["semalt.com"] + end + end + describe ":detect/2 - :empty_referer rule" do + test "event without referer produces one finding" do + event = EB.event({127, 0, 0, 1}, @base_ts, referer: nil) assert [f] = Referer.detect([event], []) assert f.rule == :empty_referer assert f.severity == :low - assert f.subject == "(none)" + assert f.subject == "127.0.0.1" assert f.evidence.event_count == 1 - assert f.evidence.ips == ["127.0.0.1"] - assert f.evidence.ips_truncated == false - assert f.evidence.matched_value == nil - assert f.sample_events == [event] + assert f.evidence.matched_referers == [] end - test "event with whitespace-only referer produces (none) finding" do + test "event with whitespace-only referer produces one finding" do event = EB.event({127, 0, 0, 1}, @base_ts, referer: " ") assert [f] = Referer.detect([event], []) - assert f.subject == "(none)" + assert f.rule == :empty_referer + assert f.evidence.matched_referers == [] end - test "events with nil and whitespace-only referers collapse into one finding" do + test "two events with different IPs produce two findings" do events = [ EB.event({127, 0, 0, 1}, @base_ts, referer: nil), - EB.event({127, 0, 0, 2}, @base_ts, referer: nil), - EB.event({127, 0, 0, 3}, @base_ts, referer: " "), - EB.event({127, 0, 0, 4}, @base_ts, referer: " ") + EB.event({127, 0, 0, 2}, @base_ts, referer: nil) ] - assert [f] = Referer.detect(events, []) - assert f.subject == "(none)" - assert f.evidence.event_count == 4 - assert length(f.evidence.ips) == 4 - assert "127.0.0.1" in f.evidence.ips - assert "127.0.0.2" in f.evidence.ips - assert "127.0.0.3" in f.evidence.ips - assert "127.0.0.4" in f.evidence.ips - assert f.evidence.ips_truncated == false + assert [f1, f2] = Referer.detect(events, []) + assert f1.subject in ["127.0.0.1", "127.0.0.2"] + assert f2.subject in ["127.0.0.1", "127.0.0.2"] + assert f1.subject != f2.subject + assert f1.evidence.matched_referers == f2.evidence.matched_referers end - test "there are only unique IPs in finding" do + test "two events with the same IP produce one finding" do events = [ EB.event({127, 0, 0, 1}, @base_ts, referer: nil), - EB.event({127, 0, 0, 1}, @base_ts, referer: nil), - EB.event({127, 0, 0, 1}, @base_ts, referer: " "), - EB.event({127, 0, 0, 1}, @base_ts, referer: " ") + EB.event({127, 0, 0, 1}, @base_ts, referer: nil) ] assert [f] = Referer.detect(events, []) - assert f.subject == "(none)" - assert f.evidence.event_count == 4 - assert length(f.evidence.ips) == 1 - assert "127.0.0.1" in f.evidence.ips + assert f.subject == "127.0.0.1" + assert f.evidence.event_count == 2 + assert f.evidence.matched_referers == [] end end @@ -84,10 +94,10 @@ defmodule Clickguard.Detector.RefererTest do ] assert [f] = Referer.detect(events, spam_domains: @custom_spam_domains) - assert f.subject == "semalt.com" + assert f.subject == "127.0.0.1" assert f.evidence.event_count == 4 - assert length(f.evidence.ips) == 1 - assert "127.0.0.1" in f.evidence.ips + assert length(f.evidence.matched_referers) == 1 + assert f.evidence.matched_referers == ["semalt.com"] end test "finding shape" do @@ -96,11 +106,9 @@ defmodule Clickguard.Detector.RefererTest do assert [f] = Referer.detect([event], spam_domains: @custom_spam_domains) assert f.rule == :spam_referer assert f.severity == :low - assert f.subject == "semalt.com" + assert f.subject == "127.0.0.1" assert f.evidence.event_count == 1 - assert f.evidence.ips == ["127.0.0.1"] - assert f.evidence.ips_truncated == false - assert f.evidence.matched_value == "semalt.com" + assert f.evidence.matched_referers == ["semalt.com"] assert f.sample_events == [event] end @@ -111,42 +119,21 @@ defmodule Clickguard.Detector.RefererTest do EB.event({127, 0, 0, 1}, @base_ts, referer: "GARBAGE") ] - assert Referer.detect(events, []) == [] + assert Referer.detect(events, spam_domains: @custom_spam_domains) == [] end - test "two different spam domains produce two findings" do + test "two different spam domains produce one findings" do events = [ EB.event({127, 0, 0, 1}, @base_ts, referer: "http://semalt.com"), EB.event({127, 0, 0, 1}, @base_ts, referer: "https://badtestdomain.com") ] - assert [f1, f2] = Referer.detect(events, spam_domains: @custom_spam_domains) - assert "semalt.com" in [f1.subject, f2.subject] - assert "badtestdomain.com" in [f1.subject, f2.subject] - end - - test "ips is truncated" do - events = - for n <- 1..51, - do: EB.event({127, 0, 0, n}, @base_ts, referer: "http://semalt.com") - - assert [f] = Referer.detect(events, spam_domains: @custom_spam_domains) - assert f.subject == "semalt.com" - assert f.evidence.event_count == 51 - assert length(f.evidence.ips) == 50 - assert f.evidence.ips_truncated == true - end - - test "ips_truncated boundary" do - events = - for n <- 1..50, - do: EB.event({127, 0, 0, n}, @base_ts, referer: "http://semalt.com") - assert [f] = Referer.detect(events, spam_domains: @custom_spam_domains) - assert f.subject == "semalt.com" - assert f.evidence.event_count == 50 - assert length(f.evidence.ips) == 50 - assert f.evidence.ips_truncated == false + assert "semalt.com" in f.evidence.matched_referers + assert "badtestdomain.com" in f.evidence.matched_referers + assert f.rule == :spam_referer + assert f.subject == "127.0.0.1" + assert length(f.evidence.matched_referers) == 2 end test "use default domain list" do @@ -158,27 +145,10 @@ defmodule Clickguard.Detector.RefererTest do ] assert [f] = Referer.detect(events, []) - assert f.subject == "brandedleadgeneration.com" + assert f.subject == "127.0.0.1" assert f.evidence.event_count == 4 - assert length(f.evidence.ips) == 1 - assert "127.0.0.1" in f.evidence.ips - end - end - - describe "detect/2 both rules in one batch" do - test "one :empty_referer event and one :spam_referer event produce exactly two findings" do - events = [ - EB.event({127, 0, 0, 1}, @base_ts, referer: "http://semalt.com"), - EB.event({127, 0, 0, 2}, @base_ts, referer: nil) - ] - - assert [f1, f2] = Referer.detect(events, spam_domains: @custom_spam_domains) - assert f1.subject in ["semalt.com", "(none)"] - assert f2.subject in ["semalt.com", "(none)"] - assert f1.rule in [:empty_referer, :spam_referer] - assert f2.rule in [:empty_referer, :spam_referer] - assert f1.rule != f2.rule - assert f1.subject != f2.subject + assert length(f.evidence.matched_referers) == 1 + assert f.evidence.matched_referers == ["brandedleadgeneration.com"] end end end diff --git a/test/clickguard/pipeline_golden_test.exs b/test/clickguard/pipeline_golden_test.exs new file mode 100644 index 0000000..1b2c8a1 --- /dev/null +++ b/test/clickguard/pipeline_golden_test.exs @@ -0,0 +1,68 @@ +defmodule Clickguard.PipelineGoldenTest do + use ExUnit.Case, async: true + + @moduledoc """ + Golden test. The fixture generator is :rand-seeded, so the full pipeline must + yield exactly this set of findings. Guards against silent fixture/detector drift + (the kind that let three keyword-mismatch bugs through unnoticed). + + Compared as a SORTED SET, not a list: finding order is unspecified + (Task.async_stream is ordered: false, and each detector ends in group_by). + An intentional generator/detector change means updating @expected. + """ + + @expected [ + {:high_frequency_ip, "127.0.0.1", 300}, + {:automation_tool, "10.0.0.1", 1}, + {:automation_tool, "10.0.0.2", 1}, + {:automation_tool, "10.0.0.10", 2}, + {:headless_browser, "10.0.0.10", 2}, + {:empty_ua, "10.0.0.10", 1}, + {:spam_referer, "192.168.1.1", 1}, + {:spam_referer, "192.168.1.2", 1}, + {:spam_referer, "192.168.1.10", 1}, + {:empty_referer, "192.168.1.10", 1} + ] + + setup do + path = + Path.join(System.tmp_dir!(), "clickguard_golden_#{System.unique_integer([:positive])}.log") + + {_total, ^path} = + Clickguard.Fixtures.generate(out: path, freqip: true, bad_ua: true, bad_referer: true) + + on_exit(fn -> File.rm(path) end) + {:ok, path: path} + end + + test "pipeline yields exactly the expected findings", %{path: path} do + {:ok, findings} = Clickguard.run(path) + + actual = + findings + |> Enum.map(&{&1.rule, &1.subject, &1.evidence.event_count}) + |> Enum.sort() + + assert actual == Enum.sort(@expected) + end + + test "evidence carries the matched values", %{path: path} do + {:ok, findings} = Clickguard.run(path) + + find = fn rule, subject -> + Enum.find(findings, &(&1.rule == rule and &1.subject == subject)) + end + + assert find.(:spam_referer, "192.168.1.1").evidence.matched_referers == [ + "brandedleadgeneration.com" + ] + + assert find.(:spam_referer, "192.168.1.2").evidence.matched_referers == ["addshoppers.com"] + assert find.(:empty_referer, "192.168.1.10").evidence.matched_referers == [] + + assert Enum.sort(find.(:automation_tool, "10.0.0.10").evidence.matched_uas) == + Enum.sort(["Wget/1.21.4", "Go-http-client/2.0"]) + + assert find.(:empty_ua, "10.0.0.10").evidence.matched_uas == [] + end +end