From ade89ffd75ed1941aa33bac56a065646243c0abe Mon Sep 17 00:00:00 2001 From: Justin Middler Date: Wed, 29 Jul 2026 16:28:43 +1000 Subject: [PATCH] feat(threat-intel): ask Brolga for reputation on extracted indicators Adds Brolga as a ReputationProvider alongside VirusTotal, AbuseIPDB, and GreyNoise. Brolga is the operator's own intelligence store rather than a third party's, so it is asked about every indicator kind Tawny extracts whenever it is configured, not just IPs. Follows the KelpieAlertSink template: typed HttpClient, bearer token from IOptions, absolute-URL validation, per-call timeout from the existing ReputationOptions. The disposition mapping is the part worth reviewing, because a wrong answer here changes whether an alert fires: - unknown -> Unknown, never Clean. Brolga's "unknown" means it has not heard of the indicator. Reading that as clean would suppress an alert that nothing has actually cleared. - Anything unrecognised also falls back to Unknown, so a later Brolga adding a disposition cannot silently start suppressing alerts. - allow_listed gets its own verdict rather than being folded into Clean. Clean is a finding about the indicator; allow-listed is a decision about how it is treated regardless of the finding, and collapsing them would let a feed's opinion override an operator's decision. ReputationVerdict.AllowListed = 5; existing values keep their numbers because cached rows are keyed by them. - A failed lookup is Error, not Unknown. "Brolga did not answer" and "Brolga has not heard of this" are different facts and only one says anything about the indicator. The pack's evidence, entities, and gaps are carried into the cached detail. A verdict an analyst cannot trace to a source is one they cannot act on with confidence. A configured base URL with no token means Brolga is not offered at all: Brolga refuses to serve a reachable address without one, so asking would only produce a 401 and a cached Error. The test handler records every request rather than the last one. GreyNoise needs no API key and always runs for an IPv4 indicator, so a handler keeping only the most recent request would have made these assertions depend on provider order. Co-Authored-By: Claude Opus 5 --- backend/src/Tawny.Api/appsettings.json | 7 + backend/src/Tawny.Domain/Enums.cs | 11 + .../ThreatIntel/ReputationEnricher.cs | 123 ++++++++++ .../Tawny.Api.Tests/BrolgaReputationTests.cs | 221 ++++++++++++++++++ docker/docker-compose.yml | 3 + 5 files changed, 365 insertions(+) create mode 100644 backend/tests/Tawny.Api.Tests/BrolgaReputationTests.cs diff --git a/backend/src/Tawny.Api/appsettings.json b/backend/src/Tawny.Api/appsettings.json index 96b7412..c860e50 100644 --- a/backend/src/Tawny.Api/appsettings.json +++ b/backend/src/Tawny.Api/appsettings.json @@ -40,6 +40,13 @@ "IconEmoji": ":rotating_light:", "TimeoutSeconds": 5 }, + "Reputation": { + "BrolgaBaseUrl": "", + "BrolgaApiToken": "", + "CacheTtlHours": 24, + "TimeoutSeconds": 10, + "EnrichAlertsAutomatically": true + }, "Kelpie": { "Enabled": false, "BaseUrl": "", diff --git a/backend/src/Tawny.Domain/Enums.cs b/backend/src/Tawny.Domain/Enums.cs index 7243164..618132a 100644 --- a/backend/src/Tawny.Domain/Enums.cs +++ b/backend/src/Tawny.Domain/Enums.cs @@ -151,6 +151,7 @@ public enum ReputationProvider VirusTotal = 0, AbuseIpDb = 1, GreyNoise = 2, + Brolga = 3, } public enum ReputationVerdict @@ -160,6 +161,16 @@ public enum ReputationVerdict Suspicious = 2, Malicious = 3, Error = 4, + + /// + /// Deliberately excluded from detection, for example known-good infrastructure. + /// + /// + /// Distinct from : clean is a finding about the indicator, allow-listed is + /// a decision about how it is treated regardless of the finding. Collapsing them would let a + /// feed's opinion override an operator's decision. + /// + AllowListed = 5, } public enum CloudProvider diff --git a/backend/src/Tawny.Infrastructure/ThreatIntel/ReputationEnricher.cs b/backend/src/Tawny.Infrastructure/ThreatIntel/ReputationEnricher.cs index 7384662..f71e263 100644 --- a/backend/src/Tawny.Infrastructure/ThreatIntel/ReputationEnricher.cs +++ b/backend/src/Tawny.Infrastructure/ThreatIntel/ReputationEnricher.cs @@ -1,4 +1,5 @@ using System.Net.Http.Headers; +using System.Net.Http.Json; using System.Text.Json; using Microsoft.EntityFrameworkCore; using Microsoft.Extensions.Logging; @@ -13,6 +14,19 @@ public class ReputationOptions public string? VirusTotalApiKey { get; set; } public string? AbuseIpDbApiKey { get; set; } public string? GreyNoiseApiKey { get; set; } + + /// Origin of a Brolga instance, for example http://brolga:8787. + /// + /// Origin only: the /api/v1 prefix is added when the request is built, so a base URL + /// carrying a path would produce a doubled one. + /// + public string? BrolgaBaseUrl { get; set; } + + /// + /// Bearer token for Brolga. Required whenever is set, because + /// Brolga refuses to serve a reachable address without one. + /// + public string? BrolgaApiToken { get; set; } public int CacheTtlHours { get; set; } = 24; public int TimeoutSeconds { get; set; } = 10; public bool EnrichAlertsAutomatically { get; set; } = true; @@ -111,6 +125,7 @@ await db.ReputationCache.AddAsync(new ReputationCacheEntry ReputationProvider.VirusTotal => await ProbeVirusTotalAsync(kind, value, timeoutCts.Token), ReputationProvider.AbuseIpDb => await ProbeAbuseIpDbAsync(kind, value, timeoutCts.Token), ReputationProvider.GreyNoise => await ProbeGreyNoiseAsync(kind, value, timeoutCts.Token), + ReputationProvider.Brolga => await ProbeBrolgaAsync(kind, value, timeoutCts.Token), _ => null, }; } @@ -249,5 +264,113 @@ private IEnumerable ProvidersForKind(string kind) { yield return ReputationProvider.GreyNoise; } + // Brolga answers about every indicator kind Tawny extracts, and it is the operator's own + // intelligence rather than a third party's, so it is asked whenever it is configured. + if (!string.IsNullOrWhiteSpace(_opts.BrolgaBaseUrl) + && !string.IsNullOrWhiteSpace(_opts.BrolgaApiToken) + && BrolgaSubjectKind(kind) is not null) + { + yield return ReputationProvider.Brolga; + } + } + + /// + /// Tawny's indicator kind as Brolga spells it. + /// + /// + /// Returns null for a kind Brolga does not accept, so the provider is not offered at + /// all rather than asked and refused. Brolga normalises the value itself — case, whitespace, + /// IPv6 abbreviation — so only the kind needs translating. + /// + private static string? BrolgaSubjectKind(string kind) => kind switch + { + "sha256" => "sha256", + "sha1" => "sha1", + "md5" => "md5", + "ipv4" => "ipv4", + "ipv6" => "ipv6", + "domain" => "domain", + "url" => "url", + _ => null, + }; + + /// + /// Asks Brolga what is known about the indicator. + /// + /// + /// Brolga's unknown maps to and never to + /// . It means Brolga has not heard of the indicator, and + /// reading that as "clean" would suppress an alert that nothing has actually cleared. + /// + private async Task ProbeBrolgaAsync(string kind, string value, CancellationToken ct) + { + if (string.IsNullOrWhiteSpace(_opts.BrolgaBaseUrl)) return null; + if (string.IsNullOrWhiteSpace(_opts.BrolgaApiToken)) return null; + + var subjectKind = BrolgaSubjectKind(kind); + if (subjectKind is null) return null; + + if (!Uri.TryCreate(_opts.BrolgaBaseUrl.TrimEnd('/') + "/api/v1/context", UriKind.Absolute, out var endpoint) + || (endpoint.Scheme != Uri.UriSchemeHttp && endpoint.Scheme != Uri.UriSchemeHttps)) + { + log.LogWarning("Brolga base URL is not a usable absolute http(s) URL; skipping lookup"); + return null; + } + + using var request = new HttpRequestMessage(HttpMethod.Post, endpoint); + request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", _opts.BrolgaApiToken.Trim()); + request.Content = JsonContent.Create( + new + { + subject = new { kind = subjectKind, value }, + purpose = "incident_triage", + }, + options: JsonOptions); + + using var response = await http.SendAsync(request, ct); + if (!response.IsSuccessStatusCode) + { + return new ReputationLookup( + ReputationProvider.Brolga, + ReputationVerdict.Error, + null, + new { http_status = (int)response.StatusCode }); + } + + var body = await response.Content.ReadAsStringAsync(ct); + using var doc = JsonDocument.Parse(body); + var root = doc.RootElement; + + var disposition = root.TryGetProperty("disposition", out var d) ? d.GetString() : null; + var verdict = disposition switch + { + "malicious" => ReputationVerdict.Malicious, + "suspicious" => ReputationVerdict.Suspicious, + "benign" => ReputationVerdict.Clean, + "allow_listed" => ReputationVerdict.AllowListed, + // Covers "unknown" and any disposition a later Brolga adds. An unrecognised + // disposition must not be guessed at: treating it as clean would suppress an alert. + _ => ReputationVerdict.Unknown, + }; + + // Carried through so an analyst can see where the verdict came from. A verdict with + // nothing to cite is one nobody can act on with confidence. + var evidence = root.TryGetProperty("evidence", out var e) ? e.GetRawText() : "[]"; + var gaps = root.TryGetProperty("gaps", out var g) ? g.GetRawText() : "[]"; + var entities = root.TryGetProperty("entities", out var n) ? n.GetRawText() : "[]"; + + return new ReputationLookup( + ReputationProvider.Brolga, + verdict, + null, + new + { + disposition, + observable_id = root.TryGetProperty("observable_id", out var o) ? o.GetString() : null, + schema_version = root.TryGetProperty("schema_version", out var s) ? s.GetString() : null, + entities = JsonDocument.Parse(entities).RootElement.Clone(), + evidence = JsonDocument.Parse(evidence).RootElement.Clone(), + gaps = JsonDocument.Parse(gaps).RootElement.Clone(), + }); } } diff --git a/backend/tests/Tawny.Api.Tests/BrolgaReputationTests.cs b/backend/tests/Tawny.Api.Tests/BrolgaReputationTests.cs new file mode 100644 index 0000000..c865004 --- /dev/null +++ b/backend/tests/Tawny.Api.Tests/BrolgaReputationTests.cs @@ -0,0 +1,221 @@ +using System.Net; +using System.Text; +using FluentAssertions; +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using Tawny.Domain; +using Tawny.Infrastructure; +using Tawny.Infrastructure.ThreatIntel; +using Xunit; + +namespace Tawny.Api.Tests; + +/// +/// Brolga is the operator's own intelligence store, so its verdict carries further than a third +/// party's. These cover the mapping from its disposition onto Tawny's verdict, which is where a +/// wrong answer would quietly change whether an alert fires. +/// +public class BrolgaReputationTests +{ + private static ReputationEnricher Enricher(RecordingHandler handler, string? baseUrl = "http://brolga.test:8787") + { + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase($"brolga-{Guid.NewGuid()}") + .Options; + + return new ReputationEnricher( + new TawnyDbContext(options), + new HttpClient(handler), + Options.Create(new ReputationOptions + { + BrolgaBaseUrl = baseUrl, + BrolgaApiToken = "0123456789abcdef0123456789abcdef", + }), + TimeProvider.System, + NullLogger.Instance); + } + + private static string Pack(string disposition) => $$""" + { + "schema_version": "brolga.context_pack/1.0", + "subject": { "kind": "ipv4_address", "value": "203.0.113.42" }, + "observable_id": "observable:7168327b", + "disposition": "{{disposition}}", + "entities": [{ "id": "entity:9c8e", "kind": "report", "name": "C2 infrastructure" }], + "claims": [], + "relationships": [], + "evidence": [{ "source_object_id": "source:12fc" }], + "gaps": [], + "exclusions": [] + } + """; + + [Theory] + [InlineData("malicious", ReputationVerdict.Malicious)] + [InlineData("suspicious", ReputationVerdict.Suspicious)] + [InlineData("benign", ReputationVerdict.Clean)] + [InlineData("allow_listed", ReputationVerdict.AllowListed)] + public async Task Disposition_MapsOntoTheMatchingVerdict(string disposition, ReputationVerdict expected) + { + var handler = new RecordingHandler(Pack(disposition)); + var results = await Enricher(handler).LookupAsync(Guid.NewGuid(), "ipv4", "203.0.113.42", CancellationToken.None); + + results.Should().ContainSingle(r => r.Provider == ReputationProvider.Brolga) + .Which.Verdict.Should().Be(expected); + } + + /// + /// The mapping that matters most. Brolga's "unknown" means it has not heard of the indicator. + /// Reading that as Clean would suppress an alert that nothing has actually cleared. + /// + [Fact] + public async Task Unknown_IsNeverReadAsClean() + { + var handler = new RecordingHandler(Pack("unknown")); + var results = await Enricher(handler).LookupAsync(Guid.NewGuid(), "ipv4", "203.0.113.42", CancellationToken.None); + + var brolga = results.Single(r => r.Provider == ReputationProvider.Brolga); + brolga.Verdict.Should().Be(ReputationVerdict.Unknown); + brolga.Verdict.Should().NotBe(ReputationVerdict.Clean); + } + + /// + /// A disposition from a later Brolga must not be guessed at. Defaulting to Clean would mean a + /// Brolga upgrade could silently start suppressing alerts. + /// + [Fact] + public async Task AnUnrecognisedDisposition_FallsBackToUnknownRatherThanClean() + { + var handler = new RecordingHandler(Pack("something_brolga_added_later")); + var results = await Enricher(handler).LookupAsync(Guid.NewGuid(), "ipv4", "203.0.113.42", CancellationToken.None); + + results.Single(r => r.Provider == ReputationProvider.Brolga) + .Verdict.Should().Be(ReputationVerdict.Unknown); + } + + [Fact] + public async Task TheRequestGoesToTheVersionedRouteWithABearerToken() + { + var handler = new RecordingHandler(Pack("malicious")); + await Enricher(handler).LookupAsync(Guid.NewGuid(), "ipv4", "203.0.113.42", CancellationToken.None); + + var request = handler.Brolga.Should().NotBeNull().And.Subject.As(); + request.Uri.Should().Be("http://brolga.test:8787/api/v1/context"); + request.Method.Should().Be(HttpMethod.Post); + request.Authorization.Should().Be("Bearer 0123456789abcdef0123456789abcdef"); + request.Body.Should().Contain("\"kind\":\"ipv4\"").And.Contain("203.0.113.42"); + } + + /// + /// A base URL that already carries a path would otherwise produce `/api/v1/api/v1/context`. + /// + [Fact] + public async Task ATrailingSlashOnTheBaseUrlDoesNotDoubleTheSeparator() + { + var handler = new RecordingHandler(Pack("malicious")); + await Enricher(handler, "http://brolga.test:8787/") + .LookupAsync(Guid.NewGuid(), "ipv4", "203.0.113.42", CancellationToken.None); + + handler.Brolga.Should().NotBeNull(); + handler.Brolga!.Uri.Should().Be("http://brolga.test:8787/api/v1/context"); + } + + /// + /// The evidence has to survive into the cached detail, or an analyst reading the verdict has + /// nothing to cite for it. + /// + [Fact] + public async Task TheEvidenceAndEntitiesSurviveIntoTheDetail() + { + var handler = new RecordingHandler(Pack("malicious")); + var results = await Enricher(handler).LookupAsync(Guid.NewGuid(), "ipv4", "203.0.113.42", CancellationToken.None); + + var detail = System.Text.Json.JsonSerializer.Serialize( + results.Single(r => r.Provider == ReputationProvider.Brolga).Detail); + + detail.Should().Contain("source:12fc"); + detail.Should().Contain("C2 infrastructure"); + } + + /// + /// Brolga refuses to serve a reachable address without a token, so a configured base URL with + /// no token is a misconfiguration. It must not be asked at all rather than asked and refused. + /// + [Fact] + public async Task WithoutATokenBrolgaIsNotAsked() + { + var handler = new RecordingHandler(Pack("malicious")); + var options = new DbContextOptionsBuilder() + .UseInMemoryDatabase($"brolga-{Guid.NewGuid()}") + .Options; + + var enricher = new ReputationEnricher( + new TawnyDbContext(options), + new HttpClient(handler), + Options.Create(new ReputationOptions + { + BrolgaBaseUrl = "http://brolga.test:8787", + BrolgaApiToken = null, + }), + TimeProvider.System, + NullLogger.Instance); + + var results = await enricher.LookupAsync(Guid.NewGuid(), "ipv4", "203.0.113.42", CancellationToken.None); + + results.Should().NotContain(r => r.Provider == ReputationProvider.Brolga); + // Other providers still run for an IPv4 indicator, so the assertion is that *Brolga* was + // not reached — not that nothing was. + handler.Brolga.Should().BeNull(); + } + + /// + /// A failed lookup is Error, not Unknown. "Brolga did not answer" and "Brolga has not heard of + /// this" are different facts, and only one of them says anything about the indicator. + /// + [Fact] + public async Task AFailedLookupIsAnErrorRatherThanAnAbsenceOfKnowledge() + { + var handler = new RecordingHandler("upstream is unwell", HttpStatusCode.ServiceUnavailable); + var results = await Enricher(handler).LookupAsync(Guid.NewGuid(), "ipv4", "203.0.113.42", CancellationToken.None); + + results.Single(r => r.Provider == ReputationProvider.Brolga) + .Verdict.Should().Be(ReputationVerdict.Error); + } + + private sealed record Recorded(string Uri, HttpMethod Method, string? Authorization, string? Body); + + /// + /// Records every request, not just the last one. GreyNoise needs no API key and so is always + /// asked about an IPv4 indicator; a handler that kept only the most recent request would make + /// these assertions depend on the order providers happen to run in. + /// + private sealed class RecordingHandler(string body, HttpStatusCode status = HttpStatusCode.OK) + : HttpMessageHandler + { + private readonly List _requests = []; + + public IReadOnlyList Requests => _requests; + + public Recorded? Brolga => + _requests.SingleOrDefault(request => request.Uri.Contains("/api/v1/context")); + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + _requests.Add(new Recorded( + request.RequestUri?.ToString() ?? string.Empty, + request.Method, + request.Headers.Authorization?.ToString(), + request.Content is null + ? null + : await request.Content.ReadAsStringAsync(cancellationToken))); + + return new HttpResponseMessage(status) + { + Content = new StringContent(body, Encoding.UTF8, "application/json"), + }; + } + } +} diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index c81d8fc..88314c1 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -50,6 +50,9 @@ services: Tawny__Slack__Username: "${TAWNY_SLACK_USERNAME:-Tawny}" Tawny__Slack__IconEmoji: "${TAWNY_SLACK_ICON_EMOJI:-:rotating_light:}" Tawny__Slack__TimeoutSeconds: "${TAWNY_SLACK_TIMEOUT_SECONDS:-5}" + Tawny__Reputation__BrolgaBaseUrl: "${TAWNY_BROLGA_BASE_URL:-}" + Tawny__Reputation__BrolgaApiToken: "${TAWNY_BROLGA_API_TOKEN:-}" + Tawny__Reputation__CacheTtlHours: "${TAWNY_REPUTATION_CACHE_TTL_HOURS:-24}" Tawny__Kelpie__Enabled: "${TAWNY_KELPIE_ENABLED:-false}" Tawny__Kelpie__BaseUrl: "${TAWNY_KELPIE_BASE_URL:-}" Tawny__Kelpie__ApiToken: "${TAWNY_KELPIE_API_TOKEN:-}"