From ff21e2d5924dec285b7401d85c72145f9bce026e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:00:35 +0200 Subject: [PATCH 1/2] fix(recycle): prune expired Admin/_Recycle requests with their reports (#6813) A declared retention (ActivationRecycle:Retention:{Enabled,MaxAge,MaxDeletionsPerRun}, default 30 days / 50 per run) applied by the writer: ActivationRecycle.Request prunes expired requests right after it writes a new one, as System, through DeleteNode (which cascades to the per-process reports). No timer, same owner-prunes-on-write shape as CompileActivityRetention. Fail-closed selection; the request just written is never pruned. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Architecture/StaleStateUntilRecycle.md | 28 +++- .../Configuration/ActivationRecycle.cs | 70 +++++++- .../ActivationRecycleRetention.cs | 145 +++++++++++++++++ .../ActivationRecycleRetentionTest.cs | 154 ++++++++++++++++++ 4 files changed, 395 insertions(+), 2 deletions(-) create mode 100644 src/MeshWeaver.Graph/Configuration/ActivationRecycleRetention.cs create mode 100644 test/MeshWeaver.Graph.Test/ActivationRecycleRetentionTest.cs diff --git a/src/MeshWeaver.Documentation/Data/Architecture/StaleStateUntilRecycle.md b/src/MeshWeaver.Documentation/Data/Architecture/StaleStateUntilRecycle.md index 14d011b9f7..1134f84d97 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/StaleStateUntilRecycle.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/StaleStateUntilRecycle.md @@ -220,6 +220,29 @@ bound NodeType is T"* (optionally narrowed to some addresses, or to the activati - **Only System's requests are acted on.** Each process re-validates what it reads and refuses a request node not created by System — a platform admin, who holds `Admin`, cannot bypass the checks by writing `Admin/_Recycle` directly. +- **The audit is kept for a declared window, then pruned (#6813).** Every NodeType definition recycle + issues a request, so without a retention the rows under `Admin/_Recycle` grow with every recycle, + module landing and deploy. `ActivationRecycleRetention` is the policy, read from configuration: + + | Key | Default | Meaning | + |---|---|---| + | `ActivationRecycle:Retention:Enabled` | `true` | `false` turns the prune off (no query is issued) | + | `ActivationRecycle:Retention:MaxAge` | `30.00:00:00` | a request untouched for this long is removed with its reports; a configured value below one day is clamped to one day | + | `ActivationRecycle:Retention:MaxDeletionsPerRun` | `50` | the most requests one prune removes, so a backlog drains over several requests | + + Thirty days is long enough that the audit of a recent deploy or incident is still there when it + is read, whether for a post-incident review or to check which silos recycled after last week's + release, and it reaches back a full monthly release cycle. It is short enough that the rows stay + bounded by one month of recycle traffic rather than the portal's whole history. **The writer + prunes, and no timer does.** Right after `ActivationRecycle.Request` writes a new request it + removes the expired ones, as System, through `DeleteNode`. The delete cascades, so each + `{request}/{process}` report leaves with its request. This is the same owner-prunes-on-write shape + the compile activities use (`CompileActivityRetention`, in + [NodeType Compilation](../NodeTypeCompilation)). The rows grow only through that write, so pruning + on it bounds them. The request just written is never pruned. The decision is fail-closed: a row + that is undated, not a request, or not directly under `Admin/_Recycle` is kept. A failed listing + or delete costs that prune only. It is logged, and the next request retries; the caller's ticket + never depends on it. The request is a platform write. On the control instance it is issued by the build queue as System: the `recycle` job kind (MeshWeaver.Plugins, `Hosting/Queue`) runs it, either when a NodeType of the @@ -230,7 +253,10 @@ not the admin, holds the rights. The direct `recycle` verb above is unchanged an Pinned by `ActivationRecycleTest` (monolith: two live activations of T and one of U, exactly the two disposed, re-activation on a fresh hub, redelivery, refusals) and `TwoSiloActivationRecycleTest` (both silos' activations, -one report per silo whose counts match what each hosted). +one report per silo whose counts match what each hosted). The retention is pinned by +`ActivationRecycleRetentionTest`: an expired request goes with its report when the next one is +written, a request inside the window survives with its report, and the decision is oldest first, +capped and fail-closed. ### Three rules for posting one, each paid for diff --git a/src/MeshWeaver.Graph/Configuration/ActivationRecycle.cs b/src/MeshWeaver.Graph/Configuration/ActivationRecycle.cs index 3715d35bac..dbd57cc445 100644 --- a/src/MeshWeaver.Graph/Configuration/ActivationRecycle.cs +++ b/src/MeshWeaver.Graph/Configuration/ActivationRecycle.cs @@ -7,6 +7,7 @@ using MeshWeaver.Mesh.Security; using MeshWeaver.Mesh.Services; using MeshWeaver.Messaging; +using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -129,6 +130,10 @@ public static TBuilder AddActivationRecycle(this TBuilder builder) whe HubConfiguration = config => config .AddMeshDataSource(source => source.WithContentType()), }); + // Mesh-scoped singleton, read once from the host's configuration: how long a request and its + // reports are kept (Doc/Architecture/StaleStateUntilRecycle → "Recycling a type"). + builder.ConfigureServices(services => services + .AddSingleton(sp => ActivationRecycleRetention.FromConfiguration(sp.GetService()))); builder.ConfigureHub(c => c .WithType(nameof(ActivationRecycleRequest)) .WithType(nameof(ActivationRecycleReport)) @@ -228,14 +233,77 @@ IObservable TypeExists(string type) => Content = request with { RequestedAt = request.RequestedAt == default ? DateTimeOffset.UtcNow : request.RequestedAt }, }; // The id is the dedupe key: an existing request is the answer, never a second write. + // A NEW request is the moment the rows grow, so it is also when the expired ones go + // (ActivationRecycleRetention): the writer prunes, no timer does. return Exists().SelectMany(there => there ? Observable.Return(new ActivationRecycleTicket(path, null)) : access.RunAsSystem(() => mesh.CreateNode(node).Take(1)) - .Select(_ => new ActivationRecycleTicket(path, null))); + .SelectMany(_ => Prune(hub, hub.ServiceProvider.GetRequiredService(), + DateTimeOffset.UtcNow, path) + .Select(_ => new ActivationRecycleTicket(path, null)))); }) .Take(1); } + /// + /// Removes the requests calls expired at , + /// each with its per-process reports, never . Cold; emits once, the + /// number of requests removed. + /// + /// The listing is a QUERY, which is a valid use here: it lists children, and a stale answer + /// is harmless — a row the index has not seen yet is a NEWER row and is kept, a row already gone + /// is an idempotent delete. Each removal goes through , + /// which cascades, so the {request}/{process} reports leave with their request. It runs as + /// System, as the request's own create did: infrastructure acting on records it wrote. + /// + /// A failed removal is per ROW, and a failed listing costs this prune only: each is logged, + /// the rows stay, and the next request's prune tries again. The request that triggered the prune + /// is already written and acted on, so its caller's ticket never depends on the prune. + /// + /// Any surviving hub of the mesh. + /// The policy to prune to. + /// The cutoff instant. + /// The request just written; always kept. + public static IObservable Prune(IMessageHub hub, ActivationRecycleRetention retention, DateTimeOffset now, string? keepPath = null) + { + if (!retention.Enabled) + return Observable.Return(0); + var mesh = hub.ServiceProvider.GetRequiredService(); + var access = hub.ServiceProvider.GetRequiredService(); + var logger = hub.ServiceProvider.GetService()?.CreateLogger(typeof(ActivationRecycle)); + return access.RunAsSystem(() => mesh.Query(MeshQueryRequest.FromQuery(retention.WindowQuery)) + .Where(change => change.ChangeType == QueryChangeType.Initial) + .Take(1) + .Timeout(ReadBudget)) + .Select(change => retention.SelectForPruning(change.Items, now, keepPath)) + .SelectMany(paths => + { + if (paths.IsEmpty) + return Observable.Return(0); + // Information on purpose: the platform is removing audit rows, and an operator must be + // able to see that. One line per prune, and only when something is removed. + logger?.LogInformation( + "[ActivationRecycle] retention: removing {Count} recycle request(s) older than {MaxAge}, with their reports", + paths.Count, retention.MaxAge); + return paths + .Select(p => access.RunAsSystem(() => mesh.DeleteNode(p)) + .Take(1) + .Select(removed => removed ? 1 : 0) + .Catch((Exception ex) => + { + logger?.LogWarning(ex, "[ActivationRecycle] retention: {Path} was not removed; the next request's prune retries it", p); + return Observable.Return(0); + })) + .Concat() + .Sum(); + }) + .Catch((Exception ex) => + { + logger?.LogWarning(ex, "[ActivationRecycle] retention: the request window could not be read; nothing was pruned, the next request retries"); + return Observable.Return(0); + }); + } + /// The reports written so far for a request, read as System. Cold; one snapshot. public static IObservable> Reports(IMessageHub hub, string requestPath) { diff --git a/src/MeshWeaver.Graph/Configuration/ActivationRecycleRetention.cs b/src/MeshWeaver.Graph/Configuration/ActivationRecycleRetention.cs new file mode 100644 index 0000000000..9b644bc617 --- /dev/null +++ b/src/MeshWeaver.Graph/Configuration/ActivationRecycleRetention.cs @@ -0,0 +1,145 @@ +using System.Collections.Immutable; +using MeshWeaver.Mesh; +using Microsoft.Extensions.Configuration; + +namespace MeshWeaver.Graph.Configuration; + +/// +/// How long an at Admin/_Recycle/{id} — and with it +/// every per-process under it — is kept, and how many one +/// prune may remove. The whole policy: the selection makes and the +/// bounds it works within, with no mesh, no hub and no clock of its own. +/// +/// Why requests get a retention. A request plus its reports is the audit of a recycle: +/// who asked, when, why, which types, and what each process disposed. Nothing ever removed one, and +/// once every NodeType definition recycle issues a request, the rows grow with every recycle, module +/// landing and deploy (Systemorph/MeshWeaver#6813). +/// +/// Who prunes. The writer does, at the moment it writes a new request: +/// prunes right after its create, the same owner-prunes-on-write +/// shape as CompileActivityRetention. No timer and no sweeper: the rows grow only through +/// that write, so pruning on it bounds them, and a mesh that stops recycling stops growing. Each +/// prune is capped at , so a backlog drains over successive +/// requests. +/// +/// Declared, not derived: every bound comes from configuration +/// () with the shipped values as the fallback. +/// +public sealed record ActivationRecycleRetention +{ + /// Config key turning the prune off entirely. Default true. + public const string EnabledConfigKey = "ActivationRecycle:Retention:Enabled"; + + /// Config key overriding (a string, e.g. 30.00:00:00). + public const string MaxAgeConfigKey = "ActivationRecycle:Retention:MaxAge"; + + /// Config key overriding . + public const string MaxDeletionsPerRunConfigKey = "ActivationRecycle:Retention:MaxDeletionsPerRun"; + + /// + /// The floor clamps a CONFIGURED to. A + /// window of zero in a chart is a typo, not a request to delete the audit of the recycle that + /// just ran. The clamp lives at the configuration edge only; a directly constructed policy is + /// applied exactly as written. + /// + public static readonly TimeSpan MinimumMaxAge = TimeSpan.FromDays(1); + + /// The shipped policy: 30 days, 50 removals per prune, enabled. + public static readonly ActivationRecycleRetention Default = new(); + + /// Whether a prune removes anything at all. Off means no query is issued. + public bool Enabled { get; init; } = true; + + /// + /// A request (and its reports) is removed once its node has not been written for this long. + /// + /// Thirty days: long enough that the audit of a recent deploy or incident is still there + /// when someone reads it — a post-incident review, or the question "which silos actually + /// recycled after last week's release" — and a full monthly release cycle back. Short enough + /// that the rows stay bounded by one month of recycle traffic instead of the portal's whole + /// history. + /// + public TimeSpan MaxAge { get; init; } = TimeSpan.FromDays(30); + + /// + /// The cap on how many requests ONE prune removes (each with its reports). A backlog drains by + /// this many per new request, instead of one request turning into a mass delete. + /// + public int MaxDeletionsPerRun { get; init; } = 50; + + /// + /// The query that reads the oldest window of requests. It bounds the work; the decision is + /// 's, so a backend that ignored the sort would only drain slower, + /// never delete a row the policy has not called expired. One more than the cap, so the request + /// the caller just wrote never crowds out a row that may go. + /// + public string WindowQuery + => $"path:{ActivationRecycleRequest.Namespace} scope:children nodeType:{ActivationRecycleRequest.NodeType} " + + $"sort:LastModified-asc limit:{MaxDeletionsPerRun + 1}"; + + /// + /// Whether is a request this policy removes at . + /// The clock is — the quantity the window query orders by. + /// + /// 🚨 Fail closed: not a request, not directly under Admin/_Recycle, or undated ⇒ + /// kept. This decides what is deleted, so every uncertainty resolves to keeping the row. + /// + /// The candidate row. + /// The current time, passed in so the rule is a pure function. + public bool IsExpired(MeshNode? node, DateTimeOffset now) + { + if (!Enabled || node is null) + return false; + if (!string.Equals(node.NodeType, ActivationRecycleRequest.NodeType, StringComparison.OrdinalIgnoreCase)) + return false; + if (!string.Equals(node.Namespace?.Trim('/'), ActivationRecycleRequest.Namespace, StringComparison.OrdinalIgnoreCase)) + return false; + if (node.LastModified == default) + return false; + return node.LastModified <= now - MaxAge; + } + + /// + /// The request paths among this policy removes at + /// : expired ones, oldest first, never , at most + /// . + /// + /// Candidate rows, in any order. + /// The current time. + /// The request the caller just wrote; never selected. + public ImmutableList SelectForPruning(IEnumerable requests, DateTimeOffset now, string? keepPath = null) + => !Enabled + ? ImmutableList.Empty + : requests + .Where(n => IsExpired(n, now)) + .Where(n => !string.Equals(n.Path, keepPath, StringComparison.OrdinalIgnoreCase)) + .OrderBy(n => n.LastModified) + .ThenBy(n => n.Id, StringComparer.Ordinal) + .Select(n => n.Path) + .Distinct(StringComparer.OrdinalIgnoreCase) + .Take(MaxDeletionsPerRun) + .ToImmutableList(); + + /// + /// Reads the policy from configuration. Every key falls back to its default when absent or + /// malformed; a configured window below is clamped to it, so a typo + /// never widens what gets deleted. + /// + /// The host's configuration, or null on a host that has none. + public static ActivationRecycleRetention FromConfiguration(IConfiguration? configuration) + { + var retention = Default; + if (configuration is null) + return retention; + + if (bool.TryParse(configuration[EnabledConfigKey], out var enabled)) + retention = retention with { Enabled = enabled }; + if (TimeSpan.TryParse(configuration[MaxAgeConfigKey], System.Globalization.CultureInfo.InvariantCulture, out var maxAge) + && maxAge > TimeSpan.Zero) + retention = retention with { MaxAge = maxAge < MinimumMaxAge ? MinimumMaxAge : maxAge }; + if (int.TryParse(configuration[MaxDeletionsPerRunConfigKey], out var max) && max >= 1) + retention = retention with { MaxDeletionsPerRun = max }; + + return retention; + } +} diff --git a/test/MeshWeaver.Graph.Test/ActivationRecycleRetentionTest.cs b/test/MeshWeaver.Graph.Test/ActivationRecycleRetentionTest.cs new file mode 100644 index 0000000000..adfc20d12a --- /dev/null +++ b/test/MeshWeaver.Graph.Test/ActivationRecycleRetentionTest.cs @@ -0,0 +1,154 @@ +using System; +using System.Collections.Immutable; +using System.Linq; +using System.Reactive.Linq; +using System.Threading.Tasks; +using MeshWeaver.Fixture; +using MeshWeaver.Graph.Configuration; +using MeshWeaver.Hosting.Monolith.TestBase; +using MeshWeaver.Mesh; +using MeshWeaver.Mesh.Services; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Xunit; + +namespace MeshWeaver.Graph.Test; + +/// +/// Retention of Admin/_Recycle (#6813): a NEW request prunes the requests older than the +/// declared window together with their per-process reports, and leaves the ones inside it — audit +/// and all — readable. +/// +public class ActivationRecycleRetentionTest(ITestOutputHelper output) : MonolithMeshTestBase(output) +{ + private const string TypeR = "RetentionTypeR"; + + /// + /// The mesh's registered policy: a window of one millisecond, constructed directly (the + /// configuration clamp does not apply), so a request whose report has landed is already + /// expired by the time the next request is written — no clock to fake, no sleep. + /// + private static readonly ActivationRecycleRetention Tight = new() { MaxAge = TimeSpan.FromMilliseconds(1) }; + + /// + protected override MeshBuilder ConfigureMesh(MeshBuilder builder) + => base.ConfigureMesh(builder) + .AddMeshNodes( + new MeshNode(TypeR) { Name = "Retention Type R" }, + new MeshNode("RetentionR1", TestPartition) { Name = "R one", NodeType = TypeR }) + .ConfigureServices(services => services.AddSingleton(Tight)); + + private IObservable> RequestPaths() => + MeshQuery.Query(MeshQueryRequest.FromQuery( + $"path:{ActivationRecycleRequest.Namespace} scope:children nodeType:{ActivationRecycleRequest.NodeType}").AsSystem()) + .Take(1) + .Select(c => c.Items.Select(n => n.Path).ToImmutableList()); + + private Task Issue(string reason) => + ActivationRecycle.Request(RequestHub, new ActivationRecycleRequest + { + NodeTypes = ImmutableList.Create(TypeR), + Reason = reason, + RequestedBy = nameof(ActivationRecycleRetentionTest), + }).FirstAsync().Await(TestContext.Current.CancellationToken); + + /// The request's report (this monolith's one process) has been written. + private Task ReportWritten(string requestPath) => + Observable.Interval(TimeSpan.FromMilliseconds(50)).StartWith(0L) + .SelectMany(_ => ActivationRecycle.Reports(RequestHub, requestPath)) + .Where(reports => !reports.IsEmpty) + .Take(1) + .Should().Within(TestTimeouts.Convergence) + .Emit($"the process reports on {requestPath}", cancellationToken: TestContext.Current.CancellationToken); + + /// + /// A request older than the window is pruned with its report when the next request is written; + /// the new request itself is kept. Negative control: with the prune disabled the old request + /// and its report stay, and this test fails on the first assertion below. + /// + [Fact(Timeout = 120_000)] + public async Task ANewRequest_PrunesAnExpiredRequestWithItsReports_AndKeepsItself() + { + var ct = TestContext.Current.CancellationToken; + Mesh.ServiceProvider.GetRequiredService().Should().BeSameAs(Tight, + "the precondition: the test's declared policy is the one Request applies"); + + var old = await Issue("test: the old request"); + old.Accepted.Should().BeTrue(); + await ReportWritten(old.Path!); + + var fresh = await Issue("test: the new request"); + fresh.Accepted.Should().BeTrue(); + + await Observable.Interval(TimeSpan.FromMilliseconds(50)).StartWith(0L) + .SelectMany(_ => RequestPaths()) + .Where(paths => !paths.Contains(old.Path!, StringComparer.OrdinalIgnoreCase)) + .Take(1) + .Should().Within(TestTimeouts.Convergence) + .Emit("the old request is past the window and the new request's prune removes it", cancellationToken: ct); + (await ActivationRecycle.Reports(RequestHub, old.Path!).FirstAsync().Await(ct)) + .Should().BeEmpty("the per-process reports leave with their request"); + (await RequestPaths().FirstAsync().Await(ct)) + .Should().Contain(fresh.Path!, "the request that triggered the prune is never pruned by it"); + } + + /// A request inside the window survives a prune, its report still readable. + [Fact(Timeout = 120_000)] + public async Task ARequestInsideTheWindow_SurvivesThePrune_WithItsReport() + { + var ct = TestContext.Current.CancellationToken; + var ticket = await Issue("test: a recent request"); + await ReportWritten(ticket.Path!); + + var removed = await ActivationRecycle.Prune(RequestHub, ActivationRecycleRetention.Default, DateTimeOffset.UtcNow) + .FirstAsync().Await(ct); + + removed.Should().Be(0, "nothing here is thirty days old"); + (await RequestPaths().FirstAsync().Await(ct)).Should().Contain(ticket.Path!); + (await ActivationRecycle.Reports(RequestHub, ticket.Path!).FirstAsync().Await(ct)) + .Should().ContainSingle("the audit stays readable inside the window"); + } + + /// The decision is pure, oldest first, capped, and fails closed on anything it cannot age. + [Fact] + public void SelectForPruning_TakesExpiredRequestsOldestFirst_AndFailsClosed() + { + var now = new DateTimeOffset(2026, 10, 15, 12, 0, 0, TimeSpan.Zero); + var policy = new ActivationRecycleRetention { MaxAge = TimeSpan.FromDays(30), MaxDeletionsPerRun = 2 }; + MeshNode Row(string id, int daysOld, string nodeType = ActivationRecycleRequest.NodeType, + string ns = ActivationRecycleRequest.Namespace) => + new(id, ns) { NodeType = nodeType, LastModified = daysOld < 0 ? default : now.AddDays(-daysOld) }; + + var rows = new[] + { + Row("fresh", 29), + Row("old40", 40), + Row("old60", 60), + Row("old50", 50), + Row("undated", -1), + Row("report", 90, ActivationRecycleReport.NodeType), + Row("elsewhere", 90, ns: "Admin/_Other"), + }; + + policy.SelectForPruning(rows, now).Should().Equal( + $"{ActivationRecycleRequest.Namespace}/old60", $"{ActivationRecycleRequest.Namespace}/old50"); + policy.SelectForPruning(rows, now, $"{ActivationRecycleRequest.Namespace}/old60").Should().Equal( + $"{ActivationRecycleRequest.Namespace}/old50", $"{ActivationRecycleRequest.Namespace}/old40"); + (policy with { Enabled = false }).SelectForPruning(rows, now).Should().BeEmpty(); + } + + /// The configured window is clamped to the floor; a malformed value keeps the default. + [Fact] + public void FromConfiguration_ClampsTheWindow_AndIgnoresMalformedValues() + { + IConfiguration Config(string maxAge) => new ConfigurationBuilder() + .AddInMemoryCollection([new(ActivationRecycleRetention.MaxAgeConfigKey, maxAge)]).Build(); + + ActivationRecycleRetention.FromConfiguration(Config("00:00:01")).MaxAge + .Should().Be(ActivationRecycleRetention.MinimumMaxAge); + ActivationRecycleRetention.FromConfiguration(Config("not-a-span")).MaxAge + .Should().Be(ActivationRecycleRetention.Default.MaxAge); + ActivationRecycleRetention.FromConfiguration(Config("7.00:00:00")).MaxAge + .Should().Be(TimeSpan.FromDays(7)); + } +} From 1248c22046de080a99135b9a1a41692cca17d4b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Roland=20B=C3=BCrgi?= <6334612+rbuergi@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:27:38 +0200 Subject: [PATCH 2/2] fix(recycle): the ticket never waits for the prune; every parsed MaxAge is clamped (#6813) The prune that follows a new request ran inside the ticket's pipeline, so a slow listing (up to ReadBudget) plus sequential deletes could time a caller out on a request already committed and acted on. It is now its own subscription, started once the write lands. A configured zero or negative MaxAge parsed but fell back to the 30-day default instead of the documented one-day floor. Co-Authored-By: Claude Opus 5.5 (1M context) --- .../Architecture/StaleStateUntilRecycle.md | 5 ++-- .../Configuration/ActivationRecycle.cs | 25 ++++++++++++++++--- .../ActivationRecycleRetention.cs | 5 ++-- .../ActivationRecycleRetentionTest.cs | 4 +++ 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/src/MeshWeaver.Documentation/Data/Architecture/StaleStateUntilRecycle.md b/src/MeshWeaver.Documentation/Data/Architecture/StaleStateUntilRecycle.md index 1134f84d97..df51df5bc2 100644 --- a/src/MeshWeaver.Documentation/Data/Architecture/StaleStateUntilRecycle.md +++ b/src/MeshWeaver.Documentation/Data/Architecture/StaleStateUntilRecycle.md @@ -227,7 +227,7 @@ bound NodeType is T"* (optionally narrowed to some addresses, or to the activati | Key | Default | Meaning | |---|---|---| | `ActivationRecycle:Retention:Enabled` | `true` | `false` turns the prune off (no query is issued) | - | `ActivationRecycle:Retention:MaxAge` | `30.00:00:00` | a request untouched for this long is removed with its reports; a configured value below one day is clamped to one day | + | `ActivationRecycle:Retention:MaxAge` | `30.00:00:00` | a request untouched for this long is removed with its reports; a configured value below one day — zero and negative included — is clamped to one day; only an absent or unparseable value keeps the default | | `ActivationRecycle:Retention:MaxDeletionsPerRun` | `50` | the most requests one prune removes, so a backlog drains over several requests | Thirty days is long enough that the audit of a recent deploy or incident is still there when it @@ -242,7 +242,8 @@ bound NodeType is T"* (optionally narrowed to some addresses, or to the activati on it bounds them. The request just written is never pruned. The decision is fail-closed: a row that is undated, not a request, or not directly under `Admin/_Recycle` is kept. A failed listing or delete costs that prune only. It is logged, and the next request retries; the caller's ticket - never depends on it. + never depends on it — nor waits for it: the prune is a subscription of its own, started once the + request is committed, so the ticket is answered as soon as the write lands. The request is a platform write. On the control instance it is issued by the build queue as System: the `recycle` job kind (MeshWeaver.Plugins, `Hosting/Queue`) runs it, either when a NodeType of the diff --git a/src/MeshWeaver.Graph/Configuration/ActivationRecycle.cs b/src/MeshWeaver.Graph/Configuration/ActivationRecycle.cs index dbd57cc445..bfd1ba35f4 100644 --- a/src/MeshWeaver.Graph/Configuration/ActivationRecycle.cs +++ b/src/MeshWeaver.Graph/Configuration/ActivationRecycle.cs @@ -234,17 +234,34 @@ IObservable TypeExists(string type) => }; // The id is the dedupe key: an existing request is the answer, never a second write. // A NEW request is the moment the rows grow, so it is also when the expired ones go - // (ActivationRecycleRetention): the writer prunes, no timer does. + // (ActivationRecycleRetention): the writer prunes, no timer does. The prune is its + // OWN subscription, started once the request is committed: the ticket is answered + // the moment the write lands and never waits for the listing or the deletes. return Exists().SelectMany(there => there ? Observable.Return(new ActivationRecycleTicket(path, null)) : access.RunAsSystem(() => mesh.CreateNode(node).Take(1)) - .SelectMany(_ => Prune(hub, hub.ServiceProvider.GetRequiredService(), - DateTimeOffset.UtcNow, path) - .Select(_ => new ActivationRecycleTicket(path, null)))); + .Do(_ => StartPrune(hub, path)) + .Select(_ => new ActivationRecycleTicket(path, null))); }) .Take(1); } + /// + /// Starts the prune that follows a newly committed request as a subscription of its own, so the + /// request's ticket never waits for it. already maps every fault to a logged + /// zero; the error handler here is the backstop for anything that escapes it. + /// + private static void StartPrune(IMessageHub hub, string keepPath) + { + var logger = hub.ServiceProvider.GetService()?.CreateLogger(typeof(ActivationRecycle)); + Prune(hub, hub.ServiceProvider.GetRequiredService(), DateTimeOffset.UtcNow, keepPath) + .Subscribe( + _ => { }, + ex => logger?.LogWarning(ex, + "[ActivationRecycle] retention: the prune after {Path} faulted; expired requests stay until the next request prunes them", + keepPath)); + } + /// /// Removes the requests calls expired at , /// each with its per-process reports, never . Cold; emits once, the diff --git a/src/MeshWeaver.Graph/Configuration/ActivationRecycleRetention.cs b/src/MeshWeaver.Graph/Configuration/ActivationRecycleRetention.cs index 9b644bc617..cfd26207a1 100644 --- a/src/MeshWeaver.Graph/Configuration/ActivationRecycleRetention.cs +++ b/src/MeshWeaver.Graph/Configuration/ActivationRecycleRetention.cs @@ -134,8 +134,9 @@ public static ActivationRecycleRetention FromConfiguration(IConfiguration? confi if (bool.TryParse(configuration[EnabledConfigKey], out var enabled)) retention = retention with { Enabled = enabled }; - if (TimeSpan.TryParse(configuration[MaxAgeConfigKey], System.Globalization.CultureInfo.InvariantCulture, out var maxAge) - && maxAge > TimeSpan.Zero) + // Every PARSED span is clamped, zero and negative included: only an absent or unparseable + // value falls back to the default. + if (TimeSpan.TryParse(configuration[MaxAgeConfigKey], System.Globalization.CultureInfo.InvariantCulture, out var maxAge)) retention = retention with { MaxAge = maxAge < MinimumMaxAge ? MinimumMaxAge : maxAge }; if (int.TryParse(configuration[MaxDeletionsPerRunConfigKey], out var max) && max >= 1) retention = retention with { MaxDeletionsPerRun = max }; diff --git a/test/MeshWeaver.Graph.Test/ActivationRecycleRetentionTest.cs b/test/MeshWeaver.Graph.Test/ActivationRecycleRetentionTest.cs index adfc20d12a..0d53c336df 100644 --- a/test/MeshWeaver.Graph.Test/ActivationRecycleRetentionTest.cs +++ b/test/MeshWeaver.Graph.Test/ActivationRecycleRetentionTest.cs @@ -146,6 +146,10 @@ public void FromConfiguration_ClampsTheWindow_AndIgnoresMalformedValues() ActivationRecycleRetention.FromConfiguration(Config("00:00:01")).MaxAge .Should().Be(ActivationRecycleRetention.MinimumMaxAge); + ActivationRecycleRetention.FromConfiguration(Config("00:00:00")).MaxAge + .Should().Be(ActivationRecycleRetention.MinimumMaxAge, "a parsed zero is below the floor, not malformed"); + ActivationRecycleRetention.FromConfiguration(Config("-1.00:00:00")).MaxAge + .Should().Be(ActivationRecycleRetention.MinimumMaxAge, "a parsed negative span is below the floor, not malformed"); ActivationRecycleRetention.FromConfiguration(Config("not-a-span")).MaxAge .Should().Be(ActivationRecycleRetention.Default.MaxAge); ActivationRecycleRetention.FromConfiguration(Config("7.00:00:00")).MaxAge