diff --git a/Engine/Results/LiveTradingResultHandler.cs b/Engine/Results/LiveTradingResultHandler.cs index 954c95cd3411..f20519272052 100644 --- a/Engine/Results/LiveTradingResultHandler.cs +++ b/Engine/Results/LiveTradingResultHandler.cs @@ -254,8 +254,8 @@ private void Update() var deltaStatistics = new Dictionary(); var orders = new Dictionary(TransactionHandler.Orders); - var complete = new LiveResultPacket(_job, new LiveResult(new LiveResultParameters(chartComplete, orders, - Algorithm.Transactions.TransactionRecord, holdings, Algorithm.Portfolio.CashBook, deltaStatistics, + var complete = new LiveResultPacket(_job, new LiveResult(new LiveResultParameters(chartComplete, orders, + Algorithm.Transactions.TransactionRecord, holdings, Algorithm.Portfolio.CashBook, deltaStatistics, runtimeStatistics, orderEvents, statistics.TotalPerformance, serverStatistics, state: GetAlgorithmState()))); StoreResult(complete); _nextChartsUpdate = DateTime.UtcNow.Add(ChartUpdateInterval); @@ -336,21 +336,7 @@ private void Update() if (utcNow > _nextChartTrimming) { Log.Debug("LiveTradingResultHandler.Update(): Trimming charts"); - var timeLimitUtc = utcNow.AddDays(-2); - lock (ChartLock) - { - foreach (var chart in Charts) - { - foreach (var series in chart.Value.Series) - { - // trim data that's older than 2 days - series.Value.Values = - (from v in series.Value.Values - where v.Time > timeLimitUtc - select v).ToList(); - } - } - } + TrimCharts(utcNow); _nextChartTrimming = DateTime.UtcNow.AddMinutes(10); Log.Debug("LiveTradingResultHandler.Update(): Finished trimming charts"); } @@ -382,6 +368,56 @@ protected virtual void SetNextStatusUpdate() _nextStatusUpdate = DateTime.UtcNow.AddMinutes(10); } + /// + /// Trims old points from each chart series. The statistics series (equity, return and benchmark) keep + /// full resolution for the last 2 days and a daily sample for up to 2 years. Every other series keeps + /// only the last 2 days. + /// + protected virtual void TrimCharts(DateTime utcNow) + { + var fullResolutionLimit = utcNow.AddDays(-2); + var dailySampleLimit = utcNow.AddDays(-730); + + lock (ChartLock) + { + foreach (var chart in Charts) + { + foreach (var series in chart.Value.Series) + { + var isStatisticsSeries = + (chart.Key == StrategyEquityKey && (series.Key == EquityKey || series.Key == ReturnKey)) || + (chart.Key == BenchmarkKey && series.Key == BenchmarkKey); + + if (isStatisticsSeries) + { + series.Value.Values = TrimToDailySample(series.Value.Values, fullResolutionLimit, dailySampleLimit); + } + else + { + series.Value.Values = series.Value.Values + .Where(point => point.Time > fullResolutionLimit) + .ToList(); + } + } + } + } + } + + /// + /// Keeps all points within the full resolution limit, then one per day down to the daily sample limit, and drops the rest + /// + private static List TrimToDailySample(List values, DateTime fullResolutionLimit, DateTime dailySampleLimit) + { + var dailySamples = values + .Where(point => point.Time > dailySampleLimit && point.Time <= fullResolutionLimit) + .GroupBy(point => point.Time.Date) + .Select(group => group.Last()); + + var fullResolution = values.Where(point => point.Time > fullResolutionLimit); + + return dailySamples.Concat(fullResolution).ToList(); + } + /// /// Stores the order events /// diff --git a/Tests/Engine/Results/LiveTradingResultHandlerTests.cs b/Tests/Engine/Results/LiveTradingResultHandlerTests.cs index ba3da3aac2d1..079a109cd139 100644 --- a/Tests/Engine/Results/LiveTradingResultHandlerTests.cs +++ b/Tests/Engine/Results/LiveTradingResultHandlerTests.cs @@ -149,7 +149,7 @@ public void DailySampleValueBasedOnMarketHour(bool extendedMarketHoursEnabled) using var messagging = new QuantConnect.Messaging.Messaging(); var referenceDate = new DateTime(2020, 11, 25); var resultHandler = new LiveTradingResultHandler(); - resultHandler.Initialize(new (new LiveNodePacket(), messagging, api, new BacktestingTransactionHandler(), null)); + resultHandler.Initialize(new(new LiveNodePacket(), messagging, api, new BacktestingTransactionHandler(), null)); try { @@ -190,6 +190,82 @@ public void DailySampleValueBasedOnMarketHour(bool extendedMarketHoursEnabled) } } + [Test] + public void TrimChartsKeepsDailySampleOfStatisticsSeries() + { + var handler = new TestableLiveTradingResultHandler(); + var utcNow = new DateTime(2020, 11, 25, 12, 0, 0, DateTimeKind.Utc); + + var benchmarkChart = new Chart(BaseResultsHandler.BenchmarkKey); + benchmarkChart.Series.Add(BaseResultsHandler.BenchmarkKey, new Series(BaseResultsHandler.BenchmarkKey)); + handler.Charts[BaseResultsHandler.BenchmarkKey] = benchmarkChart; + + var customChart = new Chart("MyCustomChart"); + customChart.Series.Add("MyMetric", new Series("MyMetric")); + handler.Charts["MyCustomChart"] = customChart; + + var returnSeries = handler.Charts[BaseResultsHandler.StrategyEquityKey].Series[BaseResultsHandler.ReturnKey]; + var equitySeries = handler.Charts[BaseResultsHandler.StrategyEquityKey].Series[BaseResultsHandler.EquityKey]; + var benchmarkSeries = benchmarkChart.Series[BaseResultsHandler.BenchmarkKey]; + var customSeries = customChart.Series["MyMetric"]; + + // Return and Benchmark: one point per day, going beyond 2 years + for (var i = 800; i >= 1; i--) + { + var t = utcNow.AddDays(-i); + returnSeries.Values.Add(new ChartPoint(t, i)); + benchmarkSeries.Values.Add(new ChartPoint(t, i)); + } + + // Equity: several points per day for older days, plus a couple of recent ones + foreach (var day in new[] { 5, 4, 3 }) + { + var date = utcNow.AddDays(-day).Date; + equitySeries.Values.Add(new Candlestick(date.AddHours(10), 100, 110, 90, 101)); + equitySeries.Values.Add(new Candlestick(date.AddHours(14), 100, 110, 90, 102)); + equitySeries.Values.Add(new Candlestick(date.AddHours(16), 100, 110, 90, 103)); // last sample of the day + } + // Two recent points on the same day, within the 2 day window + equitySeries.Values.Add(new Candlestick(utcNow.AddHours(-5), 100, 110, 90, 200)); + equitySeries.Values.Add(new Candlestick(utcNow.AddHours(-1), 100, 110, 90, 201)); + + // Custom chart: not a statistics series, so no daily sample + for (var i = 5; i >= 1; i--) + { + customSeries.Values.Add(new ChartPoint(utcNow.AddDays(-i), i)); + } + + handler.PublicTrimCharts(utcNow); + + // Return and Benchmark keep one point per day, up to 2 years + var dailyStatsCutoff = utcNow.AddDays(-730); + Assert.IsTrue(returnSeries.Values.All(v => v.Time > dailyStatsCutoff)); + Assert.IsTrue(benchmarkSeries.Values.All(v => v.Time > dailyStatsCutoff)); + Assert.AreEqual(729, returnSeries.Values.Count); + Assert.AreEqual(729, benchmarkSeries.Values.Count); + + // Equity keeps all recent points and one per day for older ones + Assert.AreEqual(5, equitySeries.Values.Count); + foreach (var day in new[] { 5, 4, 3 }) + { + var date = utcNow.AddDays(-day).Date; + var samplesForDay = equitySeries.Values.Where(v => v.Time.Date == date).ToList(); + Assert.AreEqual(1, samplesForDay.Count); // only one point per day + Assert.AreEqual(103, ((Candlestick)samplesForDay[0]).Close); // and it is the last of the day + } + Assert.AreEqual(2, equitySeries.Values.Count(v => v.Time > utcNow.AddDays(-2))); + + // Custom chart keeps only the last 2 days + var defaultCutoff = utcNow.AddDays(-2); + Assert.IsTrue(customSeries.Values.All(v => v.Time > defaultCutoff)); + Assert.AreEqual(1, customSeries.Values.Count); + } + + private class TestableLiveTradingResultHandler : LiveTradingResultHandler + { + public void PublicTrimCharts(DateTime utcNow) => TrimCharts(utcNow); + } + private class TestDataFeed : IDataFeed { public bool IsActive { get; }