Fix zero live statistics caused by chart trimming

Engine/Results/LiveTradingResultHandler.cs

@@ -254,8 +254,8 @@ private void Update()
 
                         var deltaStatistics = new Dictionary<string, string>();
                         var orders = new Dictionary<int, Order>(TransactionHandler.Orders);
-                        var complete = new LiveResultPacket(_job, new LiveResult(new LiveResultParameters(chartComplete, orders, 
-                            Algorithm.Transactions.TransactionRecord, holdings, Algorithm.Portfolio.CashBook, deltaStatistics, 
+                        var complete = new LiveResultPacket(_job, new LiveResult(new LiveResultParameters(chartComplete, orders,
+                            Algorithm.Transactions.TransactionRecord, holdings, Algorithm.Portfolio.CashBook, deltaStatistics,
                             runtimeStatistics, orderEvents, statistics.TotalPerformance, serverStatistics, state: GetAlgorithmState())));
                         StoreResult(complete);
                         _nextChartsUpdate = DateTime.UtcNow.Add(ChartUpdateInterval);
@@ -336,21 +336,7 @@ private void Update()
                     if (utcNow > _nextChartTrimming)
                     {
                         Log.Debug("LiveTradingResultHandler.Update(): Trimming charts");
-                        var timeLimitUtc = utcNow.AddDays(-2);
-                        lock (ChartLock)
-                        {
-                            foreach (var chart in Charts)
-                            {
-                                foreach (var series in chart.Value.Series)
-                                {
-                                    // trim data that's older than 2 days
-                                    series.Value.Values =
-                                        (from v in series.Value.Values
-                                         where v.Time > timeLimitUtc
-                                         select v).ToList();
-                                }
-                            }
-                        }
+                        TrimCharts(utcNow);
                         _nextChartTrimming = DateTime.UtcNow.AddMinutes(10);
                         Log.Debug("LiveTradingResultHandler.Update(): Finished trimming charts");
                     }
@@ -382,6 +368,56 @@ protected virtual void SetNextStatusUpdate()
             _nextStatusUpdate = DateTime.UtcNow.AddMinutes(10);
         }
 
+        /// <summary>
+        /// Trims old points from each chart series. The statistics series (equity, return and benchmark) keep
+        /// full resolution for the last 2 days and a daily sample for up to 2 years. Every other series keeps
+        /// only the last 2 days.
+        /// </summary>
+        protected virtual void TrimCharts(DateTime utcNow)
+        {
+            var fullResolutionLimit = utcNow.AddDays(-2);
+            var dailySampleLimit = utcNow.AddDays(-730);
+
+            lock (ChartLock)
+            {
+                foreach (var chart in Charts)
+                {
+                    foreach (var series in chart.Value.Series)
+                    {
+                        var isStatisticsSeries =
+                            (chart.Key == StrategyEquityKey && (series.Key == EquityKey || series.Key == ReturnKey)) ||
+                            (chart.Key == BenchmarkKey && series.Key == BenchmarkKey);
+
+                        if (isStatisticsSeries)
+                        {
+                            series.Value.Values = TrimToDailySample(series.Value.Values, fullResolutionLimit, dailySampleLimit);
+                        }
+                        else
+                        {
+                            series.Value.Values = series.Value.Values
+                                .Where(point => point.Time > fullResolutionLimit)
+                                .ToList();
+                        }
+                    }
+                }
+            }
+        }
+
+        /// <summary>
+        /// Keeps all points within the full resolution limit, then one per day down to the daily sample limit, and drops the rest
+        /// </summary>
+        private static List<ISeriesPoint> TrimToDailySample(List<ISeriesPoint> values, DateTime fullResolutionLimit, DateTime dailySampleLimit)
+        {
+            var dailySamples = values
+                .Where(point => point.Time > dailySampleLimit && point.Time <= fullResolutionLimit)
+                .GroupBy(point => point.Time.Date)
+                .Select(group => group.Last());
+
+            var fullResolution = values.Where(point => point.Time > fullResolutionLimit);
+
+            return dailySamples.Concat(fullResolution).ToList();
+        }
+
         /// <summary>
         /// Stores the order events
         /// </summary>

Tests/Engine/Results/LiveTradingResultHandlerTests.cs

@@ -149,7 +149,7 @@ public void DailySampleValueBasedOnMarketHour(bool extendedMarketHoursEnabled)
             using var messagging = new QuantConnect.Messaging.Messaging();
             var referenceDate = new DateTime(2020, 11, 25);
             var resultHandler = new LiveTradingResultHandler();
-            resultHandler.Initialize(new (new LiveNodePacket(), messagging, api, new BacktestingTransactionHandler(), null));
+            resultHandler.Initialize(new(new LiveNodePacket(), messagging, api, new BacktestingTransactionHandler(), null));
 
             try
             {
@@ -190,6 +190,82 @@ public void DailySampleValueBasedOnMarketHour(bool extendedMarketHoursEnabled)
             }
         }
 
+        [Test]
+        public void TrimChartsKeepsDailySampleOfStatisticsSeries()
+        {
+            var handler = new TestableLiveTradingResultHandler();
+            var utcNow = new DateTime(2020, 11, 25, 12, 0, 0, DateTimeKind.Utc);
+
+            var benchmarkChart = new Chart(BaseResultsHandler.BenchmarkKey);
+            benchmarkChart.Series.Add(BaseResultsHandler.BenchmarkKey, new Series(BaseResultsHandler.BenchmarkKey));
+            handler.Charts[BaseResultsHandler.BenchmarkKey] = benchmarkChart;
+
+            var customChart = new Chart("MyCustomChart");
+            customChart.Series.Add("MyMetric", new Series("MyMetric"));
+            handler.Charts["MyCustomChart"] = customChart;
+
+            var returnSeries = handler.Charts[BaseResultsHandler.StrategyEquityKey].Series[BaseResultsHandler.ReturnKey];
+            var equitySeries = handler.Charts[BaseResultsHandler.StrategyEquityKey].Series[BaseResultsHandler.EquityKey];
+            var benchmarkSeries = benchmarkChart.Series[BaseResultsHandler.BenchmarkKey];
+            var customSeries = customChart.Series["MyMetric"];
+
+            // Return and Benchmark: one point per day, going beyond 2 years
+            for (var i = 800; i >= 1; i--)
+            {
+                var t = utcNow.AddDays(-i);
+                returnSeries.Values.Add(new ChartPoint(t, i));
+                benchmarkSeries.Values.Add(new ChartPoint(t, i));
+            }
+
+            // Equity: several points per day for older days, plus a couple of recent ones
+            foreach (var day in new[] { 5, 4, 3 })
+            {
+                var date = utcNow.AddDays(-day).Date;
+                equitySeries.Values.Add(new Candlestick(date.AddHours(10), 100, 110, 90, 101));
+                equitySeries.Values.Add(new Candlestick(date.AddHours(14), 100, 110, 90, 102));
+                equitySeries.Values.Add(new Candlestick(date.AddHours(16), 100, 110, 90, 103)); // last sample of the day
+            }
+            // Two recent points on the same day, within the 2 day window
+            equitySeries.Values.Add(new Candlestick(utcNow.AddHours(-5), 100, 110, 90, 200));
+            equitySeries.Values.Add(new Candlestick(utcNow.AddHours(-1), 100, 110, 90, 201));
+
+            // Custom chart: not a statistics series, so no daily sample
+            for (var i = 5; i >= 1; i--)
+            {
+                customSeries.Values.Add(new ChartPoint(utcNow.AddDays(-i), i));
+            }
+
+            handler.PublicTrimCharts(utcNow);
+
+            // Return and Benchmark keep one point per day, up to 2 years
+            var dailyStatsCutoff = utcNow.AddDays(-730);
+            Assert.IsTrue(returnSeries.Values.All(v => v.Time > dailyStatsCutoff));
+            Assert.IsTrue(benchmarkSeries.Values.All(v => v.Time > dailyStatsCutoff));
+            Assert.AreEqual(729, returnSeries.Values.Count);
+            Assert.AreEqual(729, benchmarkSeries.Values.Count);
+
+            // Equity keeps all recent points and one per day for older ones
+            Assert.AreEqual(5, equitySeries.Values.Count);
+            foreach (var day in new[] { 5, 4, 3 })
+            {
+                var date = utcNow.AddDays(-day).Date;
+                var samplesForDay = equitySeries.Values.Where(v => v.Time.Date == date).ToList();
+                Assert.AreEqual(1, samplesForDay.Count); // only one point per day
+                Assert.AreEqual(103, ((Candlestick)samplesForDay[0]).Close); // and it is the last of the day
+            }
+            Assert.AreEqual(2, equitySeries.Values.Count(v => v.Time > utcNow.AddDays(-2)));
+
+            // Custom chart keeps only the last 2 days
+            var defaultCutoff = utcNow.AddDays(-2);
+            Assert.IsTrue(customSeries.Values.All(v => v.Time > defaultCutoff));
+            Assert.AreEqual(1, customSeries.Values.Count);
+        }
+
+        private class TestableLiveTradingResultHandler : LiveTradingResultHandler
+        {
+            public void PublicTrimCharts(DateTime utcNow) => TrimCharts(utcNow);
+        }
+
         private class TestDataFeed : IDataFeed
         {
             public bool IsActive { get; }