[SDTEST-3870] Improve planner report narrative

internal/planner/discovered_tests.go

@@ -17,6 +17,7 @@ func (tp *TestPlanner) resetDiscoveryResults() {
 	tp.testFiles = make(map[string]struct{})
 	tp.suiteAggregates = make(map[testSuiteKey]testSuiteAggregate)
 	tp.suitesBySourceFile = make(map[string][]testSuiteKey)
+	tp.reportStats = newPlanningReportStats()
 }
 
 func (tp *TestPlanner) recordFullDiscoveryResults(
@@ -46,7 +47,6 @@ func (tp *TestPlanner) recordFullDiscoveryResults(
 				break
 			}
 		}
-		slog.Debug("Skippable matcher size", "size", skippableMatcher.Count())
 	}
 
 	skippableTestsCount := 0
@@ -65,12 +65,14 @@ func (tp *TestPlanner) recordFullDiscoveryResults(
 			tp.testFiles[normalizedSourceFile] = struct{}{}
 		}
 
-		if !skippableMatcher.Contains(test) {
+		match := skippableMatcher.Match(test)
+		if !match.Skipped() {
 			slog.Debug("Test is not skipped", "test", test.DatadogTestId(), "sourceFile", test.SuiteSourceFile)
 			recordRunnableTest(tp.suiteAggregates, test, normalizedSourceFile)
 		} else {
 			slog.Debug("Test will be skipped", "test", test.DatadogTestId(), "sourceFile", test.SuiteSourceFile)
 			recordSkippedTest(tp.suiteAggregates, test, normalizedSourceFile)
+			tp.recordAppliedSkippable(match)
 			skippableTestsCount++
 		}
 	}
@@ -88,6 +90,24 @@ type skippableMatcher struct {
 	disabledTests      map[string]bool
 }
 
+type skippableMatch struct {
+	kind     skippableMatchKind
+	suiteKey testSuiteKey
+}
+
+type skippableMatchKind int
+
+const (
+	skippableMatchNone skippableMatchKind = iota
+	skippableMatchTIATest
+	skippableMatchTIASuite
+	skippableMatchDisabledTest
+)
+
+func (m skippableMatch) Skipped() bool {
+	return m.kind != skippableMatchNone
+}
+
 func newSkippableMatcher(skippables api.Skippables, disabledTests map[string]bool) skippableMatcher {
 	if skippables.Tests == nil {
 		skippables.Tests = api.SkippableTests{}
@@ -106,21 +126,40 @@ func newSkippableMatcher(skippables api.Skippables, disabledTests map[string]boo
 	}
 }
 
-func (s skippableMatcher) Contains(test testoptimization.Test) bool {
+func (s skippableMatcher) Match(test testoptimization.Test) skippableMatch {
 	suite := api.SkippableSuite{Module: test.Module, Suite: test.Suite}
-	return s.tiaSkippableTests[test.DatadogTestId()] ||
-		s.tiaSkippableSuites[suite] ||
-		s.disabledTests[test.FQN()]
-}
-
-func (s skippableMatcher) Count() int {
-	return s.TIASkippablesCount() + len(s.disabledTests)
+	if s.disabledTests[test.FQN()] {
+		return skippableMatch{kind: skippableMatchDisabledTest}
+	}
+	if s.tiaSkippableSuites[suite] {
+		return skippableMatch{
+			kind:     skippableMatchTIASuite,
+			suiteKey: testSuiteKey{Module: test.Module, Suite: test.Suite},
+		}
+	}
+	if s.tiaSkippableTests[test.DatadogTestId()] {
+		return skippableMatch{kind: skippableMatchTIATest}
+	}
+	return skippableMatch{}
 }
 
 func (s skippableMatcher) TIASkippablesCount() int {
 	return len(s.tiaSkippableTests) + len(s.tiaSkippableSuites)
 }
 
+func (tp *TestPlanner) recordAppliedSkippable(match skippableMatch) {
+	switch match.kind {
+	case skippableMatchTIATest:
+		tp.reportStats.tiaSkippableTestsApplied++
+	case skippableMatchTIASuite:
+		// Full discovery sees suite-level skippables once per test, so keep a set
+		// and report the number of suites rather than the number of skipped tests.
+		tp.reportStats.uniqueTIASkippableSuitesApplied[match.suiteKey] = struct{}{}
+	case skippableMatchDisabledTest:
+		tp.reportStats.disabledTestsApplied++
+	}
+}
+
 func (tp *TestPlanner) recordFastDiscoveryFallbackFiles(discoveredTestFiles []string) error {
 	excluder, err := discovery.NewExcluder(settings.GetTestsExcludePattern())
 	if err != nil {
@@ -178,11 +217,13 @@ func (tp *TestPlanner) recordSuiteLevelSkippables(skippableMatcher skippableMatc
 		}
 		aggregate.NumTestsSkipped = aggregate.NumTests
 		tp.suiteAggregates[key] = aggregate
+		tp.reportStats.uniqueTIASkippableSuitesApplied[key] = struct{}{}
 	}
 }
 
 func (tp *TestPlanner) keepUnskippableMarkerSuitesRunnable(testFramework framework.Framework) {
 	if testFramework == nil {
+		tp.reportStats.unskippableMarkerSuitesForced = 0
 		return
 	}
 
@@ -212,6 +253,7 @@ func (tp *TestPlanner) keepUnskippableMarkerSuitesRunnable(testFramework framewo
 	slog.Info("Checked unskippable marker suites",
 		"duration", time.Since(startTime),
 		"forceRunnableSuiteAggregatesCount", forceRunnableSuiteAggregatesCount)
+	tp.reportStats.unskippableMarkerSuitesForced = forceRunnableSuiteAggregatesCount
 }
 
 func (tp *TestPlanner) sourceFileForSuite(key testSuiteKey, testFramework framework.Framework) (string, bool) {

internal/planner/discovery_cache.go

@@ -53,6 +53,12 @@ type discoveryCache struct {
 	filePath      string
 }
 
+type discoveryCacheResult struct {
+	Configured    bool
+	Used          bool
+	NotUsedReason string
+}
+
 var discoveryCacheGitOutput = func(args ...string) ([]byte, error) {
 	return exec.Command("git", args...).Output()
 }
@@ -208,31 +214,39 @@ func readLastNonEmptyLine(filePath string) ([]byte, error) {
 	}
 }
 
-func (c discoveryCache) restore() ([]testoptimization.Test, bool) {
+func (c discoveryCache) restore() ([]testoptimization.Test, discoveryCacheResult) {
+	result := discoveryCacheResult{
+		Configured: settings.GetTestDiscoveryCache() != "",
+	}
+
 	c.importExternal()
 
 	if err := c.validate(); err != nil {
-		if settings.GetTestDiscoveryCache() != "" {
+		result.NotUsedReason = err.Error()
+		if result.Configured {
 			slog.Info("Cached test discovery not usable; full discovery will run", "reason", err)
 		} else {
 			slog.Debug("Cached test discovery not usable; full discovery will run", "reason", err)
 		}
-		return nil, false
+		return nil, result
 	}
 
 	startTime := time.Now()
 	tests, err := parseCachedDiscoveryTests(c.filePath)
 	if err != nil {
+		result.NotUsedReason = err.Error()
 		slog.Info("Cached test discovery could not be used; full discovery will run", "error", err)
-		return nil, false
+		return nil, result
 	}
 	if err := ensureDiscoveredTests(tests); err != nil {
+		result.NotUsedReason = err.Error()
 		slog.Info("Cached test discovery could not be used; full discovery will run", "error", err)
-		return nil, false
+		return nil, result
 	}
 
+	result.Used = true
 	slog.Info("Cached test discovery succeeded", "duration", time.Since(startTime), "count", len(tests))
-	return tests, true
+	return tests, result
 }
 
 func ensureDiscoveredTests(tests []testoptimization.Test) error {

internal/planner/discovery_cache_test.go

@@ -192,6 +192,10 @@ func TestDiscoveryCacheHitUsesCachedTests(t *testing.T) {
 	if len(mockFramework.DiscoverTestsFiles) != 0 {
 		t.Fatalf("expected cache hit to avoid full discovery, got %d calls", len(mockFramework.DiscoverTestsFiles))
 	}
+	report := runner.newPlanReportData(splitScore{})
+	if !report.Planning.Discovery.Cache.Used {
+		t.Fatalf("expected cache hit to be reported, got %+v", report.Planning.Discovery.Cache)
+	}
 	aggregate := runner.suiteAggregates[testSuiteKey{Module: "rspec", Suite: "Cart"}]
 	if aggregate.NumTests != 1 || aggregate.NumTestsSkipped != 1 {
 		t.Fatalf("cached aggregate = %+v, want one skipped test", aggregate)
@@ -290,6 +294,10 @@ func TestDiscoveryCacheImportsExternalCacheBeforeValidation(t *testing.T) {
 	if len(mockFramework.DiscoverTestsFiles) != 0 {
 		t.Fatalf("expected imported cache to avoid full discovery, got %d calls", len(mockFramework.DiscoverTestsFiles))
 	}
+	report := runner.newPlanReportData(splitScore{})
+	if cacheResult := report.Planning.Discovery.Cache; !cacheResult.Configured || !cacheResult.Used {
+		t.Fatalf("expected imported cache to be reported as configured and used, got %+v", cacheResult)
+	}
 	if _, err := os.Stat(discovery.TestsFilePath); err != nil {
 		t.Fatalf("expected imported cache at internal path: %v", err)
 	}
@@ -303,9 +311,12 @@ func TestDiscoveryCacheRestoreRejectsEmptyCache(t *testing.T) {
 	writePlannerDiscoveryCache(t, "base-sha", "ruby", "rspec", pattern, "", nil)
 	cache := newDiscoveryCache("ruby", &MockFramework{FrameworkName: "rspec", TestPatternValue: pattern})
 
-	tests, ok := cache.restore()
-	if ok || tests != nil {
-		t.Fatalf("expected empty cache restore to fail, got ok=%v tests=%v", ok, tests)
+	tests, report := cache.restore()
+	if report.Used || tests != nil {
+		t.Fatalf("expected empty cache restore to fail, got report=%+v tests=%v", report, tests)
+	}
+	if report.NotUsedReason != "test discovery returned no tests" {
+		t.Fatalf("unexpected empty cache restore reason: %q", report.NotUsedReason)
 	}
 }
 

internal/planner/high_skippable_integration_test.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 	"slices"
+	"strconv"
 	"strings"
 	"testing"
 
@@ -93,10 +94,7 @@ func TestTestPlanner_Plan_HighSkippableIntegrationSelectsExpectedRunnerCountAndR
 	}
 
 	expectedParallelRunners := fixture.ExpectedParallelRunners
-	if runner.planReport.Split.parallelRunners != expectedParallelRunners {
-		t.Fatalf("Plan() selected %d runners, expected %d", runner.planReport.Split.parallelRunners, expectedParallelRunners)
-	}
-	assertFileContent(t, constants.ParallelRunnersOutputPath, "1")
+	assertFileContent(t, constants.ParallelRunnersOutputPath, strconv.Itoa(expectedParallelRunners))
 
 	testFiles := readTestPlanLines(t, constants.TestFilesOutputPath)
 	if !slices.Equal(testFiles, expectedRunnableFiles) {