diff --git a/internal/planner/discovered_tests.go b/internal/planner/discovered_tests.go index 2f27f3b..ec3f771 100644 --- a/internal/planner/discovered_tests.go +++ b/internal/planner/discovered_tests.go @@ -17,6 +17,7 @@ func (tp *TestPlanner) resetDiscoveryResults() { tp.testFiles = make(map[string]struct{}) tp.suiteAggregates = make(map[testSuiteKey]testSuiteAggregate) tp.suitesBySourceFile = make(map[string][]testSuiteKey) + tp.reportStats = newPlanningReportStats() } func (tp *TestPlanner) recordFullDiscoveryResults( @@ -46,7 +47,6 @@ func (tp *TestPlanner) recordFullDiscoveryResults( break } } - slog.Debug("Skippable matcher size", "size", skippableMatcher.Count()) } skippableTestsCount := 0 @@ -65,12 +65,14 @@ func (tp *TestPlanner) recordFullDiscoveryResults( tp.testFiles[normalizedSourceFile] = struct{}{} } - if !skippableMatcher.Contains(test) { + match := skippableMatcher.Match(test) + if !match.Skipped() { slog.Debug("Test is not skipped", "test", test.DatadogTestId(), "sourceFile", test.SuiteSourceFile) recordRunnableTest(tp.suiteAggregates, test, normalizedSourceFile) } else { slog.Debug("Test will be skipped", "test", test.DatadogTestId(), "sourceFile", test.SuiteSourceFile) recordSkippedTest(tp.suiteAggregates, test, normalizedSourceFile) + tp.recordAppliedSkippable(match) skippableTestsCount++ } } @@ -88,6 +90,24 @@ type skippableMatcher struct { disabledTests map[string]bool } +type skippableMatch struct { + kind skippableMatchKind + suiteKey testSuiteKey +} + +type skippableMatchKind int + +const ( + skippableMatchNone skippableMatchKind = iota + skippableMatchTIATest + skippableMatchTIASuite + skippableMatchDisabledTest +) + +func (m skippableMatch) Skipped() bool { + return m.kind != skippableMatchNone +} + func newSkippableMatcher(skippables api.Skippables, disabledTests map[string]bool) skippableMatcher { if skippables.Tests == nil { skippables.Tests = api.SkippableTests{} @@ -106,21 +126,40 @@ func newSkippableMatcher(skippables api.Skippables, disabledTests map[string]boo } } -func (s skippableMatcher) Contains(test testoptimization.Test) bool { +func (s skippableMatcher) Match(test testoptimization.Test) skippableMatch { suite := api.SkippableSuite{Module: test.Module, Suite: test.Suite} - return s.tiaSkippableTests[test.DatadogTestId()] || - s.tiaSkippableSuites[suite] || - s.disabledTests[test.FQN()] -} - -func (s skippableMatcher) Count() int { - return s.TIASkippablesCount() + len(s.disabledTests) + if s.disabledTests[test.FQN()] { + return skippableMatch{kind: skippableMatchDisabledTest} + } + if s.tiaSkippableSuites[suite] { + return skippableMatch{ + kind: skippableMatchTIASuite, + suiteKey: testSuiteKey{Module: test.Module, Suite: test.Suite}, + } + } + if s.tiaSkippableTests[test.DatadogTestId()] { + return skippableMatch{kind: skippableMatchTIATest} + } + return skippableMatch{} } func (s skippableMatcher) TIASkippablesCount() int { return len(s.tiaSkippableTests) + len(s.tiaSkippableSuites) } +func (tp *TestPlanner) recordAppliedSkippable(match skippableMatch) { + switch match.kind { + case skippableMatchTIATest: + tp.reportStats.tiaSkippableTestsApplied++ + case skippableMatchTIASuite: + // Full discovery sees suite-level skippables once per test, so keep a set + // and report the number of suites rather than the number of skipped tests. + tp.reportStats.uniqueTIASkippableSuitesApplied[match.suiteKey] = struct{}{} + case skippableMatchDisabledTest: + tp.reportStats.disabledTestsApplied++ + } +} + func (tp *TestPlanner) recordFastDiscoveryFallbackFiles(discoveredTestFiles []string) error { excluder, err := discovery.NewExcluder(settings.GetTestsExcludePattern()) if err != nil { @@ -178,11 +217,13 @@ func (tp *TestPlanner) recordSuiteLevelSkippables(skippableMatcher skippableMatc } aggregate.NumTestsSkipped = aggregate.NumTests tp.suiteAggregates[key] = aggregate + tp.reportStats.uniqueTIASkippableSuitesApplied[key] = struct{}{} } } func (tp *TestPlanner) keepUnskippableMarkerSuitesRunnable(testFramework framework.Framework) { if testFramework == nil { + tp.reportStats.unskippableMarkerSuitesForced = 0 return } @@ -212,6 +253,7 @@ func (tp *TestPlanner) keepUnskippableMarkerSuitesRunnable(testFramework framewo slog.Info("Checked unskippable marker suites", "duration", time.Since(startTime), "forceRunnableSuiteAggregatesCount", forceRunnableSuiteAggregatesCount) + tp.reportStats.unskippableMarkerSuitesForced = forceRunnableSuiteAggregatesCount } func (tp *TestPlanner) sourceFileForSuite(key testSuiteKey, testFramework framework.Framework) (string, bool) { diff --git a/internal/planner/discovery_cache.go b/internal/planner/discovery_cache.go index cc573fb..2532683 100644 --- a/internal/planner/discovery_cache.go +++ b/internal/planner/discovery_cache.go @@ -53,6 +53,12 @@ type discoveryCache struct { filePath string } +type discoveryCacheResult struct { + Configured bool + Used bool + NotUsedReason string +} + var discoveryCacheGitOutput = func(args ...string) ([]byte, error) { return exec.Command("git", args...).Output() } @@ -208,31 +214,39 @@ func readLastNonEmptyLine(filePath string) ([]byte, error) { } } -func (c discoveryCache) restore() ([]testoptimization.Test, bool) { +func (c discoveryCache) restore() ([]testoptimization.Test, discoveryCacheResult) { + result := discoveryCacheResult{ + Configured: settings.GetTestDiscoveryCache() != "", + } + c.importExternal() if err := c.validate(); err != nil { - if settings.GetTestDiscoveryCache() != "" { + result.NotUsedReason = err.Error() + if result.Configured { slog.Info("Cached test discovery not usable; full discovery will run", "reason", err) } else { slog.Debug("Cached test discovery not usable; full discovery will run", "reason", err) } - return nil, false + return nil, result } startTime := time.Now() tests, err := parseCachedDiscoveryTests(c.filePath) if err != nil { + result.NotUsedReason = err.Error() slog.Info("Cached test discovery could not be used; full discovery will run", "error", err) - return nil, false + return nil, result } if err := ensureDiscoveredTests(tests); err != nil { + result.NotUsedReason = err.Error() slog.Info("Cached test discovery could not be used; full discovery will run", "error", err) - return nil, false + return nil, result } + result.Used = true slog.Info("Cached test discovery succeeded", "duration", time.Since(startTime), "count", len(tests)) - return tests, true + return tests, result } func ensureDiscoveredTests(tests []testoptimization.Test) error { diff --git a/internal/planner/discovery_cache_test.go b/internal/planner/discovery_cache_test.go index 3a9255c..63de691 100644 --- a/internal/planner/discovery_cache_test.go +++ b/internal/planner/discovery_cache_test.go @@ -192,6 +192,10 @@ func TestDiscoveryCacheHitUsesCachedTests(t *testing.T) { if len(mockFramework.DiscoverTestsFiles) != 0 { t.Fatalf("expected cache hit to avoid full discovery, got %d calls", len(mockFramework.DiscoverTestsFiles)) } + report := runner.newPlanReportData(splitScore{}) + if !report.Planning.Discovery.Cache.Used { + t.Fatalf("expected cache hit to be reported, got %+v", report.Planning.Discovery.Cache) + } aggregate := runner.suiteAggregates[testSuiteKey{Module: "rspec", Suite: "Cart"}] if aggregate.NumTests != 1 || aggregate.NumTestsSkipped != 1 { t.Fatalf("cached aggregate = %+v, want one skipped test", aggregate) @@ -290,6 +294,10 @@ func TestDiscoveryCacheImportsExternalCacheBeforeValidation(t *testing.T) { if len(mockFramework.DiscoverTestsFiles) != 0 { t.Fatalf("expected imported cache to avoid full discovery, got %d calls", len(mockFramework.DiscoverTestsFiles)) } + report := runner.newPlanReportData(splitScore{}) + if cacheResult := report.Planning.Discovery.Cache; !cacheResult.Configured || !cacheResult.Used { + t.Fatalf("expected imported cache to be reported as configured and used, got %+v", cacheResult) + } if _, err := os.Stat(discovery.TestsFilePath); err != nil { t.Fatalf("expected imported cache at internal path: %v", err) } @@ -303,9 +311,12 @@ func TestDiscoveryCacheRestoreRejectsEmptyCache(t *testing.T) { writePlannerDiscoveryCache(t, "base-sha", "ruby", "rspec", pattern, "", nil) cache := newDiscoveryCache("ruby", &MockFramework{FrameworkName: "rspec", TestPatternValue: pattern}) - tests, ok := cache.restore() - if ok || tests != nil { - t.Fatalf("expected empty cache restore to fail, got ok=%v tests=%v", ok, tests) + tests, report := cache.restore() + if report.Used || tests != nil { + t.Fatalf("expected empty cache restore to fail, got report=%+v tests=%v", report, tests) + } + if report.NotUsedReason != "test discovery returned no tests" { + t.Fatalf("unexpected empty cache restore reason: %q", report.NotUsedReason) } } diff --git a/internal/planner/high_skippable_integration_test.go b/internal/planner/high_skippable_integration_test.go index 308db2f..99ee041 100644 --- a/internal/planner/high_skippable_integration_test.go +++ b/internal/planner/high_skippable_integration_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "testing" @@ -93,10 +94,7 @@ func TestTestPlanner_Plan_HighSkippableIntegrationSelectsExpectedRunnerCountAndR } expectedParallelRunners := fixture.ExpectedParallelRunners - if runner.planReport.Split.parallelRunners != expectedParallelRunners { - t.Fatalf("Plan() selected %d runners, expected %d", runner.planReport.Split.parallelRunners, expectedParallelRunners) - } - assertFileContent(t, constants.ParallelRunnersOutputPath, "1") + assertFileContent(t, constants.ParallelRunnersOutputPath, strconv.Itoa(expectedParallelRunners)) testFiles := readTestPlanLines(t, constants.TestFilesOutputPath) if !slices.Equal(testFiles, expectedRunnableFiles) { diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 6bf06b8..34d2230 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -27,7 +27,7 @@ import ( type Planner interface { Plan(ctx context.Context) error - LoadPlan() (PlanInfo, error) + LoadPlan() (PlanMetadata, error) DistributeTestFiles(testFiles []string, parallelRunners int) [][]string } @@ -39,10 +39,11 @@ type testOptimizationClient interface { GetTestManagementTestsData() *api.TestManagementTestsResponseDataModules GetDisabledTests() map[string]bool GetTestSuiteDurations() *api.TestSuiteDurationsResponseData + BackendRequestTimings() testoptimization.BackendRequestTimings StoreCacheAndExit() } -type PlanInfo struct { +type PlanMetadata struct { Platform string `json:"platform"` Framework string `json:"framework"` TestSkippingLevel string `json:"testSkippingLevel,omitempty"` @@ -50,8 +51,8 @@ type PlanInfo struct { RuntimeTags map[string]string `json:"runtimeTags"` } -func NewPlanInfo(tags map[string]string, platformName, frameworkName string, testSkippingLevel settings.TestSkippingLevel) PlanInfo { - return PlanInfo{ +func NewPlanMetadata(tags map[string]string, platformName, frameworkName string, testSkippingLevel settings.TestSkippingLevel) PlanMetadata { + return PlanMetadata{ Platform: platformName, Framework: frameworkName, TestSkippingLevel: testSkippingLevel.String(), @@ -60,7 +61,7 @@ func NewPlanInfo(tags map[string]string, platformName, frameworkName string, tes } } -func (p PlanInfo) IsZero() bool { +func (p PlanMetadata) IsZero() bool { return p.Platform == "" && p.Framework == "" && p.TestSkippingLevel == "" && @@ -75,11 +76,11 @@ type TestPlanner struct { testSuiteDurations map[string]map[string]api.TestSuiteDurationInfo testFileWeights map[string]int testFileDurationSources map[string]testFileDurationSource + reportStats planningReportStats skippablePercentage float64 - planReport planReport planLoaded bool runInfo runmetadata.RunInfo - planInfo PlanInfo + planMetadata PlanMetadata platformDetector platform.PlatformDetector optimizationClient testOptimizationClient newOptimizationClient func(testSkippingLevel settings.TestSkippingLevel) testOptimizationClient @@ -178,6 +179,7 @@ func newTestPlannerWithDefaults() *TestPlanner { testSuiteDurations: make(map[string]map[string]api.TestSuiteDurationInfo), testFileWeights: make(map[string]int), testFileDurationSources: make(map[string]testFileDurationSource), + reportStats: newPlanningReportStats(), skippablePercentage: 0.0, reportWriter: os.Stderr, } @@ -234,13 +236,8 @@ func (tp *TestPlanner) Plan(ctx context.Context) error { return fmt.Errorf("failed to create test splits: %w", err) } - tp.planReport.Split = parallelRunnerSplit - if settings.GetReportEnabled() { - tp.planReport.DDTestSettings = settings.Get() - tp.planReport.LongSeparateRunnerSuites = tp.longSeparateRunnerSuitesReport(parallelRunners, parallelRunnerSplit) - tp.planReport.SlowestTestSuitesOverall = tp.slowestTestSuitesOverallReport(slowestTestSuitesReportLimit) - printPlanReport(tp.reportWriter, tp.planReport) + printPlanReport(tp.reportWriter, tp, parallelRunnerSplit) } tp.planLoaded = true @@ -287,7 +284,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { strictDiscovery := settings.GetStrictDiscovery() fullDiscoveryNeeded := fullTestDiscoverySupported && (isTestLevelSkipping || forceFullTestDiscovery) tp.runInfo = runmetadata.New(environment.GetCITags()) - tp.planInfo = NewPlanInfo(tags, detectedPlatform.Name(), testFramework.Name(), testSkippingLevel) + tp.planMetadata = NewPlanMetadata(tags, detectedPlatform.Name(), testFramework.Name(), testSkippingLevel) if tp.optimizationClient == nil { if tp.newOptimizationClient == nil { return fmt.Errorf("failed to create optimization client: missing client factory") @@ -307,6 +304,11 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { var fullDiscoveryErr error var fastDiscoveryErr error var tiaSkippingEnabled bool + var fullDiscoveryDuration time.Duration + var fastDiscoveryDuration time.Duration + var selectedDiscoveryMode discoveryMode + var selectedDiscoveryDuration time.Duration + var cacheResult discoveryCacheResult tp.resetDiscoveryResults() tp.testSuiteDurations = make(map[string]map[string]api.TestSuiteDurationInfo) @@ -331,7 +333,6 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } repositorySettings := tp.optimizationClient.GetSettings() - tp.planReport.DatadogSettings = newDatadogSettingsReport(repositorySettings) tiaSkippingEnabled = false if repositorySettings != nil { slog.Debug("Repository settings", "tia_enabled", repositorySettings.ItrEnabled, "tests_skipping", repositorySettings.TestsSkipping) @@ -349,14 +350,14 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } skipMatcher = tp.fetchSkippables(tiaSkippingEnabled) - tp.planReport.SkippableTestsCount = skipMatcher.Count() if tiaSkippingEnabled && skipMatcher.TIASkippablesCount() == 0 && !forceFullTestDiscovery { slog.Info("No TIA-skippable tests or suites found for this run, cancelling full test discovery") cancelDiscovery() } - if testSuiteDurations := tp.optimizationClient.GetTestSuiteDurations(); testSuiteDurations != nil && testSuiteDurations.TestSuites != nil { + testSuiteDurations := tp.optimizationClient.GetTestSuiteDurations() + if testSuiteDurations != nil && testSuiteDurations.TestSuites != nil { tp.testSuiteDurations = testSuiteDurations.TestSuites } @@ -365,6 +366,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { // Goroutine 2: Tests discovery (respects context cancellation) g.Go(func() error { + fullDiscoveryStartTime := time.Now() if !fullDiscoveryNeeded { if isSuiteLevelSkipping && !forceFullTestDiscovery { slog.Info("Suite-level skipping does not require full test discovery; using fast test file discovery fallback", "framework", testFramework.Name()) @@ -378,9 +380,12 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { return nil } - if res, ok := discoveryCache.restore(); ok { + res, restoredCacheResult := discoveryCache.restore() + cacheResult = restoredCacheResult + if restoredCacheResult.Used { discoveredTests = res fullDiscoverySucceeded = true + fullDiscoveryDuration = time.Since(fullDiscoveryStartTime) return nil } @@ -394,6 +399,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { discoveryCache.store() discoveredTests = res fullDiscoverySucceeded = true + fullDiscoveryDuration = time.Since(fullDiscoveryStartTime) return nil }) @@ -415,7 +421,8 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } } discoveredTestFiles = res - slog.Info("Discovered test files (fast)", "duration", time.Since(startTime), "count", len(discoveredTestFiles)) + fastDiscoveryDuration = time.Since(startTime) + slog.Info("Discovered test files (fast)", "duration", fastDiscoveryDuration, "count", len(discoveredTestFiles)) return nil }) @@ -431,6 +438,8 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { if err := tp.recordFullDiscoveryResults(discoveredTests, skipMatcher); err != nil { return err } + selectedDiscoveryMode = discoveryModeFull + selectedDiscoveryDuration = fullDiscoveryDuration // if we have data on which tests exist in the local repository, we will aggregate them // into a collection of testSuiteAggregate structs. // This collection is used to calculate the skippable percentage and the weighted test files. @@ -448,6 +457,8 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { if err := tp.recordFastDiscoveryFallbackFiles(discoveredTestFiles); err != nil { return err } + selectedDiscoveryMode = discoveryModeFast + selectedDiscoveryDuration = fastDiscoveryDuration tp.addDurationDataForFastDiscoveryFallback() if isSuiteLevelSkipping && tiaSkippingEnabled { tp.recordSuiteLevelSkippables(skipMatcher, testFramework) @@ -462,9 +473,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { tp.skippablePercentage = calculateSavedTimePercentage(tp.suiteAggregates) tp.testFileWeights = tp.calculateFileWeights() - tp.planReport.RunInfo = tp.runInfo - tp.planReport.PlanInfo = tp.planInfo - tp.planReport.Planning = tp.newPlanningReport() + tp.recordDiscoveryReport(selectedDiscoveryMode, cacheResult, selectedDiscoveryDuration) slog.Info("Test files prepared", "testFilesCount", len(tp.testFiles)) @@ -503,18 +512,16 @@ func (tp *TestPlanner) fetchSkippables(tiaSkippingEnabled bool) skippableMatcher skippables = tp.optimizationClient.GetSkippables() } - tp.planReport.KnownTests = newKnownTestsReport(tp.optimizationClient.GetKnownTests()) - testManagementTestsData := tp.optimizationClient.GetTestManagementTestsData() - tp.planReport.ManagedFlakyTests = newManagedFlakyTestsReport(testManagementTestsData) - disabledTests := tp.optimizationClient.GetDisabledTests() + matcher := newSkippableMatcher(skippables, disabledTests) + slog.Info("Fetched skippables", "duration", time.Since(startTime), "tiaSkippableTestsCount", len(skippables.Tests), "tiaSkippableSuitesCount", len(skippables.Suites), "disabledTestsCount", len(disabledTests)) - return newSkippableMatcher(skippables, disabledTests) + return matcher } func (tp *TestPlanner) estimateDiscoveredSuiteDurations() { @@ -632,6 +639,14 @@ func calculateSavedTimePercentage(suiteAggregates map[testSuiteKey]testSuiteAggr return (totalDuration - estimatedDuration) / totalDuration * 100.0 } +func countSuiteAggregateTests(suiteAggregates map[testSuiteKey]testSuiteAggregate) int { + count := 0 + for _, aggregate := range suiteAggregates { + count += aggregate.NumTests + } + return count +} + func indexSuitesBySourceFile(suiteAggregates map[testSuiteKey]testSuiteAggregate) map[string][]testSuiteKey { sourceFileLookup := make(map[string][]testSuiteKey) for key, aggregate := range suiteAggregates { diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 6b4281f..b0b59e2 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -354,18 +354,19 @@ func (m *longRunningDiscoveryFramework) DiscoverTests(ctx context.Context, testF // MockTestOptimizationClient mocks the test optimization client type MockTestOptimizationClient struct { - InitializeCalled bool - InitializeErr error - Settings *api.SettingsResponseData - Skippables api.Skippables - GetSkippablesCalled bool - KnownTests *api.KnownTestsResponseData - TestManagementTests *api.TestManagementTestsResponseDataModules - DisabledTests map[string]bool - Durations map[string]map[string]api.TestSuiteDurationInfo - DurationsCalled bool - ShutdownCalled bool - Tags map[string]string + InitializeCalled bool + InitializeErr error + Settings *api.SettingsResponseData + Skippables api.Skippables + GetSkippablesCalled bool + KnownTests *api.KnownTestsResponseData + TestManagementTests *api.TestManagementTestsResponseDataModules + DisabledTests map[string]bool + Durations map[string]map[string]api.TestSuiteDurationInfo + BackendRequestTimingValues testoptimization.BackendRequestTimings + DurationsCalled bool + ShutdownCalled bool + Tags map[string]string } func testSkippables(tests map[string]bool) api.Skippables { @@ -391,6 +392,9 @@ func (m *MockTestOptimizationClient) GetSettings() *api.SettingsResponseData { func (m *MockTestOptimizationClient) GetSkippables() api.Skippables { m.GetSkippablesCalled = true + if m.Settings != nil && !m.Settings.TestsSkipping { + return api.NewSkippables() + } skippables := m.Skippables if skippables.Tests == nil { skippables.Tests = api.SkippableTests{} @@ -398,6 +402,7 @@ func (m *MockTestOptimizationClient) GetSkippables() api.Skippables { if skippables.Suites == nil { skippables.Suites = api.SkippableSuites{} } + m.Skippables = skippables return skippables } @@ -448,6 +453,10 @@ func (m *MockTestOptimizationClient) GetTestSuiteDurations() *api.TestSuiteDurat return &api.TestSuiteDurationsResponseData{TestSuites: m.Durations} } +func (m *MockTestOptimizationClient) BackendRequestTimings() testoptimization.BackendRequestTimings { + return m.BackendRequestTimingValues +} + func (m *MockTestOptimizationClient) StoreCacheAndExit() { m.ShutdownCalled = true } @@ -739,11 +748,12 @@ func TestTestPlanner_Setup_WithParallelRunners(t *testing.T) { report := reportOutput.String() for _, expectedReportLine := range []string{ - "Split\n", - " Runners: 1\n", - " Expected wall time:", - " Imbalance:", - " Total estimated runtime:", + "Planning\n", + " Runner split\n", + " Estimated runtime:", + " Runners: 1", + " Expected wall time:", + " Imbalance:", } { if !strings.Contains(report, expectedReportLine) { t.Errorf("Expected report to contain %q, got report: %s", expectedReportLine, report) @@ -817,6 +827,7 @@ func TestTestPlanner_Plan_JestSuiteSkippingFetchesSkippablesWithoutFullDiscovery t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED", "false") + t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_TEST_DISCOVERY_CACHE", filepath.Join(tempDir, "test-discovery-cache.json")) settings.Init() mockFramework := &MockFramework{ @@ -858,6 +869,30 @@ func TestTestPlanner_Plan_JestSuiteSkippingFetchesSkippablesWithoutFullDiscovery if !mockOptimizationClient.GetSkippablesCalled { t.Fatal("expected planner to fetch suite skippables when full test discovery is unsupported") } + report := runner.newPlanReportData(splitScore{}) + if report.Skippables.TestSkippingLevel != settings.TestSkippingLevelSuite || + report.Skippables.TIATests != 0 || + report.Skippables.TIASuites != 1 { + t.Errorf("expected suite-level skippables to be reported separately, got %+v", report.Skippables) + } + if report.Planning.Discovery.Mode != discoveryModeFast || + report.Planning.Discovery.TestFiles != 2 || + report.Planning.Discovery.Suites != 0 { + t.Errorf("expected fast discovery report with files but no local suites, got %+v", report.Planning.Discovery) + } + if cacheResult := report.Planning.Discovery.Cache; cacheResult != (discoveryCacheResult{}) { + t.Errorf("expected cache result to be empty when full discovery is not applicable, got %+v", cacheResult) + } + if report.Planning.Durations.BackendDurationsApplied != 2 || + report.Planning.Durations.BackendSuitesAdded != 2 || + report.Planning.Durations.SuitesWithoutDurations != 0 { + t.Errorf("expected backend duration application report for fast discovery, got %+v", report.Planning.Durations) + } + if report.Planning.Skipping.TIASuites != 1 || + report.Planning.Skipping.FullySkippedFiles != 1 || + report.Planning.TestFilesToRun != 1 { + t.Errorf("expected suite skip application report, got planning=%+v", report.Planning) + } expectedTestFiles := "src/b.test.ts\n" assertFileContent(t, constants.TestFilesOutputPath, expectedTestFiles) @@ -930,8 +965,8 @@ func TestTestPlanner_PreparePlanningData_RubySuiteModeSkipsFullDiscoveryAndSkips if _, ok := runner.testFileWeights["spec/models/payment_spec.rb"]; !ok { t.Fatalf("expected runnable suite file to remain, got weights: %+v", runner.testFileWeights) } - if runner.planInfo.TestSkippingLevel != settings.TestSkippingLevelSuite.String() { - t.Fatalf("plan test skipping level = %q, want suite", runner.planInfo.TestSkippingLevel) + if runner.planMetadata.TestSkippingLevel != settings.TestSkippingLevelSuite.String() { + t.Fatalf("plan test skipping level = %q, want suite", runner.planMetadata.TestSkippingLevel) } } @@ -1013,6 +1048,17 @@ func TestTestPlanner_PreparePlanningData_RubySuiteModeForceFullDiscovery(t *test if guarded.NumTests != 1 || guarded.NumTestsSkipped != 0 { t.Fatalf("expected guarded suite to remain runnable, got %+v", guarded) } + report := runner.newPlanReportData(splitScore{}) + if report.Planning.Discovery.Mode != discoveryModeFull || + report.Planning.Discovery.Suites != 4 || + report.Planning.Discovery.Tests != 5 { + t.Errorf("expected full discovery suite/test report, got %+v", report.Planning.Discovery) + } + if report.Planning.Skipping.TIASuites != 3 || + report.Planning.Skipping.UnskippableMarkerSuitesForced != 1 || + report.Planning.Skipping.FullySkippedFiles != 1 { + t.Errorf("expected full discovery skipping application report, got %+v", report.Planning.Skipping) + } } func TestTestPlanner_PreparePlanningData_ForceFullDiscoveryKeepsRunningWithNoTIASkippables(t *testing.T) { @@ -1976,6 +2022,12 @@ func TestTestPlanner_PreparePlanningData_Success(t *testing.T) { if !mockOptimizationClient.DurationsCalled { t.Error("PreparePlanningData() should fetch test suite durations") } + report := runner.newPlanReportData(splitScore{}) + if !report.TestSuiteDurations.Available || + report.TestSuiteDurations.Modules != 1 || + report.TestSuiteDurations.Suites != 1 { + t.Errorf("Expected plan report to summarize fetched durations, got %+v", report.TestSuiteDurations) + } } func TestTestPlanner_PreparePlanningData_DisabledTestManagementTestsAreSkipped(t *testing.T) { @@ -2062,8 +2114,10 @@ func TestTestPlanner_PreparePlanningData_DisabledTestManagementTestsAreSkipped(t t.Errorf("Expected attempt-to-fix file to be scheduled, got %v", runner.testFileWeights) } - if runner.planReport.SkippableTestsCount != 2 { - t.Errorf("Expected planner skip set to include TIA-skippable and disabled tests, got %d", runner.planReport.SkippableTestsCount) + report := runner.newPlanReportData(splitScore{}) + if report.Skippables.TIATests != 1 || report.Skippables.TIASuites != 0 || report.Planning.Skipping.DisabledTests != 1 { + t.Errorf("Expected planner report to split TIA skippables and disabled tests, got skippables=%+v planning=%+v", + report.Skippables, report.Planning.Skipping) } } @@ -2254,8 +2308,10 @@ func TestTestPlanner_PreparePlanningData_TestManagementDoesNotKeepFullDiscoveryW t.Errorf("Expected fast discovery fallback to keep all discovered files, got %v", runner.testFileWeights) } - if runner.planReport.SkippableTestsCount != 2 { - t.Errorf("Expected planner skip set to include fetched disabled tests for reporting, got %d", runner.planReport.SkippableTestsCount) + report := runner.newPlanReportData(splitScore{}) + if report.Skippables.TIATests != 0 || report.Skippables.TIASuites != 0 || report.Planning.Skipping.DisabledTests != 0 { + t.Errorf("Expected planner report to split TIA skippables and disabled tests, got skippables=%+v planning=%+v", + report.Skippables, report.Planning.Skipping) } } @@ -2444,6 +2500,12 @@ func TestTestPlanner_PreparePlanningData_EmptyDurationsContinues(t *testing.T) { if len(runner.testSuiteDurations) != 0 { t.Errorf("Expected empty in-memory test suite durations on empty response, got %v", runner.testSuiteDurations) } + report := runner.newPlanReportData(splitScore{}) + if !report.TestSuiteDurations.Available || + report.TestSuiteDurations.Modules != 0 || + report.TestSuiteDurations.Suites != 0 { + t.Errorf("Expected plan report to summarize empty durations response, got %+v", report.TestSuiteDurations) + } } func TestTestPlanner_PreparePlanningData_NonEmptyDurationsUsesP50ForMatchingSuites(t *testing.T) { @@ -2493,6 +2555,12 @@ func TestTestPlanner_PreparePlanningData_NonEmptyDurationsUsesP50ForMatchingSuit if len(runner.testSuiteDurations) != 1 { t.Fatalf("Expected stored durations data, got %v", runner.testSuiteDurations) } + report := runner.newPlanReportData(splitScore{}) + if !report.TestSuiteDurations.Available || + report.TestSuiteDurations.Modules != 1 || + report.TestSuiteDurations.Suites != 2 { + t.Errorf("Expected plan report to summarize fetched durations, got %+v", report.TestSuiteDurations) + } if _, ok := runner.testFiles["spec/file1_test.rb"]; !ok { t.Error("Expected file1 in test files") diff --git a/internal/planner/report.go b/internal/planner/report.go index 9ff9c93..a521cda 100644 --- a/internal/planner/report.go +++ b/internal/planner/report.go @@ -3,6 +3,7 @@ package planner import ( "fmt" "io" + "reflect" "slices" "strconv" "strings" @@ -13,10 +14,14 @@ import ( "github.com/DataDog/ddtest/internal/settings" ) -func printPlanReport(w io.Writer, report planReport) { +func printPlanReport(w io.Writer, tp *TestPlanner, split splitScore) { + printPlanReportData(w, tp.newPlanReportData(split)) +} + +func printPlanReportData(w io.Writer, report PlanReportData) { reportFprintln(w, "+++ DDTest: plan report") reportFprintln(w) - printRunInfoReport(w, report.RunInfo, report.PlanInfo) + printRunInfoReport(w, report.RunInfo, report.PlanMetadata) reportFprintln(w) printDDTestSettingsReport(w, report.DDTestSettings) reportFprintln(w) @@ -24,24 +29,22 @@ func printPlanReport(w io.Writer, report planReport) { reportFprintln(w) printBackendDataReport(w, report) reportFprintln(w) - printPlanningReport(w, report.Planning) - reportFprintln(w) - printSplitReport(w, report.Split) + printPlanningReport(w, report) reportFprintln(w) printLongSeparateRunnerSuitesReport(w, report.LongSeparateRunnerSuites) reportFprintln(w) printSlowestTestSuitesOverallReport(w, report.SlowestTestSuitesOverall) } -func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planInfo PlanInfo) { +func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planMetadata PlanMetadata) { reportFprintln(w, "Run") reportFprintf(w, " Service: %s\n", valueOrNotAvailable(runInfo.Service)) reportFprintf(w, " Repository: %s\n", valueOrNotAvailable(runInfo.Repository)) reportFprintf(w, " Commit: %s\n", valueOrNotAvailable(runInfo.Commit)) reportFprintf(w, " Branch: %s\n", valueOrNotAvailable(runInfo.Branch)) - reportFprintf(w, " Platform: %s\n", formatPlatform(planInfo.Platform, planInfo.Framework)) - reportFprintf(w, " OS tags: %s\n", formatTagList(planInfo.OSTags, constants.OSPlatform, constants.OSArchitecture, constants.OSVersion)) - reportFprintf(w, " Runtime tags: %s\n", formatTagList(planInfo.RuntimeTags, constants.RuntimeName, constants.RuntimeVersion)) + reportFprintf(w, " Platform: %s\n", formatPlatform(planMetadata.Platform, planMetadata.Framework)) + reportFprintf(w, " OS tags: %s\n", formatTagList(planMetadata.OSTags, constants.OSPlatform, constants.OSArchitecture, constants.OSVersion)) + reportFprintf(w, " Runtime tags: %s\n", formatTagList(planMetadata.RuntimeTags, constants.RuntimeName, constants.RuntimeVersion)) } func printDDTestSettingsReport(w io.Writer, config *settings.Config) { @@ -51,27 +54,106 @@ func printDDTestSettingsReport(w io.Writer, config *settings.Config) { return } - reportFprintf(w, " Platform: %s\n", valueOrNotSet(config.Platform)) - reportFprintf(w, " Framework: %s\n", valueOrNotSet(config.Framework)) - reportFprintf(w, " Min parallelism: %s\n", formatCount(config.MinParallelism)) - reportFprintf(w, " Max parallelism: %s\n", formatCount(config.MaxParallelism)) - reportFprintf(w, " CI job overhead: %s\n", formatDuration(config.ParallelRunnerOverhead)) - reportFprintf(w, " Worker env: %s\n", formatWorkerEnvKeys(config.WorkerEnv)) - reportFprintf(w, " CI node: %s\n", formatCount(config.CiNode)) - reportFprintf(w, " CI node workers: %s\n", formatCount(config.CiNodeWorkers)) - reportFprintf(w, " Command: %s\n", valueOrNotSet(config.Command)) - reportFprintf(w, " Tests location: %s\n", valueOrNotSet(config.TestsLocation)) - reportFprintf(w, " Runtime tags: %s\n", valueOrNotSet(config.RuntimeTags)) - reportFprintf(w, " Report enabled: %s\n", strconv.FormatBool(config.ReportEnabled)) + if !printChangedDDTestSettings(w, config) { + reportFprintln(w, " Settings: defaults") + } +} + +func printChangedDDTestSettings(w io.Writer, config *settings.Config) bool { + defaults := defaultDDTestSettings() + configValue := reflect.ValueOf(*config) + defaultsValue := reflect.ValueOf(defaults) + configType := configValue.Type() + printed := false + + for i := range configValue.NumField() { + value := configValue.Field(i) + if reflect.DeepEqual(value.Interface(), defaultsValue.Field(i).Interface()) { + continue + } + + field := configType.Field(i) + reportFprintf(w, " %s: %s\n", formatDDTestSettingName(field), formatDDTestSettingValue(field, value)) + printed = true + } + + return printed +} + +func formatDDTestSettingName(field reflect.StructField) string { + key := configFieldKey(field) + if key == "" { + return field.Name + } + if key == "parallel_runner_overhead" { + return "CI job overhead" + } + + words := strings.Split(key, "_") + for i, word := range words { + if word == "ci" { + words[i] = "CI" + continue + } + if i == 0 { + words[i] = strings.ToUpper(word[:1]) + word[1:] + } + } + return strings.Join(words, " ") +} + +func formatDDTestSettingValue(field reflect.StructField, value reflect.Value) string { + if configFieldKey(field) == "worker_env" { + return formatWorkerEnvKeys(value.String()) + } + + switch value.Type() { + case reflect.TypeOf(time.Duration(0)): + return formatDuration(time.Duration(value.Int())) + case reflect.TypeOf(settings.TestSkippingLevel("")): + return valueOrNotSet(value.Interface().(settings.TestSkippingLevel).String()) + } + + switch value.Kind() { + case reflect.String: + return valueOrNotSet(value.String()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return formatCount(int(value.Int())) + case reflect.Bool: + return strconv.FormatBool(value.Bool()) + default: + return fmt.Sprint(value.Interface()) + } +} + +func configFieldKey(field reflect.StructField) string { + key, _, _ := strings.Cut(field.Tag.Get("mapstructure"), ",") + return key +} + +func defaultDDTestSettings() settings.Config { + return settings.Config{ + Platform: "ruby", + Framework: "rspec", + MinParallelism: settings.DefaultParallelism(), + MaxParallelism: settings.DefaultParallelism(), + ParallelRunnerOverhead: settings.DefaultParallelRunnerOverhead(), + CiNode: -1, + CiNodeWorkers: 1, + TestSkippingLevel: settings.TestSkippingLevelTest, + ReportEnabled: true, + } } func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { reportFprintln(w, "Datadog settings") if !report.Available { reportFprintln(w, " Settings: not available") + printFetchDuration(w, report.FetchDuration) return } + printFetchDuration(w, report.FetchDuration) reportFprintf(w, " Test Impact Analysis: %s\n", enabledWord(report.TestImpactAnalysis)) reportFprintf(w, " Test skipping: %s\n", enabledWord(report.TestSkipping)) reportFprintf(w, " Test impact collection: %s\n", enabledWord(report.TestImpactCollection)) @@ -81,30 +163,21 @@ func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { reportFprintf(w, " Flaky test management: %s\n", enabledWord(report.FlakyTestManagement)) } -func printBackendDataReport(w io.Writer, report planReport) { +func printBackendDataReport(w io.Writer, report PlanReportData) { reportFprintln(w, "Backend data") - reportFprintf(w, " Known tests: %s\n", formatKnownTests(report.DatadogSettings, report.KnownTests)) - reportFprintf(w, " Skippable tests for this run: %s\n", formatSkippableTests(report.DatadogSettings, report.SkippableTestsCount)) - reportFprintf(w, " Managed flaky tests: %s\n", formatManagedFlakyTests(report.DatadogSettings, report.ManagedFlakyTests)) + reportFprintf(w, " Known tests: %s\n", formatBackendDataValue(formatKnownTests(report.DatadogSettings, report.KnownTests), report.KnownTests.FetchDuration)) + reportFprintf(w, " TIA skippables returned: %s\n", formatBackendDataValue(formatTIASkippables(report.DatadogSettings, report.Skippables), report.Skippables.FetchDuration)) + reportFprintf(w, " Managed flaky tests: %s\n", formatBackendDataValue(formatManagedFlakyTests(report.DatadogSettings, report.ManagedFlakyTests), report.ManagedFlakyTests.FetchDuration)) + reportFprintf(w, " Test suite durations: %s\n", formatBackendDataValue(formatTestSuiteDurations(report.TestSuiteDurations), report.TestSuiteDurations.FetchDuration)) } -func printPlanningReport(w io.Writer, report planningReport) { +func printPlanningReport(w io.Writer, report PlanReportData) { reportFprintln(w, "Planning") - reportFprintf(w, " Test files discovered: %s\n", formatCount(report.TestFilesDiscovered)) - reportFprintf(w, " Fully skipped files: %s\n", formatCount(report.FullySkippedFiles)) - reportFprintf(w, " Test files to run: %s\n", formatCount(report.TestFilesToRun)) - reportFprintf(w, " Duration source: %s known, %s default\n", - formatCount(report.DurationSources.Known), - formatCount(report.DurationSources.Default)) - reportFprintf(w, " Estimated time saved: %.2f%%\n", report.EstimatedTimeSaved) -} - -func printSplitReport(w io.Writer, report splitScore) { - reportFprintln(w, "Split") - reportFprintf(w, " Runners: %s\n", formatCount(report.parallelRunners)) - reportFprintf(w, " Expected wall time: %s\n", formatDuration(report.wallTimeDuration())) - reportFprintf(w, " Imbalance: %s\n", formatDuration(report.imbalanceDuration())) - reportFprintf(w, " Total estimated runtime: %s\n", formatDuration(report.totalRuntimeDuration())) + printDiscoveryPlanningReport(w, report.Planning.Discovery) + printDurationEstimatesPlanningReport(w, report.Planning.Durations) + printSkippingPlanningReport(w, report.DatadogSettings, report.Skippables, report.ManagedFlakyTests, report.Planning.Skipping) + printRunSetPlanningReport(w, report.Planning) + printRunnerSplitPlanningReport(w, report.Planning, report.Split) } func printLongSeparateRunnerSuitesReport(w io.Writer, suites []testSuiteTimingReport) { @@ -211,6 +284,152 @@ func reportFprintf(w io.Writer, format string, args ...any) { _, _ = fmt.Fprintf(w, format, args...) } +func printDiscoveryPlanningReport(w io.Writer, discovery discoveryReport) { + if !discovery.Available { + reportFprintln(w, " Discovery: not available") + return + } + + reportFprintln(w, " Discovery") + reportFprintf(w, " Method: %s\n", valueOrNotAvailable(string(discovery.Mode))) + reportFprintf(w, " Test files: %s\n", formatCount(discovery.TestFiles)) + if discovery.Cache.Configured || discovery.Cache.Used || discovery.Cache.NotUsedReason != "" { + reportFprintf(w, " Cache: %s\n", formatDiscoveryCache(discovery.Cache)) + } + reportFprintf(w, " Duration: %s\n", formatOptionalDuration(discovery.Duration)) + switch discovery.Mode { + case discoveryModeFull: + reportFprintf(w, " Suites discovered: %s\n", formatCount(discovery.Suites)) + reportFprintf(w, " Tests discovered: %s\n", formatCount(discovery.Tests)) + } +} + +func formatDiscoveryCache(cache discoveryCacheResult) string { + if cache.Used { + return "used" + } + if !cache.Configured { + return "not configured" + } + if cache.NotUsedReason == "" { + return "not used" + } + return "not used (" + cache.NotUsedReason + ")" +} + +func printFetchDuration(w io.Writer, duration time.Duration) { + if duration > 0 { + reportFprintf(w, " Fetch duration: %s\n", formatDuration(duration)) + } +} + +func formatBackendDataValue(value string, duration time.Duration) string { + if duration <= 0 { + return value + } + return fmt.Sprintf("%s (fetched in %s)", value, formatDuration(duration)) +} + +func formatOptionalDuration(duration time.Duration) string { + if duration <= 0 { + return "not available" + } + return formatDuration(duration) +} + +func printDurationEstimatesPlanningReport(w io.Writer, durations durationApplicationReport) { + if !durations.Available { + reportFprintln(w, " Duration estimates: not available") + return + } + + reportFprintln(w, " Duration estimates") + reportFprintf(w, " Backend durations used: %s\n", formatCountWithUnit(durations.BackendDurationsApplied, "suite", "suites")) + reportFprintf(w, " Default durations used: %s\n", formatCountWithUnit(durations.SuitesWithoutDurations, "suite", "suites")) + reportFprintf(w, " Backend-only suites added: %s\n", formatCount(durations.BackendSuitesAdded)) +} + +func printSkippingPlanningReport( + w io.Writer, + datadogSettings datadogSettingsReport, + skippables skippablesReport, + managed managedFlakyTestsReport, + skipping skippingApplicationReport, +) { + if !skipping.Available { + reportFprintln(w, " Skipping: not available") + return + } + + reportFprintln(w, " Skipping") + reportFprintf(w, " TIA skippables applied: %s\n", formatAppliedTIASkippables(datadogSettings, skippables, skipping)) + reportFprintf(w, " Disabled tests applied: %s\n", formatAppliedDisabledTests(datadogSettings, managed, skipping)) + reportFprintf(w, " Suites marked unskippable: %s\n", formatCount(skipping.UnskippableMarkerSuitesForced)) + reportFprintf(w, " Files fully skipped: %s\n", formatCount(skipping.FullySkippedFiles)) +} + +func printRunSetPlanningReport(w io.Writer, planning planningReport) { + if !planning.Discovery.Available { + reportFprintln(w, " Run set: not available") + return + } + + reportFprintln(w, " Run set") + reportFprintf(w, " Test files to run: %s\n", formatCount(planning.TestFilesToRun)) + reportFprintf(w, " Estimated time saved: %.2f%%\n", planning.EstimatedTimeSaved) +} + +func printRunnerSplitPlanningReport(w io.Writer, planning planningReport, split splitScore) { + if split.parallelRunners <= 0 { + reportFprintln(w, " Runner split: not available") + return + } + + fullDuration := "not available" + if planning.Durations.ExpectedFullDuration > 0 { + fullDuration = formatDuration(planning.Durations.ExpectedFullDuration) + } + + reportFprintln(w, " Runner split") + reportFprintf(w, " Full runtime: %s\n", fullDuration) + reportFprintf(w, " Estimated runtime: %s\n", formatDuration(split.totalRuntimeDuration())) + reportFprintf(w, " Runners: %s\n", formatCount(split.parallelRunners)) + reportFprintf(w, " Expected wall time: %s\n", formatDuration(split.wallTimeDuration())) + reportFprintf(w, " Imbalance: %s\n", formatDuration(split.imbalanceDuration())) +} + +func formatAppliedTIASkippables(datadogSettings datadogSettingsReport, skippables skippablesReport, skipping skippingApplicationReport) string { + if datadogSettings.Available && !datadogSettings.TestSkipping { + return "disabled" + } + if !skippables.Available { + return "not available" + } + + switch skippables.TestSkippingLevel { + case settings.TestSkippingLevelTest: + return formatCountWithUnit(skipping.TIATests, "test", "tests") + case settings.TestSkippingLevelSuite: + return formatCountWithUnit(skipping.TIASuites, "suite", "suites") + case "": + return "mode not available" + default: + return fmt.Sprintf("%s, %s", + formatCountWithUnit(skipping.TIATests, "test", "tests"), + formatCountWithUnit(skipping.TIASuites, "suite", "suites")) + } +} + +func formatAppliedDisabledTests(datadogSettings datadogSettingsReport, managed managedFlakyTestsReport, skipping skippingApplicationReport) string { + if datadogSettings.Available && !datadogSettings.FlakyTestManagement { + return "disabled" + } + if !managed.Available { + return "not available" + } + return formatCountWithUnit(skipping.DisabledTests, "test", "tests") +} + func formatKnownTests(settings datadogSettingsReport, known knownTestsReport) string { if settings.Available && !settings.KnownTests { return "disabled" @@ -224,11 +443,25 @@ func formatKnownTests(settings datadogSettingsReport, known knownTestsReport) st formatCount(known.Tests)) } -func formatSkippableTests(settings datadogSettingsReport, count int) string { - if settings.Available && !settings.TestSkipping { +func formatTIASkippables(datadogSettings datadogSettingsReport, skippables skippablesReport) string { + if datadogSettings.Available && !datadogSettings.TestSkipping { return "disabled" } - return formatCount(count) + if !skippables.Available { + return "not available" + } + switch skippables.TestSkippingLevel { + case settings.TestSkippingLevelTest: + return formatCountWithUnit(skippables.TIATests, "test", "tests") + case settings.TestSkippingLevelSuite: + return formatCountWithUnit(skippables.TIASuites, "suite", "suites") + case "": + return "skipping mode not available" + default: + return fmt.Sprintf("%s, %s", + formatCountWithUnit(skippables.TIATests, "test", "tests"), + formatCountWithUnit(skippables.TIASuites, "suite", "suites")) + } } func formatManagedFlakyTests(settings datadogSettingsReport, managed managedFlakyTestsReport) string { @@ -245,6 +478,15 @@ func formatManagedFlakyTests(settings datadogSettingsReport, managed managedFlak formatCount(managed.AttemptToFix)) } +func formatTestSuiteDurations(durations testSuiteDurationsReport) string { + if !durations.Available { + return "not available" + } + return fmt.Sprintf("%s modules, %s suites", + formatCount(durations.Modules), + formatCount(durations.Suites)) +} + func formatTagList(tags map[string]string, keys ...string) string { parts := make([]string, 0, len(keys)) for _, key := range keys { @@ -319,6 +561,14 @@ func formatCount(count int) string { return builder.String() } +func formatCountWithUnit(count int, singular string, plural string) string { + unit := plural + if count == 1 { + unit = singular + } + return formatCount(count) + " " + unit +} + func formatDuration(duration time.Duration) string { if duration < time.Millisecond { return duration.String() diff --git a/internal/planner/report_data.go b/internal/planner/report_data.go index ad3f948..20d05b6 100644 --- a/internal/planner/report_data.go +++ b/internal/planner/report_data.go @@ -4,6 +4,7 @@ import ( "slices" "time" + "github.com/DataDog/ddtest/internal/constants" "github.com/DataDog/ddtest/internal/runmetadata" "github.com/DataDog/ddtest/internal/settings" "github.com/DataDog/ddtest/internal/testoptimization/api" @@ -11,6 +12,7 @@ import ( type datadogSettingsReport struct { Available bool + FetchDuration time.Duration TestImpactAnalysis bool TestSkipping bool TestImpactCollection bool @@ -20,12 +22,35 @@ type datadogSettingsReport struct { FlakyTestManagement bool } -func newDatadogSettingsReport(settings *api.SettingsResponseData) datadogSettingsReport { +func addBackendDataReports(report PlanReportData, client testOptimizationClient) PlanReportData { + timings := client.BackendRequestTimings() + + report.DatadogSettings = reportDatadogSettings(client.GetSettings(), timings.Settings) + report.KnownTests = reportKnownTests(client.GetKnownTests(), timings.KnownTests) + report.Skippables = reportSkippables( + client.GetSkippables(), + settings.TestSkippingLevel(report.PlanMetadata.TestSkippingLevel), + timings.Skippables, + ) + report.ManagedFlakyTests = reportManagedFlakyTests( + client.GetTestManagementTestsData(), + timings.TestManagementTests, + ) + report.TestSuiteDurations = reportTestSuiteDurations( + client.GetTestSuiteDurations(), + timings.TestSuiteDurations, + ) + + return report +} + +func reportDatadogSettings(settings *api.SettingsResponseData, fetchDuration time.Duration) datadogSettingsReport { if settings == nil { - return datadogSettingsReport{} + return datadogSettingsReport{FetchDuration: fetchDuration} } return datadogSettingsReport{ Available: true, + FetchDuration: fetchDuration, TestImpactAnalysis: settings.ItrEnabled, TestSkipping: settings.TestsSkipping, TestImpactCollection: settings.CodeCoverage, @@ -37,20 +62,22 @@ func newDatadogSettingsReport(settings *api.SettingsResponseData) datadogSetting } type knownTestsReport struct { - Available bool - Modules int - Suites int - Tests int + Available bool + FetchDuration time.Duration + Modules int + Suites int + Tests int } -func newKnownTestsReport(knownTests *api.KnownTestsResponseData) knownTestsReport { +func reportKnownTests(knownTests *api.KnownTestsResponseData, fetchDuration time.Duration) knownTestsReport { if knownTests == nil { - return knownTestsReport{} + return knownTestsReport{FetchDuration: fetchDuration} } report := knownTestsReport{ - Available: true, - Modules: len(knownTests.Tests), + Available: true, + FetchDuration: fetchDuration, + Modules: len(knownTests.Tests), } for _, suites := range knownTests.Tests { report.Suites += len(suites) @@ -62,19 +89,23 @@ func newKnownTestsReport(knownTests *api.KnownTestsResponseData) knownTestsRepor } type managedFlakyTestsReport struct { - Available bool - Total int - Quarantined int - Disabled int - AttemptToFix int + Available bool + FetchDuration time.Duration + Total int + Quarantined int + Disabled int + AttemptToFix int } -func newManagedFlakyTestsReport(testManagementTests *api.TestManagementTestsResponseDataModules) managedFlakyTestsReport { +func reportManagedFlakyTests( + testManagementTests *api.TestManagementTestsResponseDataModules, + fetchDuration time.Duration, +) managedFlakyTestsReport { if testManagementTests == nil { - return managedFlakyTestsReport{} + return managedFlakyTestsReport{FetchDuration: fetchDuration} } - report := managedFlakyTestsReport{Available: true} + report := managedFlakyTestsReport{Available: true, FetchDuration: fetchDuration} for _, suites := range testManagementTests.Modules { for _, tests := range suites.Suites { for _, test := range tests.Tests { @@ -94,17 +125,54 @@ func newManagedFlakyTestsReport(testManagementTests *api.TestManagementTestsResp return report } -type durationSourceReport struct { - Known int - Default int +type skippablesReport struct { + Available bool + FetchDuration time.Duration + TestSkippingLevel settings.TestSkippingLevel + TIATests int + TIASuites int } -type planningReport struct { - TestFilesDiscovered int - FullySkippedFiles int - TestFilesToRun int - DurationSources durationSourceReport - EstimatedTimeSaved float64 +func reportSkippables( + skippables api.Skippables, + testSkippingLevel settings.TestSkippingLevel, + fetchDuration time.Duration, +) skippablesReport { + return skippablesReport{ + Available: skippables.Tests != nil || + skippables.Suites != nil || + fetchDuration > 0, + FetchDuration: fetchDuration, + TestSkippingLevel: testSkippingLevel, + TIATests: len(skippables.Tests), + TIASuites: len(skippables.Suites), + } +} + +type testSuiteDurationsReport struct { + Available bool + FetchDuration time.Duration + Modules int + Suites int +} + +func reportTestSuiteDurations( + testSuiteDurations *api.TestSuiteDurationsResponseData, + fetchDuration time.Duration, +) testSuiteDurationsReport { + if testSuiteDurations == nil { + return testSuiteDurationsReport{FetchDuration: fetchDuration} + } + + report := testSuiteDurationsReport{ + Available: true, + FetchDuration: fetchDuration, + Modules: len(testSuiteDurations.TestSuites), + } + for _, suites := range testSuiteDurations.TestSuites { + report.Suites += len(suites) + } + return report } type testSuiteTimingReport struct { @@ -117,46 +185,193 @@ type testSuiteTimingReport struct { DurationSource testFileDurationSource } -type planReport struct { +type discoveryMode string + +const ( + discoveryModeFast discoveryMode = "fast" + discoveryModeFull discoveryMode = "full" +) + +type discoveryReport struct { + Available bool + Mode discoveryMode + Cache discoveryCacheResult + Duration time.Duration + TestFiles int + Suites int + Tests int +} + +type durationApplicationReport struct { + Available bool + BackendDurationsApplied int + BackendSuitesAdded int + SuitesWithoutDurations int + FilesWithoutDurations int + ExpectedFullDuration time.Duration +} + +type skippingApplicationReport struct { + Available bool + TIATests int + TIASuites int + DisabledTests int + UnskippableMarkerSuitesForced int + FullySkippedFiles int +} + +type planningReport struct { + Discovery discoveryReport + Durations durationApplicationReport + Skipping skippingApplicationReport + TestFilesToRun int + EstimatedTimeSaved float64 +} + +type PlanReportData struct { RunInfo runmetadata.RunInfo - PlanInfo PlanInfo + PlanMetadata PlanMetadata DDTestSettings *settings.Config DatadogSettings datadogSettingsReport KnownTests knownTestsReport - SkippableTestsCount int + Skippables skippablesReport ManagedFlakyTests managedFlakyTestsReport + TestSuiteDurations testSuiteDurationsReport Planning planningReport LongSeparateRunnerSuites []testSuiteTimingReport SlowestTestSuitesOverall []testSuiteTimingReport Split splitScore } +type planningReportStats struct { + discoveryMode discoveryMode + discoveryCache discoveryCacheResult + discoveryDuration time.Duration + tiaSkippableTestsApplied int + uniqueTIASkippableSuitesApplied map[testSuiteKey]struct{} + disabledTestsApplied int + unskippableMarkerSuitesForced int +} + +func newPlanningReportStats() planningReportStats { + return planningReportStats{ + uniqueTIASkippableSuitesApplied: make(map[testSuiteKey]struct{}), + } +} + +func (tp *TestPlanner) newPlanReportData(split splitScore) PlanReportData { + report := PlanReportData{ + RunInfo: tp.runInfo, + PlanMetadata: tp.planMetadata, + DDTestSettings: settings.Get(), + Planning: tp.newPlanningReport(), + LongSeparateRunnerSuites: tp.longSeparateRunnerSuitesReport(split.parallelRunners, split), + SlowestTestSuitesOverall: tp.slowestTestSuitesOverallReport(slowestTestSuitesReportLimit), + Split: split, + } + return addBackendDataReports(report, tp.optimizationClient) +} + +func (tp *TestPlanner) recordDiscoveryReport(mode discoveryMode, cache discoveryCacheResult, duration time.Duration) { + tp.reportStats.discoveryMode = mode + tp.reportStats.discoveryCache = cache + tp.reportStats.discoveryDuration = duration +} + func (tp *TestPlanner) newPlanningReport() planningReport { fullySkippedFiles := len(tp.testFiles) - len(tp.testFileWeights) if fullySkippedFiles < 0 { fullySkippedFiles = 0 } + discoveredSuites := 0 + discoveredTests := 0 + mode := tp.reportStats.discoveryMode + if mode == discoveryModeFull { + discoveredSuites = len(tp.suiteAggregates) + discoveredTests = countSuiteAggregateTests(tp.suiteAggregates) + } return planningReport{ - TestFilesDiscovered: len(tp.testFiles), - FullySkippedFiles: fullySkippedFiles, - TestFilesToRun: len(tp.testFileWeights), - DurationSources: tp.durationSourceReport(), - EstimatedTimeSaved: tp.skippablePercentage, + Discovery: discoveryReport{ + Available: true, + Mode: mode, + Cache: tp.reportStats.discoveryCache, + Duration: tp.reportStats.discoveryDuration, + TestFiles: len(tp.testFiles), + Suites: discoveredSuites, + Tests: discoveredTests, + }, + Durations: durationApplicationReport{ + Available: true, + BackendDurationsApplied: tp.backendDurationApplicationsCount(), + BackendSuitesAdded: tp.backendSuitesAddedCount(mode), + SuitesWithoutDurations: tp.suitesWithoutBackendDurationsCount(), + FilesWithoutDurations: tp.filesWithoutBackendDurationsCount(), + ExpectedFullDuration: tp.expectedFullDuration(), + }, + Skipping: skippingApplicationReport{ + Available: true, + TIATests: tp.reportStats.tiaSkippableTestsApplied, + TIASuites: len(tp.reportStats.uniqueTIASkippableSuitesApplied), + DisabledTests: tp.reportStats.disabledTestsApplied, + UnskippableMarkerSuitesForced: tp.reportStats.unskippableMarkerSuitesForced, + FullySkippedFiles: fullySkippedFiles, + }, + TestFilesToRun: len(tp.testFileWeights), + EstimatedTimeSaved: tp.skippablePercentage, + } +} + +func (tp *TestPlanner) backendDurationApplicationsCount() int { + count := 0 + for _, aggregate := range tp.suiteAggregates { + if aggregate.DurationSource == testFileDurationSourceKnown { + count++ + } } + return count } -func (tp *TestPlanner) durationSourceReport() durationSourceReport { - var report durationSourceReport +func (tp *TestPlanner) backendSuitesAddedCount(mode discoveryMode) int { + if mode == discoveryModeFull { + return 0 + } + return len(tp.suiteAggregates) +} + +func (tp *TestPlanner) suitesWithoutBackendDurationsCount() int { + count := 0 + for _, aggregate := range tp.suiteAggregates { + if aggregate.DurationSource != testFileDurationSourceKnown { + count++ + } + } + return count +} + +func (tp *TestPlanner) filesWithoutBackendDurationsCount() int { + count := 0 for _, source := range tp.testFileDurationSources { - switch source { - case testFileDurationSourceKnown: - report.Known++ - default: - report.Default++ + if source != testFileDurationSourceKnown { + count++ } } - return report + return count +} + +func (tp *TestPlanner) expectedFullDuration() time.Duration { + var total float64 + for testFile := range tp.testFiles { + suiteKeys := tp.suitesBySourceFile[testFile] + if len(suiteKeys) == 0 { + total += float64(time.Duration(constants.DefaultTestFileWeight) * time.Millisecond) + continue + } + for _, key := range suiteKeys { + total += tp.suiteAggregates[key].TotalDuration + } + } + return durationFromNanoseconds(total) } func (tp *TestPlanner) longSeparateRunnerSuitesReport(parallelRunners int, split splitScore) []testSuiteTimingReport { diff --git a/internal/planner/report_test.go b/internal/planner/report_test.go index a2c0e79..654085d 100644 --- a/internal/planner/report_test.go +++ b/internal/planner/report_test.go @@ -1,26 +1,31 @@ package planner import ( + "fmt" + "reflect" "strings" "testing" "time" "github.com/DataDog/ddtest/internal/runmetadata" "github.com/DataDog/ddtest/internal/settings" + "github.com/DataDog/ddtest/internal/testoptimization" "github.com/DataDog/ddtest/internal/testoptimization/api" ) func TestPrintPlanReport_AllData(t *testing.T) { var output strings.Builder + minParallelism := settings.DefaultParallelism() + 1 + maxParallelism := settings.DefaultParallelism() + 2 - printPlanReport(&output, planReport{ + printPlanReportData(&output, PlanReportData{ RunInfo: runmetadata.RunInfo{ Service: "checkout-api", Repository: "https://github.com/acme/checkout.git", Commit: "9f3a1c7d2b4e", Branch: "feature/split-report", }, - PlanInfo: PlanInfo{ + PlanMetadata: PlanMetadata{ Platform: "ruby", Framework: "rspec", OSTags: map[string]string{ @@ -34,21 +39,27 @@ func TestPrintPlanReport_AllData(t *testing.T) { }, }, DDTestSettings: &settings.Config{ - Platform: "ruby", - Framework: "rspec", - MinParallelism: 2, - MaxParallelism: 8, - ParallelRunnerOverhead: 25 * time.Second, + Platform: "python", + Framework: "pytest", + MinParallelism: minParallelism, + MaxParallelism: maxParallelism, + ParallelRunnerOverhead: 30 * time.Second, WorkerEnv: "RAILS_ENV=test;DATABASE_PASSWORD=secret", - CiNode: -1, + CiNode: 0, CiNodeWorkers: 2, - Command: "bundle exec rspec", + Command: "pytest -q", TestsLocation: "spec/**/*_spec.rb", + TestsExcludePattern: "spec/system/**/*_spec.rb", + TestDiscoveryCache: ".ddtest-cache/tests.json", + TestSkippingLevel: settings.TestSkippingLevelSuite, + ForceFullTestDiscovery: true, + StrictDiscovery: true, RuntimeTags: `{"runtime.version":"3.3.4"}`, - ReportEnabled: true, + ReportEnabled: false, }, DatadogSettings: datadogSettingsReport{ Available: true, + FetchDuration: 240 * time.Millisecond, TestImpactAnalysis: true, TestSkipping: true, TestImpactCollection: false, @@ -58,27 +69,61 @@ func TestPrintPlanReport_AllData(t *testing.T) { FlakyTestManagement: true, }, KnownTests: knownTestsReport{ - Available: true, - Modules: 4, - Suites: 1284, - Tests: 18921, + Available: true, + FetchDuration: 80 * time.Millisecond, + Modules: 4, + Suites: 1284, + Tests: 18921, + }, + Skippables: skippablesReport{ + Available: true, + FetchDuration: 110 * time.Millisecond, + TestSkippingLevel: settings.TestSkippingLevelSuite, + TIASuites: 312, }, - SkippableTestsCount: 312, ManagedFlakyTests: managedFlakyTestsReport{ - Available: true, - Total: 26, - Quarantined: 8, - Disabled: 3, - AttemptToFix: 5, + Available: true, + FetchDuration: 90 * time.Millisecond, + Total: 26, + Quarantined: 8, + Disabled: 3, + AttemptToFix: 5, + }, + TestSuiteDurations: testSuiteDurationsReport{ + Available: true, + FetchDuration: 140 * time.Millisecond, + Modules: 3, + Suites: 1491, }, Planning: planningReport{ - TestFilesDiscovered: 642, - FullySkippedFiles: 118, - TestFilesToRun: 524, - DurationSources: durationSourceReport{ - Known: 431, - Default: 90, + Discovery: discoveryReport{ + Available: true, + Mode: discoveryModeFull, + Cache: discoveryCacheResult{ + Configured: true, + Used: true, + }, + Duration: 3 * time.Second, + TestFiles: 642, + Suites: 1284, + Tests: 18921, + }, + Durations: durationApplicationReport{ + Available: true, + BackendDurationsApplied: 431, + BackendSuitesAdded: 12, + SuitesWithoutDurations: 90, + FilesWithoutDurations: 90, + ExpectedFullDuration: 37*time.Minute + 12*time.Second, + }, + Skipping: skippingApplicationReport{ + Available: true, + TIASuites: 312, + DisabledTests: 3, + UnskippableMarkerSuitesForced: 5, + FullySkippedFiles: 118, }, + TestFilesToRun: 524, EstimatedTimeSaved: 38.4, }, Split: splitScore{ @@ -118,7 +163,7 @@ func TestPrintPlanReport_AllData(t *testing.T) { }, }) - expected := `+++ DDTest: plan report + expected := fmt.Sprintf(`+++ DDTest: plan report Run Service: checkout-api @@ -130,20 +175,26 @@ Run Runtime tags: runtime.name=ruby, runtime.version=3.3.4 DDTest settings - Platform: ruby - Framework: rspec - Min parallelism: 2 - Max parallelism: 8 - CI job overhead: 25s + Platform: python + Framework: pytest + Min parallelism: %s + Max parallelism: %s + CI job overhead: 30s Worker env: DATABASE_PASSWORD, RAILS_ENV - CI node: -1 + CI node: 0 CI node workers: 2 - Command: bundle exec rspec + Command: pytest -q Tests location: spec/**/*_spec.rb + Tests exclude pattern: spec/system/**/*_spec.rb + Test discovery cache: .ddtest-cache/tests.json + Test skipping mode: suite + Force full test discovery: true + Strict discovery: true Runtime tags: {"runtime.version":"3.3.4"} - Report enabled: true + Report enabled: false Datadog settings + Fetch duration: 240ms Test Impact Analysis: enabled Test skipping: enabled Test impact collection: disabled @@ -153,22 +204,37 @@ Datadog settings Flaky test management: enabled Backend data - Known tests: 4 modules, 1,284 suites, 18,921 tests - Skippable tests for this run: 312 - Managed flaky tests: 26 total, 8 quarantined, 3 disabled, 5 attempt-to-fix + Known tests: 4 modules, 1,284 suites, 18,921 tests (fetched in 80ms) + TIA skippables returned: 312 suites (fetched in 110ms) + Managed flaky tests: 26 total, 8 quarantined, 3 disabled, 5 attempt-to-fix (fetched in 90ms) + Test suite durations: 3 modules, 1,491 suites (fetched in 140ms) Planning - Test files discovered: 642 - Fully skipped files: 118 - Test files to run: 524 - Duration source: 431 known, 90 default - Estimated time saved: 38.40% - -Split - Runners: 6 - Expected wall time: 4m12s - Imbalance: 11s - Total estimated runtime: 23m46s + Discovery + Method: full + Test files: 642 + Cache: used + Duration: 3s + Suites discovered: 1,284 + Tests discovered: 18,921 + Duration estimates + Backend durations used: 431 suites + Default durations used: 90 suites + Backend-only suites added: 12 + Skipping + TIA skippables applied: 312 suites + Disabled tests applied: 3 tests + Suites marked unskippable: 5 + Files fully skipped: 118 + Run set + Test files to run: 524 + Estimated time saved: 38.40%% + Runner split + Full runtime: 37m12s + Estimated runtime: 23m46s + Runners: 6 + Expected wall time: 4m12s + Imbalance: 11s Slow suites on dedicated runners ATTENTION: 1 dedicated runner @@ -177,16 +243,167 @@ Slow suites on dedicated runners 10 slowest test suites overall 1. rspec / Checkout::VerySlow (spec/very_slow_spec.rb): historical duration 3m0s, estimated runtime 3m0s 2. rspec / Checkout::Slow (spec/slow_spec.rb): historical duration 2m0s, estimated runtime 1m40s -` +`, formatCount(minParallelism), formatCount(maxParallelism)) if output.String() != expected { t.Errorf("unexpected plan report:\n%s", output.String()) } } +func TestPrintPlanReport_FastDiscovery(t *testing.T) { + var output strings.Builder + + printPlanReportData(&output, PlanReportData{ + RunInfo: runmetadata.RunInfo{ + Service: "checkout-api", + }, + PlanMetadata: PlanMetadata{ + Platform: "ruby", + }, + DatadogSettings: datadogSettingsReport{ + Available: true, + FetchDuration: 50 * time.Millisecond, + TestImpactAnalysis: true, + TestSkipping: true, + TestImpactCollection: true, + KnownTests: true, + }, + KnownTests: knownTestsReport{ + Available: true, + FetchDuration: 20 * time.Millisecond, + Modules: 1, + Suites: 10, + Tests: 125, + }, + Skippables: skippablesReport{ + Available: true, + FetchDuration: 30 * time.Millisecond, + TestSkippingLevel: settings.TestSkippingLevelSuite, + TIASuites: 8, + }, + TestSuiteDurations: testSuiteDurationsReport{ + Available: true, + FetchDuration: 40 * time.Millisecond, + Modules: 1, + Suites: 12, + }, + Planning: planningReport{ + Discovery: discoveryReport{ + Available: true, + Mode: discoveryModeFast, + Duration: 120 * time.Millisecond, + TestFiles: 24, + Suites: 0, + }, + Durations: durationApplicationReport{ + Available: true, + BackendDurationsApplied: 12, + BackendSuitesAdded: 2, + SuitesWithoutDurations: 1, + }, + Skipping: skippingApplicationReport{ + Available: true, + TIASuites: 8, + FullySkippedFiles: 6, + }, + TestFilesToRun: 18, + EstimatedTimeSaved: 25, + }, + Split: splitScore{ + parallelRunners: 3, + wallTime: 90_000, + imbalance: 500, + totalRuntime: 210_000, + }, + SlowestTestSuitesOverall: []testSuiteTimingReport{ + { + Module: "rspec", + Suite: "Checkout::Order", + SourceFile: "spec/models/order_spec.rb", + TotalDuration: 2*time.Minute + 30*time.Second, + EstimatedDuration: 105 * time.Second, + DurationSource: testFileDurationSourceKnown, + }, + { + Module: "rspec", + Suite: "Checkout::Payment", + SourceFile: "spec/models/payment_spec.rb", + TotalDuration: time.Minute, + EstimatedDuration: 30 * time.Second, + DurationSource: testFileDurationSourceKnown, + }, + }, + }) + + expected := `+++ DDTest: plan report + +Run + Service: checkout-api + Repository: not available + Commit: not available + Branch: not available + Platform: ruby + OS tags: not available + Runtime tags: not available + +DDTest settings + Settings: not available + +Datadog settings + Fetch duration: 50ms + Test Impact Analysis: enabled + Test skipping: enabled + Test impact collection: enabled + Known tests: enabled + Early flake detection: disabled + Auto test retries: disabled + Flaky test management: disabled + +Backend data + Known tests: 1 modules, 10 suites, 125 tests (fetched in 20ms) + TIA skippables returned: 8 suites (fetched in 30ms) + Managed flaky tests: disabled + Test suite durations: 1 modules, 12 suites (fetched in 40ms) + +Planning + Discovery + Method: fast + Test files: 24 + Duration: 120ms + Duration estimates + Backend durations used: 12 suites + Default durations used: 1 suite + Backend-only suites added: 2 + Skipping + TIA skippables applied: 8 suites + Disabled tests applied: disabled + Suites marked unskippable: 0 + Files fully skipped: 6 + Run set + Test files to run: 18 + Estimated time saved: 25.00% + Runner split + Full runtime: not available + Estimated runtime: 3m30s + Runners: 3 + Expected wall time: 1m30s + Imbalance: 500ms + +Slow suites on dedicated runners + None + +10 slowest test suites overall + 1. rspec / Checkout::Order (spec/models/order_spec.rb): historical duration 2m30s, estimated runtime 1m45s + 2. rspec / Checkout::Payment (spec/models/payment_spec.rb): historical duration 1m0s, estimated runtime 30s +` + if output.String() != expected { + t.Errorf("unexpected fast discovery report:\n%s", output.String()) + } +} + func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { var output strings.Builder - printPlanReport(&output, planReport{}) + printPlanReportData(&output, PlanReportData{}) report := output.String() if !strings.Contains(report, "DDTest settings\n Settings: not available") { @@ -195,18 +412,114 @@ func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { if !strings.Contains(report, " Settings: not available") { t.Errorf("expected missing settings message, got:\n%s", report) } - if !strings.Contains(report, " Known tests: not available") { - t.Errorf("expected missing known tests message, got:\n%s", report) + if !strings.Contains(report, "Planning\n Discovery: not available") { + t.Errorf("expected missing discovery message, got:\n%s", report) + } + if !strings.Contains(report, "Backend data\n Known tests: not available") { + t.Errorf("expected missing backend data message, got:\n%s", report) + } + if !strings.Contains(report, " TIA skippables returned: not available") { + t.Errorf("expected missing TIA skippables message, got:\n%s", report) } if !strings.Contains(report, " Managed flaky tests: not available") { t.Errorf("expected missing managed flaky tests message, got:\n%s", report) } + if !strings.Contains(report, " Test suite durations: not available") { + t.Errorf("expected missing test suite durations message, got:\n%s", report) + } + if !strings.Contains(report, " Duration estimates: not available") { + t.Errorf("expected missing duration estimates message, got:\n%s", report) + } + if !strings.Contains(report, " Skipping: not available") { + t.Errorf("expected missing skipping message, got:\n%s", report) + } + if !strings.Contains(report, " Run set: not available") { + t.Errorf("expected missing run set message, got:\n%s", report) + } + if !strings.Contains(report, " Runner split: not available") { + t.Errorf("expected missing runner split message, got:\n%s", report) + } +} + +func TestPrintPlanReport_DefaultSettings(t *testing.T) { + var output strings.Builder + defaults := defaultDDTestSettings() + + printPlanReportData(&output, PlanReportData{ + DDTestSettings: &defaults, + }) + + report := output.String() + if !strings.Contains(report, "DDTest settings\n Settings: defaults") { + t.Errorf("expected default ddtest settings message, got:\n%s", report) + } +} + +func TestPrintDDTestSettingsReport_AllSupportedSettings(t *testing.T) { + config := defaultDDTestSettings() + config.Platform = "python" + config.Framework = "pytest" + config.Command = "pytest -q" + config.MinParallelism++ + config.MaxParallelism += 2 + config.ParallelRunnerOverhead += time.Second + config.CiNode = 0 + config.CiNodeWorkers = 2 + config.WorkerEnv = "TOKEN=secret" + config.TestsLocation = "tests/**/*_test.py" + config.TestsExcludePattern = "tests/system/**/*_test.py" + config.TestDiscoveryCache = ".ddtest-cache/tests.json" + config.TestSkippingLevel = settings.TestSkippingLevelSuite + config.ForceFullTestDiscovery = true + config.StrictDiscovery = true + config.RuntimeTags = `{"runtime.version":"3.3.4"}` + config.ReportEnabled = false + + var output strings.Builder + printDDTestSettingsReport(&output, &config) + + names := make([]string, 0) + for _, line := range strings.Split(output.String(), "\n") { + if !strings.HasPrefix(line, " ") { + continue + } + name, _, ok := strings.Cut(strings.TrimSpace(line), ":") + if ok { + names = append(names, name) + } + } + if len(names) != reflect.TypeOf(settings.Config{}).NumField() { + t.Fatalf("expected every supported setting to be reported, got %d names from:\n%s", len(names), output.String()) + } + + expectedNames := []string{ + "Platform", + "Framework", + "Min parallelism", + "Max parallelism", + "CI job overhead", + "Worker env", + "CI node", + "CI node workers", + "Command", + "Tests location", + "Tests exclude pattern", + "Test discovery cache", + "Test skipping mode", + "Force full test discovery", + "Strict discovery", + "Runtime tags", + "Report enabled", + } + if !reflect.DeepEqual(names, expectedNames) { + t.Fatalf("unexpected changed setting names:\ngot: %v\nwant: %v", names, expectedNames) + } } func TestPrintPlanReport_DisabledFeatures(t *testing.T) { var output strings.Builder - printPlanReport(&output, planReport{ + printPlanReportData(&output, PlanReportData{ DatadogSettings: datadogSettingsReport{ Available: true, }, @@ -216,7 +529,7 @@ func TestPrintPlanReport_DisabledFeatures(t *testing.T) { if !strings.Contains(report, " Known tests: disabled") { t.Errorf("expected disabled known tests, got:\n%s", report) } - if !strings.Contains(report, " Skippable tests for this run: disabled") { + if !strings.Contains(report, " TIA skippables returned: disabled") { t.Errorf("expected disabled skippable tests, got:\n%s", report) } if !strings.Contains(report, " Managed flaky tests: disabled") { @@ -225,39 +538,234 @@ func TestPrintPlanReport_DisabledFeatures(t *testing.T) { } func TestReportSummaries(t *testing.T) { - known := newKnownTestsReport(&api.KnownTestsResponseData{ - Tests: api.KnownTestsResponseDataModules{ - "module-a": api.KnownTestsResponseDataSuites{ - "suite-a": []string{"test-a", "test-b"}, - }, - "module-b": api.KnownTestsResponseDataSuites{ - "suite-b": []string{"test-c"}, - "suite-c": []string{"test-d", "test-e"}, + client := &MockTestOptimizationClient{ + KnownTests: &api.KnownTestsResponseData{ + Tests: api.KnownTestsResponseDataModules{ + "module-a": api.KnownTestsResponseDataSuites{ + "suite-a": []string{"test-a", "test-b"}, + }, + "module-b": api.KnownTestsResponseDataSuites{ + "suite-b": []string{"test-c"}, + "suite-c": []string{"test-d", "test-e"}, + }, }, }, - }) - if known.Modules != 2 || known.Suites != 3 || known.Tests != 5 { - t.Errorf("unexpected known test summary: %+v", known) - } - - managed := newManagedFlakyTestsReport(&api.TestManagementTestsResponseDataModules{ - Modules: map[string]api.TestManagementTestsResponseDataSuites{ - "module-a": { - Suites: map[string]api.TestManagementTestsResponseDataTests{ - "suite-a": { - Tests: map[string]api.TestManagementTestsResponseDataTestProperties{ - "test-a": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{Quarantined: true}}, - "test-b": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{Disabled: true}}, - "test-c": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{AttemptToFix: true}}, + TestManagementTests: &api.TestManagementTestsResponseDataModules{ + Modules: map[string]api.TestManagementTestsResponseDataSuites{ + "module-a": { + Suites: map[string]api.TestManagementTestsResponseDataTests{ + "suite-a": { + Tests: map[string]api.TestManagementTestsResponseDataTestProperties{ + "test-a": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{Quarantined: true}}, + "test-b": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{Disabled: true}}, + "test-c": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{AttemptToFix: true}}, + }, }, }, }, }, }, - }) + BackendRequestTimingValues: testoptimization.BackendRequestTimings{ + KnownTests: time.Millisecond, + TestManagementTests: time.Millisecond, + TestSuiteDurations: time.Millisecond, + }, + Durations: map[string]map[string]api.TestSuiteDurationInfo{ + "module-a": { + "suite-a": {}, + "suite-b": {}, + }, + "module-b": { + "suite-c": {}, + }, + }, + } + + client.GetTestSuiteDurations() + timings := client.BackendRequestTimings() + + known := reportKnownTests(client.GetKnownTests(), timings.KnownTests) + if known.Modules != 2 || known.Suites != 3 || known.Tests != 5 { + t.Errorf("unexpected known test summary: %+v", known) + } + + managed := reportManagedFlakyTests(client.GetTestManagementTestsData(), timings.TestManagementTests) if managed.Total != 3 || managed.Quarantined != 1 || managed.Disabled != 1 || managed.AttemptToFix != 1 { t.Errorf("unexpected managed flaky test summary: %+v", managed) } + + durationReport := reportTestSuiteDurations(client.GetTestSuiteDurations(), timings.TestSuiteDurations) + if durationReport.Modules != 2 || durationReport.Suites != 3 { + t.Errorf("unexpected test suite durations summary: %+v", durationReport) + } +} + +func TestFormatTIASkippablesReportsOnlyActiveCount(t *testing.T) { + settingsReport := datadogSettingsReport{Available: true, TestSkipping: true} + + testLevel := formatTIASkippables(settingsReport, skippablesReport{ + Available: true, + TestSkippingLevel: settings.TestSkippingLevelTest, + TIATests: 123, + TIASuites: 456, + }) + if testLevel != "123 tests" { + t.Fatalf("unexpected test-level TIA skippables report: %s", testLevel) + } + + suiteLevel := formatTIASkippables(settingsReport, skippablesReport{ + Available: true, + TestSkippingLevel: settings.TestSkippingLevelSuite, + TIATests: 123, + TIASuites: 456, + }) + if suiteLevel != "456 suites" { + t.Fatalf("unexpected suite-level TIA skippables report: %s", suiteLevel) + } +} + +func TestReportFormattingVariants(t *testing.T) { + t.Run("suite labels", func(t *testing.T) { + tests := []struct { + name string + suite testSuiteTimingReport + want string + }{ + {name: "missing", suite: testSuiteTimingReport{}, want: "not available"}, + {name: "module only", suite: testSuiteTimingReport{Module: "rspec"}, want: "rspec"}, + {name: "suite only", suite: testSuiteTimingReport{Suite: "CartSuite"}, want: "CartSuite"}, + {name: "module and suite", suite: testSuiteTimingReport{Module: "rspec", Suite: "CartSuite"}, want: "rspec / CartSuite"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := formatSuiteLabel(tt.suite); got != tt.want { + t.Fatalf("formatSuiteLabel() = %q, want %q", got, tt.want) + } + }) + } + }) + + t.Run("basic values", func(t *testing.T) { + tests := []struct { + name string + got string + want string + }{ + {name: "plural dedicated runners", got: formatScheduledTestSuiteCount(2), want: "2 dedicated runners"}, + {name: "empty worker env", got: formatWorkerEnvKeys(" "), want: "not set"}, + {name: "platform only", got: formatPlatform("ruby", ""), want: "ruby"}, + {name: "framework only", got: formatPlatform("", "rspec"), want: "rspec"}, + {name: "empty setting", got: valueOrNotSet(""), want: "not set"}, + {name: "negative count", got: formatCount(-123456), want: "-123,456"}, + {name: "three digit group count", got: formatCount(123456), want: "123,456"}, + {name: "singular count unit", got: formatCountWithUnit(1, "suite", "suites"), want: "1 suite"}, + {name: "sub-millisecond duration", got: formatDuration(500 * time.Microsecond), want: "500µs"}, + {name: "cache not configured", got: formatDiscoveryCache(discoveryCacheResult{}), want: "not configured"}, + {name: "cache used", got: formatDiscoveryCache(discoveryCacheResult{Configured: true, Used: true}), want: "used"}, + {name: "cache configured but not used", got: formatDiscoveryCache(discoveryCacheResult{Configured: true}), want: "not used"}, + {name: "cache configured but skipped with reason", got: formatDiscoveryCache(discoveryCacheResult{Configured: true, NotUsedReason: "full discovery not required"}), want: "not used (full discovery not required)"}, + {name: "backend data without duration", got: formatBackendDataValue("not available", 0), want: "not available"}, + {name: "optional duration missing", got: formatOptionalDuration(0), want: "not available"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.got != tt.want { + t.Fatalf("got %q, want %q", tt.got, tt.want) + } + }) + } + }) + + t.Run("ddtest setting fallback formatting", func(t *testing.T) { + unnamedField := reflect.StructField{Name: "CustomSetting"} + if got := formatDDTestSettingName(unnamedField); got != "CustomSetting" { + t.Fatalf("formatDDTestSettingName() = %q, want CustomSetting", got) + } + + field := reflect.StructField{Name: "CustomSetting", Tag: `mapstructure:"custom_setting"`} + value := reflect.ValueOf(struct{ Enabled bool }{Enabled: true}) + if got := formatDDTestSettingValue(field, value); got != "{true}" { + t.Fatalf("formatDDTestSettingValue() = %q, want {true}", got) + } + }) +} + +func TestSkippableReportFormattingVariants(t *testing.T) { + t.Run("applied TIA skippables", func(t *testing.T) { + tests := []struct { + name string + datadogSettings datadogSettingsReport + skippables skippablesReport + skipping skippingApplicationReport + want string + }{ + { + name: "disabled", + datadogSettings: datadogSettingsReport{Available: true, TestSkipping: false}, + skippables: skippablesReport{Available: true, TestSkippingLevel: settings.TestSkippingLevelTest}, + skipping: skippingApplicationReport{TIATests: 4}, + want: "disabled", + }, + { + name: "not available", + skipping: skippingApplicationReport{TIATests: 4}, + want: "not available", + }, + { + name: "test level", + skippables: skippablesReport{Available: true, TestSkippingLevel: settings.TestSkippingLevelTest}, + skipping: skippingApplicationReport{TIATests: 1}, + want: "1 test", + }, + { + name: "mode not available", + skippables: skippablesReport{Available: true}, + skipping: skippingApplicationReport{TIATests: 2}, + want: "mode not available", + }, + { + name: "mixed mode fallback", + skippables: skippablesReport{Available: true, TestSkippingLevel: settings.TestSkippingLevel("mixed")}, + skipping: skippingApplicationReport{TIATests: 2, TIASuites: 3}, + want: "2 tests, 3 suites", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := formatAppliedTIASkippables(tt.datadogSettings, tt.skippables, tt.skipping) + if got != tt.want { + t.Fatalf("formatAppliedTIASkippables() = %q, want %q", got, tt.want) + } + }) + } + }) + + t.Run("returned TIA skippables", func(t *testing.T) { + got := formatTIASkippables( + datadogSettingsReport{Available: true, TestSkipping: true}, + skippablesReport{Available: true}, + ) + if got != "skipping mode not available" { + t.Fatalf("formatTIASkippables() = %q, want skipping mode not available", got) + } + + got = formatTIASkippables( + datadogSettingsReport{Available: true, TestSkipping: true}, + skippablesReport{Available: true, TestSkippingLevel: settings.TestSkippingLevel("mixed"), TIATests: 1, TIASuites: 2}, + ) + if got != "1 test, 2 suites" { + t.Fatalf("formatTIASkippables() = %q, want mixed fallback", got) + } + }) + + t.Run("disabled tests", func(t *testing.T) { + if got := formatAppliedDisabledTests(datadogSettingsReport{}, managedFlakyTestsReport{}, skippingApplicationReport{}); got != "not available" { + t.Fatalf("formatAppliedDisabledTests() = %q, want not available", got) + } + if got := formatAppliedDisabledTests(datadogSettingsReport{Available: true}, managedFlakyTestsReport{}, skippingApplicationReport{}); got != "disabled" { + t.Fatalf("formatAppliedDisabledTests() = %q, want disabled", got) + } + }) } func TestFormatWorkerEnvKeys(t *testing.T) { diff --git a/internal/planner/test_optimization_plan_cache.go b/internal/planner/test_optimization_plan_cache.go index 32ebdc2..42560e0 100644 --- a/internal/planner/test_optimization_plan_cache.go +++ b/internal/planner/test_optimization_plan_cache.go @@ -1,7 +1,6 @@ package planner import ( - "encoding/json" "log/slog" "github.com/DataDog/ddtest/internal/runmetadata" @@ -16,7 +15,7 @@ type testOptimizationPlanCache struct { TestFileWeights map[string]int `json:"testFileWeights"` TestFileDurationSources map[string]testFileDurationSource `json:"testFileDurationSources"` RunInfo runmetadata.RunInfo `json:"runInfo"` - PlanInfo PlanInfo `json:"planInfo"` + PlanMetadata PlanMetadata `json:"planMetadata"` } func (tp *TestPlanner) storeTestOptimizationPlanCache() error { @@ -27,24 +26,24 @@ func (tp *TestPlanner) storeTestOptimizationPlanCache() error { TestFileWeights: tp.testFileWeights, TestFileDurationSources: tp.testFileDurationSources, RunInfo: tp.runInfo, - PlanInfo: tp.planInfo, + PlanMetadata: tp.planMetadata, } return testoptimization.NewCacheManager().StoreTestOptimizationPlanCache(cache) } -func LoadPlan() (PlanInfo, error) { +func LoadPlan() (PlanMetadata, error) { return newTestPlannerWithDefaults().LoadPlan() } -func (tp *TestPlanner) LoadPlan() (PlanInfo, error) { +func (tp *TestPlanner) LoadPlan() (PlanMetadata, error) { if !tp.planLoaded { if err := tp.restoreTestOptimizationPlanCache(); err != nil { - return PlanInfo{}, err + return PlanMetadata{}, err } } - return tp.planInfo, nil + return tp.planMetadata, nil } func (tp *TestPlanner) restoreTestOptimizationPlanCache() error { @@ -59,7 +58,7 @@ func (tp *TestPlanner) restoreTestOptimizationPlanCache() error { tp.testFileWeights = cache.TestFileWeights tp.testFileDurationSources = cache.TestFileDurationSources tp.runInfo = cache.RunInfo - tp.planInfo = cache.PlanInfo + tp.planMetadata = cache.PlanMetadata tp.planLoaded = true testSuitesCount := countTestSuites(tp.testSuiteDurations) @@ -77,63 +76,6 @@ func (tp *TestPlanner) restoreTestOptimizationPlanCache() error { return nil } -type legacyRunInfo struct { - Service string `json:"service"` - Repository string `json:"repository"` - Commit string `json:"commit"` - Branch string `json:"branch"` - Platform string `json:"platform"` - Framework string `json:"framework"` - OSTags map[string]string `json:"osTags"` - RuntimeTags map[string]string `json:"runtimeTags"` -} - -func (c *testOptimizationPlanCache) UnmarshalJSON(data []byte) error { - var decoded struct { - TestSuiteDurations map[string]map[string]api.TestSuiteDurationInfo `json:"testSuiteDurations"` - SuiteAggregates map[testSuiteKey]testSuiteAggregate `json:"suiteAggregates"` - SuitesBySourceFile map[string][]testSuiteKey `json:"suitesBySourceFile"` - TestFileWeights map[string]int `json:"testFileWeights"` - TestFileDurationSources map[string]testFileDurationSource `json:"testFileDurationSources"` - RunInfo legacyRunInfo `json:"runInfo"` - PlanInfo PlanInfo `json:"planInfo"` - } - if err := json.Unmarshal(data, &decoded); err != nil { - return err - } - - c.TestSuiteDurations = decoded.TestSuiteDurations - c.SuiteAggregates = decoded.SuiteAggregates - c.SuitesBySourceFile = decoded.SuitesBySourceFile - c.TestFileWeights = decoded.TestFileWeights - c.TestFileDurationSources = decoded.TestFileDurationSources - c.RunInfo = decoded.RunInfo.runInfo() - c.PlanInfo = decoded.PlanInfo - if c.PlanInfo.IsZero() { - c.PlanInfo = decoded.RunInfo.planInfo() - } - - return nil -} - -func (r legacyRunInfo) runInfo() runmetadata.RunInfo { - return runmetadata.RunInfo{ - Service: r.Service, - Repository: r.Repository, - Commit: r.Commit, - Branch: r.Branch, - } -} - -func (r legacyRunInfo) planInfo() PlanInfo { - return PlanInfo{ - Platform: r.Platform, - Framework: r.Framework, - OSTags: r.OSTags, - RuntimeTags: r.RuntimeTags, - } -} - func readAndNormalizeTestOptimizationPlanCache(cache *testOptimizationPlanCache) error { if err := testoptimization.NewCacheManager().ReadTestOptimizationPlanCache(cache); err != nil { return err diff --git a/internal/planner/test_optimization_plan_cache_test.go b/internal/planner/test_optimization_plan_cache_test.go index a83cee7..27f8de5 100644 --- a/internal/planner/test_optimization_plan_cache_test.go +++ b/internal/planner/test_optimization_plan_cache_test.go @@ -55,6 +55,13 @@ func TestTestPlanner_Plan_StoresTestOptimizationPlanCache(t *testing.T) { if _, err := os.Stat(cachePath); err != nil { t.Fatalf("Expected test optimization plan cache file to be written: %v", err) } + cacheData, err := os.ReadFile(cachePath) + if err != nil { + t.Fatalf("Expected test optimization plan cache file to be readable: %v", err) + } + if !strings.Contains(string(cacheData), `"planMetadata"`) { + t.Fatalf("Expected test optimization plan cache to store planMetadata, got: %s", string(cacheData)) + } restored := NewWithDependencies( &MockPlatformDetector{}, @@ -74,8 +81,8 @@ func TestTestPlanner_Plan_StoresTestOptimizationPlanCache(t *testing.T) { if !reflect.DeepEqual(restored.testFileWeights, runner.testFileWeights) { t.Errorf("Expected restored test file weights to match stored weights.\nexpected: %v\nactual: %v", runner.testFileWeights, restored.testFileWeights) } - if !reflect.DeepEqual(restored.planInfo, runner.planInfo) { - t.Errorf("Expected restored plan info to match stored plan info.\nexpected: %v\nactual: %v", runner.planInfo, restored.planInfo) + if !reflect.DeepEqual(restored.planMetadata, runner.planMetadata) { + t.Errorf("Expected restored plan info to match stored plan info.\nexpected: %v\nactual: %v", runner.planMetadata, restored.planMetadata) } } @@ -214,61 +221,6 @@ func TestTestPlanner_RestoreTestOptimizationPlanCache_ComputesMissingWeights(t * } } -func TestLoadPlan_MigratesLegacyRunInfoPlanFields(t *testing.T) { - tempDir := t.TempDir() - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) - - type legacyTestOptimizationPlanCache struct { - RunInfo legacyRunInfo `json:"runInfo"` - } - cache := legacyTestOptimizationPlanCache{ - RunInfo: legacyRunInfo{ - Service: "checkout-api", - Repository: "https://github.com/acme/checkout.git", - Commit: "9f3a1c7d2b4e", - Branch: "feature/split-report", - Platform: "ruby", - Framework: "rspec", - OSTags: map[string]string{ - "os.platform": "linux", - "os.architecture": "amd64", - "os.version": "6.8.0", - }, - RuntimeTags: map[string]string{ - "runtime.name": "ruby", - "runtime.version": "3.3.4", - }, - }, - } - if err := testoptimization.NewCacheManager().StoreTestOptimizationPlanCache(cache); err != nil { - t.Fatalf("StoreTestOptimizationPlanCache() should not return error, got: %v", err) - } - - plan, err := LoadPlan() - if err != nil { - t.Fatalf("LoadPlan() should not return error, got: %v", err) - } - - expected := PlanInfo{ - Platform: "ruby", - Framework: "rspec", - OSTags: map[string]string{ - "os.platform": "linux", - "os.architecture": "amd64", - "os.version": "6.8.0", - }, - RuntimeTags: map[string]string{ - "runtime.name": "ruby", - "runtime.version": "3.3.4", - }, - } - if !reflect.DeepEqual(plan, expected) { - t.Errorf("Expected LoadPlan() to migrate legacy plan fields.\nexpected: %v\nactual: %v", expected, plan) - } -} - func TestTestPlanner_LoadPlan_UsesExistingPlannerState(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() @@ -284,7 +236,7 @@ func TestTestPlanner_LoadPlan_UsesExistingPlannerState(t *testing.T) { } planner := newTestPlannerWithDefaults() - planner.planInfo = PlanInfo{ + planner.planMetadata = PlanMetadata{ Platform: "ruby", Framework: "rspec", } diff --git a/internal/runner/report.go b/internal/runner/report.go index 181cb0d..3061b6d 100644 --- a/internal/runner/report.go +++ b/internal/runner/report.go @@ -20,30 +20,30 @@ type runExecutionReport struct { } type runReport struct { - RunInfo runmetadata.RunInfo - PlanInfo planner.PlanInfo - Execution runExecutionReport - Duration time.Duration - Err error + RunInfo runmetadata.RunInfo + PlanMetadata planner.PlanMetadata + Execution runExecutionReport + Duration time.Duration + Err error } func printRunReport(w io.Writer, report runReport) { reportFprintln(w, "+++ DDTest: run report") reportFprintln(w) - printRunInfoReport(w, report.RunInfo, report.PlanInfo) + printRunInfoReport(w, report.RunInfo, report.PlanMetadata) reportFprintln(w) printExecutionReport(w, report) } -func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planInfo planner.PlanInfo) { +func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planMetadata planner.PlanMetadata) { reportFprintln(w, "Run") reportFprintf(w, " Service: %s\n", valueOrNotAvailable(runInfo.Service)) reportFprintf(w, " Repository: %s\n", valueOrNotAvailable(runInfo.Repository)) reportFprintf(w, " Commit: %s\n", valueOrNotAvailable(runInfo.Commit)) reportFprintf(w, " Branch: %s\n", valueOrNotAvailable(runInfo.Branch)) - reportFprintf(w, " Platform: %s\n", formatPlatform(planInfo.Platform, planInfo.Framework)) - reportFprintf(w, " OS tags: %s\n", formatTagList(planInfo.OSTags, constants.OSPlatform, constants.OSArchitecture, constants.OSVersion)) - reportFprintf(w, " Runtime tags: %s\n", formatTagList(planInfo.RuntimeTags, constants.RuntimeName, constants.RuntimeVersion)) + reportFprintf(w, " Platform: %s\n", formatPlatform(planMetadata.Platform, planMetadata.Framework)) + reportFprintf(w, " OS tags: %s\n", formatTagList(planMetadata.OSTags, constants.OSPlatform, constants.OSArchitecture, constants.OSVersion)) + reportFprintf(w, " Runtime tags: %s\n", formatTagList(planMetadata.RuntimeTags, constants.RuntimeName, constants.RuntimeVersion)) } func printExecutionReport(w io.Writer, report runReport) { diff --git a/internal/runner/report_test.go b/internal/runner/report_test.go index ba6761f..dff74d0 100644 --- a/internal/runner/report_test.go +++ b/internal/runner/report_test.go @@ -21,7 +21,7 @@ func TestPrintRunReport_Passed(t *testing.T) { Commit: "9f3a1c7d2b4e", Branch: "feature/split-report", }, - PlanInfo: planner.PlanInfo{ + PlanMetadata: planner.PlanMetadata{ Platform: "ruby", Framework: "rspec", OSTags: map[string]string{ diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 60500cf..4b8704a 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -23,7 +23,7 @@ type Runner interface { type Planner interface { Plan(ctx context.Context) error - LoadPlan() (planner.PlanInfo, error) + LoadPlan() (planner.PlanMetadata, error) DistributeTestFiles(testFiles []string, parallelRunners int) [][]string } @@ -67,7 +67,7 @@ func (tr *TestRunner) Run(ctx context.Context) error { return fmt.Errorf("failed to check parallel runners count at %s: %w", constants.ParallelRunnersOutputPath, err) } - planInfo, err := tr.planner.LoadPlan() + planMetadata, err := tr.planner.LoadPlan() if err != nil { slog.Error("Test optimization plan is not available", "error", err) return fmt.Errorf("test optimization plan is not available: %w", err) @@ -96,8 +96,8 @@ func (tr *TestRunner) Run(ctx context.Context) error { } slog.Info("Framework detected", "framework", framework.Name()) runInfo := runmetadata.New(ciUtils.GetCITags()) - if planInfo.IsZero() { - planInfo = planner.NewPlanInfo(nil, detectedPlatform.Name(), framework.Name(), detectedPlatform.TestSkippingLevel()) + if planMetadata.IsZero() { + planMetadata = planner.NewPlanMetadata(nil, detectedPlatform.Name(), framework.Name(), detectedPlatform.TestSkippingLevel()) } ciNode := settings.GetCiNode() @@ -114,11 +114,11 @@ func (tr *TestRunner) Run(ctx context.Context) error { if settings.GetReportEnabled() { printRunReport(tr.reportWriter, runReport{ - RunInfo: runInfo, - PlanInfo: planInfo, - Execution: executionResult.report, - Duration: time.Since(startTime), - Err: executionResult.err, + RunInfo: runInfo, + PlanMetadata: planMetadata, + Execution: executionResult.report, + Duration: time.Since(startTime), + Err: executionResult.err, }) } return executionResult.err diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index a6798d5..ca49f52 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -21,7 +21,7 @@ type fakePlanner struct { distributeCalls int planFunc func(context.Context) error distributeFunc func([]string, int) [][]string - plan planner.PlanInfo + plan planner.PlanMetadata loadErr error distributedTestFiles [][]string distributedWorkerNums []int @@ -35,7 +35,7 @@ func (f *fakePlanner) Plan(ctx context.Context) error { return nil } -func (f *fakePlanner) LoadPlan() (planner.PlanInfo, error) { +func (f *fakePlanner) LoadPlan() (planner.PlanMetadata, error) { f.loadCalls++ return f.plan, f.loadErr } @@ -68,7 +68,7 @@ func TestTestRunner_Run_PlansThroughPublicClientWhenArtifactsMissing(t *testing. writeRunnerTestFile(t, constants.TestFilesOutputPath, "spec/a_spec.rb\n") return nil }, - plan: planner.PlanInfo{ + plan: planner.PlanMetadata{ Platform: "ruby", Framework: "rspec", }, @@ -258,7 +258,7 @@ func TestTestRunner_Run_WritesReportWhenEnabled(t *testing.T) { framework := &MockFramework{FrameworkName: "rspec"} platform := &MockPlatform{PlatformName: "ruby", Framework: framework} testPlanner := &fakePlanner{ - plan: planner.PlanInfo{ + plan: planner.PlanMetadata{ Platform: "ruby", Framework: "rspec", }, diff --git a/internal/testoptimization/api/durations_api.go b/internal/testoptimization/api/durations_api.go index 8ad493f..eb434d6 100644 --- a/internal/testoptimization/api/durations_api.go +++ b/internal/testoptimization/api/durations_api.go @@ -76,6 +76,10 @@ type ( func (c *transport) GetTestSuiteDurations() *TestSuiteDurationsResponseData { startTime := time.Now() + defer func() { + c.backendRequestTimings.TestSuiteDurations = time.Since(startTime) + }() + if c.repositoryURL == "" { slog.Error("Test durations API errored", "duration", time.Since(startTime), "error", "repository URL is required") return emptyTestSuiteDurationsResponseData() diff --git a/internal/testoptimization/api/durations_api_test.go b/internal/testoptimization/api/durations_api_test.go index 9084c4a..5c3f1a9 100644 --- a/internal/testoptimization/api/durations_api_test.go +++ b/internal/testoptimization/api/durations_api_test.go @@ -103,6 +103,9 @@ func TestClientGetTestSuiteDurationsLogsSuccess(t *testing.T) { if len(result.TestSuites) != 1 { t.Errorf("expected 1 module, got %d", len(result.TestSuites)) } + if duration := client.BackendRequestTimings().TestSuiteDurations; duration <= 0 { + t.Errorf("expected test suite durations fetch duration to be recorded, got %s", duration) + } if !strings.Contains(logs.String(), "level=INFO") || !strings.Contains(logs.String(), "Fetched test suite durations") || !strings.Contains(logs.String(), "modulesCount=1") || diff --git a/internal/testoptimization/api/known_tests_api.go b/internal/testoptimization/api/known_tests_api.go index 820cb03..0bab7d2 100644 --- a/internal/testoptimization/api/known_tests_api.go +++ b/internal/testoptimization/api/known_tests_api.go @@ -8,6 +8,7 @@ package api import ( "encoding/json" "fmt" + "time" ) const ( @@ -62,6 +63,11 @@ type ( ) func (c *transport) GetKnownTests() (*KnownTestsResponseData, error) { + startTime := time.Now() + defer func() { + c.backendRequestTimings.KnownTests = time.Since(startTime) + }() + if c.repositoryURL == "" || c.commitSha == "" { return nil, fmt.Errorf("testoptimization.GetKnownTests: repository URL and commit SHA are required") } diff --git a/internal/testoptimization/api/raw_response_test.go b/internal/testoptimization/api/raw_response_test.go index fbeca55..1945c7b 100644 --- a/internal/testoptimization/api/raw_response_test.go +++ b/internal/testoptimization/api/raw_response_test.go @@ -117,6 +117,20 @@ func TestClientStoresRawBackendResponses(t *testing.T) { if string(client.GetTestManagementTestsRawResponse()) != testManagementResponse { t.Fatalf("test management raw response mismatch:\nexpected: %s\nactual: %s", testManagementResponse, string(client.GetTestManagementTestsRawResponse())) } + + timings := client.BackendRequestTimings() + if timings.Settings <= 0 { + t.Errorf("expected settings duration to be recorded, got %s", timings.Settings) + } + if timings.KnownTests <= 0 { + t.Errorf("expected known tests duration to be recorded, got %s", timings.KnownTests) + } + if timings.Skippables <= 0 { + t.Errorf("expected skippables duration to be recorded, got %s", timings.Skippables) + } + if timings.TestManagementTests <= 0 { + t.Errorf("expected test management duration to be recorded, got %s", timings.TestManagementTests) + } } func TestClientBuildsSkippableKeyFromTestBundle(t *testing.T) { diff --git a/internal/testoptimization/api/settings_api.go b/internal/testoptimization/api/settings_api.go index c478882..717a0a7 100644 --- a/internal/testoptimization/api/settings_api.go +++ b/internal/testoptimization/api/settings_api.go @@ -8,6 +8,7 @@ package api import ( "fmt" "log/slog" + "time" "github.com/DataDog/ddtest/internal/settings" ) @@ -71,6 +72,11 @@ type ( ) func (c *transport) GetSettings() (*SettingsResponseData, error) { + startTime := time.Now() + defer func() { + c.backendRequestTimings.Settings = time.Since(startTime) + }() + if c.repositoryURL == "" || c.commitSha == "" { return nil, fmt.Errorf("testoptimization.GetSettings: repository URL and commit SHA are required") } diff --git a/internal/testoptimization/api/skippable.go b/internal/testoptimization/api/skippable.go index 72804f5..d24118e 100644 --- a/internal/testoptimization/api/skippable.go +++ b/internal/testoptimization/api/skippable.go @@ -8,6 +8,7 @@ package api import ( "fmt" "log/slog" + "time" "github.com/DataDog/ddtest/internal/settings" ) @@ -85,6 +86,18 @@ func (s Skippables) Count() int { } func (c *transport) GetSkippableTests() (correlationID string, skippables Skippables, err error) { + startTime := time.Now() + defer func() { + duration := time.Since(startTime) + c.backendRequestTimings.Skippables = duration + if err == nil { + slog.Debug("Finished fetching skippable tests and suites", + "testsCount", len(skippables.Tests), + "suitesCount", len(skippables.Suites), + "duration", duration) + } + }() + if c.repositoryURL == "" || c.commitSha == "" { err = fmt.Errorf("testoptimization.GetSkippableTests: repository URL and commit SHA are required") return diff --git a/internal/testoptimization/api/skippable_test.go b/internal/testoptimization/api/skippable_test.go index 0f36eec..107dbde 100644 --- a/internal/testoptimization/api/skippable_test.go +++ b/internal/testoptimization/api/skippable_test.go @@ -12,6 +12,7 @@ import ( ) func TestTransportGetSkippableTestsRequestAndResponse(t *testing.T) { + logs := captureRawResponseTestLogs(t) var captured skippableRequest response := skippableResponse{ Meta: skippableResponseMeta{CorrelationID: "correlation-id"}, @@ -89,6 +90,12 @@ func TestTransportGetSkippableTestsRequestAndResponse(t *testing.T) { if len(client.GetSkippableTestsRawResponse()) == 0 { t.Fatal("expected raw skippable response to be stored") } + if !strings.Contains(logs.String(), "Finished fetching skippable tests and suites") || + !strings.Contains(logs.String(), "testsCount=1") || + !strings.Contains(logs.String(), "suitesCount=0") || + !strings.Contains(logs.String(), "duration=") { + t.Fatalf("expected skippable fetch duration log, got logs: %s", logs.String()) + } } func TestTransportGetSkippableTestsRequestUsesConfiguredTestLevel(t *testing.T) { diff --git a/internal/testoptimization/api/test_management_tests_api.go b/internal/testoptimization/api/test_management_tests_api.go index 41d4be7..6162687 100644 --- a/internal/testoptimization/api/test_management_tests_api.go +++ b/internal/testoptimization/api/test_management_tests_api.go @@ -7,6 +7,7 @@ package api import ( "fmt" + "time" ) const ( @@ -65,6 +66,11 @@ type ( ) func (c *transport) GetTestManagementTests() (*TestManagementTestsResponseDataModules, error) { + startTime := time.Now() + defer func() { + c.backendRequestTimings.TestManagementTests = time.Since(startTime) + }() + if c.repositoryURL == "" { return nil, fmt.Errorf("testoptimization.GetTestManagementTests: repository URL is required") } diff --git a/internal/testoptimization/api/transport.go b/internal/testoptimization/api/transport.go index bce1b94..ada8390 100644 --- a/internal/testoptimization/api/transport.go +++ b/internal/testoptimization/api/transport.go @@ -46,6 +46,15 @@ type ( GetSkippableTestsRawResponse() json.RawMessage GetTestManagementTests() (*TestManagementTestsResponseDataModules, error) GetTestManagementTestsRawResponse() json.RawMessage + BackendRequestTimings() BackendRequestTimings + } + + BackendRequestTimings struct { + Settings time.Duration + KnownTests time.Duration + Skippables time.Duration + TestManagementTests time.Duration + TestSuiteDurations time.Duration } // transport sends requests to the Datadog backend. @@ -71,6 +80,7 @@ type ( knownTestsRawResponse json.RawMessage skippableTestsRawResponse json.RawMessage testManagementTestsRawResponse json.RawMessage + backendRequestTimings BackendRequestTimings } // testConfigurations represents the test configurations. @@ -362,6 +372,10 @@ func (c *transport) GetTestManagementTestsRawResponse() json.RawMessage { return cloneRawMessage(c.testManagementTestsRawResponse) } +func (c *transport) BackendRequestTimings() BackendRequestTimings { + return c.backendRequestTimings +} + // getURLPath returns the full URL path for the given URL path. func (c *transport) getURLPath(urlPath string) string { if c.agentless { diff --git a/internal/testoptimization/testoptimization.go b/internal/testoptimization/testoptimization.go index de71f64..4db9be0 100644 --- a/internal/testoptimization/testoptimization.go +++ b/internal/testoptimization/testoptimization.go @@ -21,17 +21,10 @@ import ( const autoDetectServiceName = "" -const ( - libraryCapabilitiesTestImpactAnalysis = "_dd.library_capabilities.test_impact_analysis" - libraryCapabilitiesEarlyFlakeDetection = "_dd.library_capabilities.early_flake_detection" - libraryCapabilitiesAutoTestRetries = "_dd.library_capabilities.auto_test_retries" - libraryCapabilitiesTestManagementQuarantine = "_dd.library_capabilities.test_management.quarantine" - libraryCapabilitiesTestManagementDisable = "_dd.library_capabilities.test_management.disable" - libraryCapabilitiesTestManagementAttemptToFix = "_dd.library_capabilities.test_management.attempt_to_fix" -) - type testOptimizationCloseAction func() +type BackendRequestTimings = api.BackendRequestTimings + type searchCommitsResponse struct { LocalCommits []string RemoteCommits []string @@ -51,10 +44,11 @@ type TestOptimizationClient struct { closeActionsMutex sync.Mutex closeActions []testOptimizationCloseAction settings *api.SettingsResponseData - knownTests api.KnownTestsResponseData + knownTests *api.KnownTestsResponseData skippables api.Skippables + testManagementTests *api.TestManagementTestsResponseDataModules + testSuiteDurations *api.TestSuiteDurationsResponseData testSkippingLevel settings.TestSkippingLevel - testManagementTests api.TestManagementTestsResponseDataModules } func NewTestOptimizationClient() *TestOptimizationClient { @@ -125,9 +119,6 @@ func (c *TestOptimizationClient) GetSettings() *api.SettingsResponseData { } func (c *TestOptimizationClient) GetSkippables() api.Skippables { - startTime := time.Now() - - slog.Debug("Fetching skippable tests and suites...") c.ensureTestOptimizationInitialized() if c.skippables.Tests == nil { c.skippables.Tests = api.SkippableTests{} @@ -142,12 +133,6 @@ func (c *TestOptimizationClient) GetSkippables() api.Skippables { } } - duration := time.Since(startTime) - slog.Debug("Finished fetching skippable tests and suites", - "testsCount", len(c.skippables.Tests), - "suitesCount", len(c.skippables.Suites), - "duration", duration) - return c.skippables } @@ -156,7 +141,7 @@ func (c *TestOptimizationClient) GetKnownTests() *api.KnownTestsResponseData { return nil } c.ensureTestOptimizationInitialized() - return &c.knownTests + return c.knownTests } func (c *TestOptimizationClient) GetTestManagementTestsData() *api.TestManagementTestsResponseDataModules { @@ -164,7 +149,7 @@ func (c *TestOptimizationClient) GetTestManagementTestsData() *api.TestManagemen return nil } c.ensureTestOptimizationInitialized() - return &c.testManagementTests + return c.testManagementTests } func (c *TestOptimizationClient) GetDisabledTests() map[string]bool { @@ -172,13 +157,26 @@ func (c *TestOptimizationClient) GetDisabledTests() map[string]bool { } func (c *TestOptimizationClient) GetTestSuiteDurations() *api.TestSuiteDurationsResponseData { + if c.testSuiteDurations != nil { + return c.testSuiteDurations + } + testOptimizationTransport := c.ensureAPITransport(autoDetectServiceName) if testOptimizationTransport == nil { - return &api.TestSuiteDurationsResponseData{ + c.testSuiteDurations = &api.TestSuiteDurationsResponseData{ TestSuites: map[string]map[string]api.TestSuiteDurationInfo{}, } + return c.testSuiteDurations } - return testOptimizationTransport.GetTestSuiteDurations() + c.testSuiteDurations = testOptimizationTransport.GetTestSuiteDurations() + return c.testSuiteDurations +} + +func (c *TestOptimizationClient) BackendRequestTimings() BackendRequestTimings { + if c.apiTransport == nil { + return BackendRequestTimings{} + } + return c.apiTransport.BackendRequestTimings() } func (c *TestOptimizationClient) StoreCacheAndExit() { @@ -337,27 +335,7 @@ func (c *TestOptimizationClient) ensureTestOptimizationInitialized() { return } - additionalTags := map[string]string{ - libraryCapabilitiesEarlyFlakeDetection: "1", - libraryCapabilitiesAutoTestRetries: "1", - libraryCapabilitiesTestImpactAnalysis: "1", - libraryCapabilitiesTestManagementQuarantine: "1", - libraryCapabilitiesTestManagementDisable: "1", - libraryCapabilitiesTestManagementAttemptToFix: "5", - } - defer func() { - if len(additionalTags) > 0 { - slog.Debug("testoptimization: adding additional tags", "tags", additionalTags) //nolint:gocritic // Map structure logging for debugging - environment.AddCITagsMap(additionalTags) - } - }() - - var additionalTagsMutex sync.Mutex - setAdditionalTags := func(key string, value string) { - additionalTagsMutex.Lock() - defer additionalTagsMutex.Unlock() - additionalTags[key] = value - } + var skippablesCorrelationID string var wg sync.WaitGroup @@ -365,11 +343,11 @@ func (c *TestOptimizationClient) ensureTestOptimizationInitialized() { wg.Add(1) go func() { defer wg.Done() - knownTests, err := c.apiTransport.GetKnownTests() + result, err := c.apiTransport.GetKnownTests() if err != nil { slog.Error("testoptimization: error getting test optimization known tests data", "err", err.Error()) - } else if knownTests != nil { - c.knownTests = *knownTests + } else if result != nil { + c.knownTests = result slog.Debug("testoptimization: known tests data loaded.") } }() @@ -379,15 +357,12 @@ func (c *TestOptimizationClient) ensureTestOptimizationInitialized() { wg.Add(1) go func() { defer wg.Done() - correlationID, skippables, err := c.apiTransport.GetSkippableTests() + correlationID, result, err := c.apiTransport.GetSkippableTests() if err != nil { slog.Error("testoptimization: error getting test optimization skippable tests", "err", err.Error()) } else { - slog.Debug("testoptimization: skippable tests loaded", - "testsCount", len(skippables.Tests), - "suitesCount", len(skippables.Suites)) - setAdditionalTags(constants.ItrCorrelationIDTag, correlationID) - c.skippables = skippables + c.skippables = result + skippablesCorrelationID = correlationID } }() } @@ -396,17 +371,23 @@ func (c *TestOptimizationClient) ensureTestOptimizationInitialized() { wg.Add(1) go func() { defer wg.Done() - testManagementTests, err := c.apiTransport.GetTestManagementTests() + result, err := c.apiTransport.GetTestManagementTests() if err != nil { slog.Error("testoptimization: error getting test optimization test management tests", "err", err.Error()) - } else if testManagementTests != nil { - c.testManagementTests = *testManagementTests + } else if result != nil { + c.testManagementTests = result slog.Debug("testoptimization: test management loaded", "attemptToFixRetries", currentSettings.TestManagement.AttemptToFixRetries) } }() } wg.Wait() + if skippablesCorrelationID != "" { + slog.Debug("testoptimization: adding skippables correlation ID tag", "correlationID", skippablesCorrelationID) + environment.AddCITagsMap(map[string]string{ + constants.ItrCorrelationIDTag: skippablesCorrelationID, + }) + } }) } diff --git a/internal/testoptimization/testoptimization_test.go b/internal/testoptimization/testoptimization_test.go index 19e59f6..3360501 100644 --- a/internal/testoptimization/testoptimization_test.go +++ b/internal/testoptimization/testoptimization_test.go @@ -55,6 +55,7 @@ type MockAPIClient struct { SendPackFilesBytes int64 SendPackFilesErr error SendPackFilesCalls int + BackendRequestTimingValues api.BackendRequestTimings } func (m *MockAPIClient) GetSettings() (*api.SettingsResponseData, error) { @@ -127,6 +128,10 @@ func (m *MockAPIClient) SendPackFiles(commitSha string, packFiles []string) (byt return m.SendPackFilesBytes, m.SendPackFilesErr } +func (m *MockAPIClient) BackendRequestTimings() api.BackendRequestTimings { + return m.BackendRequestTimingValues +} + func cleanPlanDirectory(t *testing.T) { t.Helper() if err := os.RemoveAll(constants.PlanDirectory); err != nil { @@ -251,6 +256,7 @@ func TestTestOptimizationClient_GetTestSuiteDurations(t *testing.T) { client := newTestOptimizationClientForTest(t, mockAPIClient) result := client.GetTestSuiteDurations() + cachedResult := client.GetTestSuiteDurations() if mockAPIClient.TestSuiteDurationsCalls != 1 { t.Fatalf("GetTestSuiteDurations() should fetch durations once, got %d calls", mockAPIClient.TestSuiteDurationsCalls) @@ -258,6 +264,68 @@ func TestTestOptimizationClient_GetTestSuiteDurations(t *testing.T) { if result.TestSuites["rspec"]["Suite"].SourceFile != "spec/suite_spec.rb" { t.Fatalf("GetTestSuiteDurations() returned %#v, want %#v", result, durations) } + if cachedResult != result { + t.Fatal("GetTestSuiteDurations() should return cached durations on subsequent calls") + } +} + +func TestTestOptimizationClient_BackendRequestTimings(t *testing.T) { + repositorySettings := &api.SettingsResponseData{ + KnownTestsEnabled: true, + TestsSkipping: true, + } + repositorySettings.TestManagement.Enabled = true + + mockAPIClient := &MockAPIClient{ + Settings: repositorySettings, + KnownTests: &api.KnownTestsResponseData{Tests: api.KnownTestsResponseDataModules{"module": {"suite": {"test"}}}}, + Skippables: api.NewSkippables(), + TestManagementTestsData: &api.TestManagementTestsResponseDataModules{Modules: map[string]api.TestManagementTestsResponseDataSuites{}}, + TestSuiteDurations: map[string]map[string]api.TestSuiteDurationInfo{"module": {"suite": {}}}, + BackendRequestTimingValues: api.BackendRequestTimings{ + Settings: time.Millisecond, + KnownTests: 2 * time.Millisecond, + Skippables: 3 * time.Millisecond, + TestManagementTests: 4 * time.Millisecond, + TestSuiteDurations: 5 * time.Millisecond, + }, + } + client := newTestOptimizationClientForTest(t, mockAPIClient) + + if err := client.Initialize(map[string]string{}); err != nil { + t.Fatalf("Initialize() failed: %v", err) + } + client.GetSkippables() + client.GetTestSuiteDurations() + + timings := client.BackendRequestTimings() + for name, duration := range map[string]time.Duration{ + "settings": timings.Settings, + "known tests": timings.KnownTests, + "skippables": timings.Skippables, + "test management": timings.TestManagementTests, + "test suite durations": timings.TestSuiteDurations, + } { + if duration <= 0 { + t.Errorf("expected %s duration to be recorded, got %s", name, duration) + } + } + + if client.GetSettings() != repositorySettings { + t.Errorf("expected settings operation to store fetched settings, got %+v", client.GetSettings()) + } + if knownTests := client.GetKnownTests(); knownTests == nil || len(knownTests.Tests) != 1 { + t.Errorf("expected known tests operation to store fetched data, got %+v", knownTests) + } + if skippables := client.GetSkippables(); skippables.Tests == nil || skippables.Suites == nil { + t.Errorf("expected skippables operation to be recorded, got %+v", skippables) + } + if testManagementTests := client.GetTestManagementTestsData(); testManagementTests == nil { + t.Errorf("expected test management operation to store fetched data, got %+v", testManagementTests) + } + if testSuiteDurations := client.GetTestSuiteDurations(); testSuiteDurations == nil || len(testSuiteDurations.TestSuites) != 1 { + t.Errorf("expected duration operation to store fetched durations, got %+v", testSuiteDurations) + } } func TestTestOptimizationClient_Initialize(t *testing.T) {