From 03b70043412af99542a465c4620e59d5a5503a39 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 30 Jun 2026 09:52:25 +0200 Subject: [PATCH 1/7] Report changed DDTest settings --- internal/planner/report.go | 89 +++++++++++++++++++--- internal/planner/report_test.go | 127 +++++++++++++++++++++++++++----- internal/settings/settings.go | 59 ++++++++++----- 3 files changed, 226 insertions(+), 49 deletions(-) diff --git a/internal/planner/report.go b/internal/planner/report.go index 9ff9c93..cd06e98 100644 --- a/internal/planner/report.go +++ b/internal/planner/report.go @@ -3,6 +3,7 @@ package planner import ( "fmt" "io" + "reflect" "slices" "strconv" "strings" @@ -51,18 +52,82 @@ func printDDTestSettingsReport(w io.Writer, config *settings.Config) { return } - reportFprintf(w, " Platform: %s\n", valueOrNotSet(config.Platform)) - reportFprintf(w, " Framework: %s\n", valueOrNotSet(config.Framework)) - reportFprintf(w, " Min parallelism: %s\n", formatCount(config.MinParallelism)) - reportFprintf(w, " Max parallelism: %s\n", formatCount(config.MaxParallelism)) - reportFprintf(w, " CI job overhead: %s\n", formatDuration(config.ParallelRunnerOverhead)) - reportFprintf(w, " Worker env: %s\n", formatWorkerEnvKeys(config.WorkerEnv)) - reportFprintf(w, " CI node: %s\n", formatCount(config.CiNode)) - reportFprintf(w, " CI node workers: %s\n", formatCount(config.CiNodeWorkers)) - reportFprintf(w, " Command: %s\n", valueOrNotSet(config.Command)) - reportFprintf(w, " Tests location: %s\n", valueOrNotSet(config.TestsLocation)) - reportFprintf(w, " Runtime tags: %s\n", valueOrNotSet(config.RuntimeTags)) - reportFprintf(w, " Report enabled: %s\n", strconv.FormatBool(config.ReportEnabled)) + if !printChangedDDTestSettings(w, config) { + reportFprintln(w, " Settings: defaults") + } +} + +func printChangedDDTestSettings(w io.Writer, config *settings.Config) bool { + defaults := settings.DefaultConfig() + configValue := reflect.ValueOf(*config) + defaultsValue := reflect.ValueOf(defaults) + configType := configValue.Type() + printed := false + + for i := range configValue.NumField() { + value := configValue.Field(i) + if reflect.DeepEqual(value.Interface(), defaultsValue.Field(i).Interface()) { + continue + } + + field := configType.Field(i) + reportFprintf(w, " %s: %s\n", formatDDTestSettingName(field), formatDDTestSettingValue(field, value)) + printed = true + } + + return printed +} + +func formatDDTestSettingName(field reflect.StructField) string { + if label := field.Tag.Get("report"); label != "" { + return label + } + + key := configFieldKey(field) + if key == "" { + return field.Name + } + + words := strings.Split(key, "_") + for i, word := range words { + if word == "ci" { + words[i] = "CI" + continue + } + if i == 0 { + words[i] = strings.ToUpper(word[:1]) + word[1:] + } + } + return strings.Join(words, " ") +} + +func formatDDTestSettingValue(field reflect.StructField, value reflect.Value) string { + if configFieldKey(field) == "worker_env" { + return formatWorkerEnvKeys(value.String()) + } + + switch value.Type() { + case reflect.TypeOf(time.Duration(0)): + return formatDuration(time.Duration(value.Int())) + case reflect.TypeOf(settings.TestSkippingLevel("")): + return valueOrNotSet(value.Interface().(settings.TestSkippingLevel).String()) + } + + switch value.Kind() { + case reflect.String: + return valueOrNotSet(value.String()) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return formatCount(int(value.Int())) + case reflect.Bool: + return strconv.FormatBool(value.Bool()) + default: + return fmt.Sprint(value.Interface()) + } +} + +func configFieldKey(field reflect.StructField) string { + key, _, _ := strings.Cut(field.Tag.Get("mapstructure"), ",") + return key } func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { diff --git a/internal/planner/report_test.go b/internal/planner/report_test.go index a2c0e79..18aac38 100644 --- a/internal/planner/report_test.go +++ b/internal/planner/report_test.go @@ -1,6 +1,8 @@ package planner import ( + "fmt" + "reflect" "strings" "testing" "time" @@ -12,6 +14,8 @@ import ( func TestPrintPlanReport_AllData(t *testing.T) { var output strings.Builder + minParallelism := settings.DefaultParallelism() + 1 + maxParallelism := settings.DefaultParallelism() + 2 printPlanReport(&output, planReport{ RunInfo: runmetadata.RunInfo{ @@ -34,18 +38,23 @@ func TestPrintPlanReport_AllData(t *testing.T) { }, }, DDTestSettings: &settings.Config{ - Platform: "ruby", - Framework: "rspec", - MinParallelism: 2, - MaxParallelism: 8, - ParallelRunnerOverhead: 25 * time.Second, + Platform: "python", + Framework: "pytest", + MinParallelism: minParallelism, + MaxParallelism: maxParallelism, + ParallelRunnerOverhead: 30 * time.Second, WorkerEnv: "RAILS_ENV=test;DATABASE_PASSWORD=secret", - CiNode: -1, + CiNode: 0, CiNodeWorkers: 2, - Command: "bundle exec rspec", + Command: "pytest -q", TestsLocation: "spec/**/*_spec.rb", + TestsExcludePattern: "spec/system/**/*_spec.rb", + TestDiscoveryCache: ".ddtest-cache/tests.json", + TestSkippingLevel: settings.TestSkippingLevelSuite, + ForceFullTestDiscovery: true, + StrictDiscovery: true, RuntimeTags: `{"runtime.version":"3.3.4"}`, - ReportEnabled: true, + ReportEnabled: false, }, DatadogSettings: datadogSettingsReport{ Available: true, @@ -118,7 +127,7 @@ func TestPrintPlanReport_AllData(t *testing.T) { }, }) - expected := `+++ DDTest: plan report + expected := fmt.Sprintf(`+++ DDTest: plan report Run Service: checkout-api @@ -130,18 +139,23 @@ Run Runtime tags: runtime.name=ruby, runtime.version=3.3.4 DDTest settings - Platform: ruby - Framework: rspec - Min parallelism: 2 - Max parallelism: 8 - CI job overhead: 25s + Platform: python + Framework: pytest + Min parallelism: %s + Max parallelism: %s + CI job overhead: 30s Worker env: DATABASE_PASSWORD, RAILS_ENV - CI node: -1 + CI node: 0 CI node workers: 2 - Command: bundle exec rspec + Command: pytest -q Tests location: spec/**/*_spec.rb + Tests exclude pattern: spec/system/**/*_spec.rb + Test discovery cache: .ddtest-cache/tests.json + Test skipping mode: suite + Force full test discovery: true + Strict discovery: true Runtime tags: {"runtime.version":"3.3.4"} - Report enabled: true + Report enabled: false Datadog settings Test Impact Analysis: enabled @@ -162,7 +176,7 @@ Planning Fully skipped files: 118 Test files to run: 524 Duration source: 431 known, 90 default - Estimated time saved: 38.40% + Estimated time saved: 38.40%% Split Runners: 6 @@ -177,7 +191,7 @@ Slow suites on dedicated runners 10 slowest test suites overall 1. rspec / Checkout::VerySlow (spec/very_slow_spec.rb): historical duration 3m0s, estimated runtime 3m0s 2. rspec / Checkout::Slow (spec/slow_spec.rb): historical duration 2m0s, estimated runtime 1m40s -` +`, formatCount(minParallelism), formatCount(maxParallelism)) if output.String() != expected { t.Errorf("unexpected plan report:\n%s", output.String()) } @@ -203,6 +217,81 @@ func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { } } +func TestPrintPlanReport_DefaultSettings(t *testing.T) { + var output strings.Builder + defaults := settings.DefaultConfig() + + printPlanReport(&output, planReport{ + DDTestSettings: &defaults, + }) + + report := output.String() + if !strings.Contains(report, "DDTest settings\n Settings: defaults") { + t.Errorf("expected default ddtest settings message, got:\n%s", report) + } +} + +func TestPrintDDTestSettingsReport_AllSupportedSettings(t *testing.T) { + config := settings.DefaultConfig() + config.Platform = "python" + config.Framework = "pytest" + config.Command = "pytest -q" + config.MinParallelism++ + config.MaxParallelism += 2 + config.ParallelRunnerOverhead += time.Second + config.CiNode = 0 + config.CiNodeWorkers = 2 + config.WorkerEnv = "TOKEN=secret" + config.TestsLocation = "tests/**/*_test.py" + config.TestsExcludePattern = "tests/system/**/*_test.py" + config.TestDiscoveryCache = ".ddtest-cache/tests.json" + config.TestSkippingLevel = settings.TestSkippingLevelSuite + config.ForceFullTestDiscovery = true + config.StrictDiscovery = true + config.RuntimeTags = `{"runtime.version":"3.3.4"}` + config.ReportEnabled = false + + var output strings.Builder + printDDTestSettingsReport(&output, &config) + + names := make([]string, 0) + for _, line := range strings.Split(output.String(), "\n") { + if !strings.HasPrefix(line, " ") { + continue + } + name, _, ok := strings.Cut(strings.TrimSpace(line), ":") + if ok { + names = append(names, name) + } + } + if len(names) != reflect.TypeOf(settings.Config{}).NumField() { + t.Fatalf("expected every supported setting to be reported, got %d names from:\n%s", len(names), output.String()) + } + + expectedNames := []string{ + "Platform", + "Framework", + "Min parallelism", + "Max parallelism", + "CI job overhead", + "Worker env", + "CI node", + "CI node workers", + "Command", + "Tests location", + "Tests exclude pattern", + "Test discovery cache", + "Test skipping mode", + "Force full test discovery", + "Strict discovery", + "Runtime tags", + "Report enabled", + } + if !reflect.DeepEqual(names, expectedNames) { + t.Fatalf("unexpected changed setting names:\ngot: %v\nwant: %v", names, expectedNames) + } +} + func TestPrintPlanReport_DisabledFeatures(t *testing.T) { var output strings.Builder diff --git a/internal/settings/settings.go b/internal/settings/settings.go index b159cf4..adaf371 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -99,7 +99,7 @@ type Config struct { Framework string `mapstructure:"framework"` MinParallelism int `mapstructure:"min_parallelism"` MaxParallelism int `mapstructure:"max_parallelism"` - ParallelRunnerOverhead time.Duration `mapstructure:"parallel_runner_overhead"` + ParallelRunnerOverhead time.Duration `mapstructure:"parallel_runner_overhead" report:"CI job overhead"` WorkerEnv string `mapstructure:"worker_env"` CiNode int `mapstructure:"ci_node"` CiNodeWorkers int `mapstructure:"ci_node_workers"` @@ -114,6 +114,28 @@ type Config struct { ReportEnabled bool `mapstructure:"report_enabled"` } +func DefaultConfig() Config { + return Config{ + Platform: "ruby", + Framework: "rspec", + MinParallelism: DefaultParallelism(), + MaxParallelism: DefaultParallelism(), + ParallelRunnerOverhead: defaultParallelRunnerOverhead, + WorkerEnv: "", + CiNode: -1, + CiNodeWorkers: defaultCiNodeWorkers, + Command: "", + TestsLocation: "", + TestsExcludePattern: "", + TestDiscoveryCache: "", + TestSkippingLevel: TestSkippingLevelTest, + ForceFullTestDiscovery: false, + StrictDiscovery: false, + RuntimeTags: "", + ReportEnabled: true, + } +} + var ( config *Config ) @@ -175,23 +197,24 @@ func Init() { } func setDefaults() { - viper.SetDefault("platform", "ruby") - viper.SetDefault("framework", "rspec") - viper.SetDefault("min_parallelism", DefaultParallelism()) - viper.SetDefault("max_parallelism", DefaultParallelism()) - viper.SetDefault("parallel_runner_overhead", defaultParallelRunnerOverhead.String()) - viper.SetDefault("worker_env", "") - viper.SetDefault("ci_node", -1) - viper.SetDefault("ci_node_workers", strconv.Itoa(defaultCiNodeWorkers)) - viper.SetDefault("command", "") - viper.SetDefault("tests_location", "") - viper.SetDefault("tests_exclude_pattern", "") - viper.SetDefault("test_discovery_cache", "") - viper.SetDefault("test_skipping_mode", TestSkippingLevelTest) - viper.SetDefault("force_full_test_discovery", false) - viper.SetDefault("strict_discovery", false) - viper.SetDefault("runtime_tags", "") - viper.SetDefault("report_enabled", true) + defaults := DefaultConfig() + viper.SetDefault("platform", defaults.Platform) + viper.SetDefault("framework", defaults.Framework) + viper.SetDefault("min_parallelism", defaults.MinParallelism) + viper.SetDefault("max_parallelism", defaults.MaxParallelism) + viper.SetDefault("parallel_runner_overhead", defaults.ParallelRunnerOverhead.String()) + viper.SetDefault("worker_env", defaults.WorkerEnv) + viper.SetDefault("ci_node", defaults.CiNode) + viper.SetDefault("ci_node_workers", strconv.Itoa(defaults.CiNodeWorkers)) + viper.SetDefault("command", defaults.Command) + viper.SetDefault("tests_location", defaults.TestsLocation) + viper.SetDefault("tests_exclude_pattern", defaults.TestsExcludePattern) + viper.SetDefault("test_discovery_cache", defaults.TestDiscoveryCache) + viper.SetDefault("test_skipping_mode", defaults.TestSkippingLevel.String()) + viper.SetDefault("force_full_test_discovery", defaults.ForceFullTestDiscovery) + viper.SetDefault("strict_discovery", defaults.StrictDiscovery) + viper.SetDefault("runtime_tags", defaults.RuntimeTags) + viper.SetDefault("report_enabled", defaults.ReportEnabled) } // NormalizeTestSkippingLevel accepts only the backend-supported TIA skipping modes. From b4cb55b895684218996048fe67324acd43bb1839 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 30 Jun 2026 10:22:48 +0200 Subject: [PATCH 2/7] Report backend duration data --- internal/planner/planner.go | 4 +++- internal/planner/planner_test.go | 15 +++++++++++++++ internal/planner/report.go | 10 ++++++++++ internal/planner/report_data.go | 22 ++++++++++++++++++++++ internal/planner/report_test.go | 24 ++++++++++++++++++++++++ 5 files changed, 74 insertions(+), 1 deletion(-) diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 6bf06b8..e897b72 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -356,7 +356,9 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { cancelDiscovery() } - if testSuiteDurations := tp.optimizationClient.GetTestSuiteDurations(); testSuiteDurations != nil && testSuiteDurations.TestSuites != nil { + testSuiteDurations := tp.optimizationClient.GetTestSuiteDurations() + tp.planReport.TestSuiteDurations = newTestSuiteDurationsReport(testSuiteDurations) + if testSuiteDurations != nil && testSuiteDurations.TestSuites != nil { tp.testSuiteDurations = testSuiteDurations.TestSuites } diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 6b4281f..760e917 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -1976,6 +1976,11 @@ func TestTestPlanner_PreparePlanningData_Success(t *testing.T) { if !mockOptimizationClient.DurationsCalled { t.Error("PreparePlanningData() should fetch test suite durations") } + if !runner.planReport.TestSuiteDurations.Available || + runner.planReport.TestSuiteDurations.Modules != 1 || + runner.planReport.TestSuiteDurations.Suites != 1 { + t.Errorf("Expected plan report to summarize fetched durations, got %+v", runner.planReport.TestSuiteDurations) + } } func TestTestPlanner_PreparePlanningData_DisabledTestManagementTestsAreSkipped(t *testing.T) { @@ -2444,6 +2449,11 @@ func TestTestPlanner_PreparePlanningData_EmptyDurationsContinues(t *testing.T) { if len(runner.testSuiteDurations) != 0 { t.Errorf("Expected empty in-memory test suite durations on empty response, got %v", runner.testSuiteDurations) } + if !runner.planReport.TestSuiteDurations.Available || + runner.planReport.TestSuiteDurations.Modules != 0 || + runner.planReport.TestSuiteDurations.Suites != 0 { + t.Errorf("Expected plan report to summarize empty durations response, got %+v", runner.planReport.TestSuiteDurations) + } } func TestTestPlanner_PreparePlanningData_NonEmptyDurationsUsesP50ForMatchingSuites(t *testing.T) { @@ -2493,6 +2503,11 @@ func TestTestPlanner_PreparePlanningData_NonEmptyDurationsUsesP50ForMatchingSuit if len(runner.testSuiteDurations) != 1 { t.Fatalf("Expected stored durations data, got %v", runner.testSuiteDurations) } + if !runner.planReport.TestSuiteDurations.Available || + runner.planReport.TestSuiteDurations.Modules != 1 || + runner.planReport.TestSuiteDurations.Suites != 2 { + t.Errorf("Expected plan report to summarize fetched durations, got %+v", runner.planReport.TestSuiteDurations) + } if _, ok := runner.testFiles["spec/file1_test.rb"]; !ok { t.Error("Expected file1 in test files") diff --git a/internal/planner/report.go b/internal/planner/report.go index cd06e98..320c61b 100644 --- a/internal/planner/report.go +++ b/internal/planner/report.go @@ -151,6 +151,7 @@ func printBackendDataReport(w io.Writer, report planReport) { reportFprintf(w, " Known tests: %s\n", formatKnownTests(report.DatadogSettings, report.KnownTests)) reportFprintf(w, " Skippable tests for this run: %s\n", formatSkippableTests(report.DatadogSettings, report.SkippableTestsCount)) reportFprintf(w, " Managed flaky tests: %s\n", formatManagedFlakyTests(report.DatadogSettings, report.ManagedFlakyTests)) + reportFprintf(w, " Test suite durations: %s\n", formatTestSuiteDurations(report.TestSuiteDurations)) } func printPlanningReport(w io.Writer, report planningReport) { @@ -310,6 +311,15 @@ func formatManagedFlakyTests(settings datadogSettingsReport, managed managedFlak formatCount(managed.AttemptToFix)) } +func formatTestSuiteDurations(durations testSuiteDurationsReport) string { + if !durations.Available { + return "not available" + } + return fmt.Sprintf("%s modules, %s suites", + formatCount(durations.Modules), + formatCount(durations.Suites)) +} + func formatTagList(tags map[string]string, keys ...string) string { parts := make([]string, 0, len(keys)) for _, key := range keys { diff --git a/internal/planner/report_data.go b/internal/planner/report_data.go index ad3f948..1973a94 100644 --- a/internal/planner/report_data.go +++ b/internal/planner/report_data.go @@ -94,6 +94,27 @@ func newManagedFlakyTestsReport(testManagementTests *api.TestManagementTestsResp return report } +type testSuiteDurationsReport struct { + Available bool + Modules int + Suites int +} + +func newTestSuiteDurationsReport(testSuiteDurations *api.TestSuiteDurationsResponseData) testSuiteDurationsReport { + if testSuiteDurations == nil { + return testSuiteDurationsReport{} + } + + report := testSuiteDurationsReport{ + Available: true, + Modules: len(testSuiteDurations.TestSuites), + } + for _, suites := range testSuiteDurations.TestSuites { + report.Suites += len(suites) + } + return report +} + type durationSourceReport struct { Known int Default int @@ -125,6 +146,7 @@ type planReport struct { KnownTests knownTestsReport SkippableTestsCount int ManagedFlakyTests managedFlakyTestsReport + TestSuiteDurations testSuiteDurationsReport Planning planningReport LongSeparateRunnerSuites []testSuiteTimingReport SlowestTestSuitesOverall []testSuiteTimingReport diff --git a/internal/planner/report_test.go b/internal/planner/report_test.go index 18aac38..9e32270 100644 --- a/internal/planner/report_test.go +++ b/internal/planner/report_test.go @@ -80,6 +80,11 @@ func TestPrintPlanReport_AllData(t *testing.T) { Disabled: 3, AttemptToFix: 5, }, + TestSuiteDurations: testSuiteDurationsReport{ + Available: true, + Modules: 3, + Suites: 1491, + }, Planning: planningReport{ TestFilesDiscovered: 642, FullySkippedFiles: 118, @@ -170,6 +175,7 @@ Backend data Known tests: 4 modules, 1,284 suites, 18,921 tests Skippable tests for this run: 312 Managed flaky tests: 26 total, 8 quarantined, 3 disabled, 5 attempt-to-fix + Test suite durations: 3 modules, 1,491 suites Planning Test files discovered: 642 @@ -215,6 +221,9 @@ func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { if !strings.Contains(report, " Managed flaky tests: not available") { t.Errorf("expected missing managed flaky tests message, got:\n%s", report) } + if !strings.Contains(report, " Test suite durations: not available") { + t.Errorf("expected missing test suite durations message, got:\n%s", report) + } } func TestPrintPlanReport_DefaultSettings(t *testing.T) { @@ -347,6 +356,21 @@ func TestReportSummaries(t *testing.T) { if managed.Total != 3 || managed.Quarantined != 1 || managed.Disabled != 1 || managed.AttemptToFix != 1 { t.Errorf("unexpected managed flaky test summary: %+v", managed) } + + durations := newTestSuiteDurationsReport(&api.TestSuiteDurationsResponseData{ + TestSuites: map[string]map[string]api.TestSuiteDurationInfo{ + "module-a": { + "suite-a": {}, + "suite-b": {}, + }, + "module-b": { + "suite-c": {}, + }, + }, + }) + if durations.Modules != 2 || durations.Suites != 3 { + t.Errorf("unexpected test suite durations summary: %+v", durations) + } } func TestFormatWorkerEnvKeys(t *testing.T) { From f55a2e3b995f662e52e5a8b4d9518bd4e36ef79d Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 30 Jun 2026 10:47:45 +0200 Subject: [PATCH 3/7] Clarify skippables report counts --- internal/planner/discovered_tests.go | 12 ++++++++ internal/planner/planner.go | 2 +- internal/planner/planner_test.go | 13 ++++++--- internal/planner/report.go | 32 +++++++++++++++++---- internal/planner/report_data.go | 20 ++++++++++++- internal/planner/report_test.go | 42 ++++++++++++++++++++++++++-- 6 files changed, 106 insertions(+), 15 deletions(-) diff --git a/internal/planner/discovered_tests.go b/internal/planner/discovered_tests.go index 2f27f3b..ca9bf26 100644 --- a/internal/planner/discovered_tests.go +++ b/internal/planner/discovered_tests.go @@ -121,6 +121,18 @@ func (s skippableMatcher) TIASkippablesCount() int { return len(s.tiaSkippableTests) + len(s.tiaSkippableSuites) } +func (s skippableMatcher) TIATestsCount() int { + return len(s.tiaSkippableTests) +} + +func (s skippableMatcher) TIASuitesCount() int { + return len(s.tiaSkippableSuites) +} + +func (s skippableMatcher) DisabledTestsCount() int { + return len(s.disabledTests) +} + func (tp *TestPlanner) recordFastDiscoveryFallbackFiles(discoveredTestFiles []string) error { excluder, err := discovery.NewExcluder(settings.GetTestsExcludePattern()) if err != nil { diff --git a/internal/planner/planner.go b/internal/planner/planner.go index e897b72..f355ec2 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -349,7 +349,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } skipMatcher = tp.fetchSkippables(tiaSkippingEnabled) - tp.planReport.SkippableTestsCount = skipMatcher.Count() + tp.planReport.Skippables = newSkippablesReport(skipMatcher, testSkippingLevel) if tiaSkippingEnabled && skipMatcher.TIASkippablesCount() == 0 && !forceFullTestDiscovery { slog.Info("No TIA-skippable tests or suites found for this run, cancelling full test discovery") diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 760e917..093cd0d 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -858,6 +858,11 @@ func TestTestPlanner_Plan_JestSuiteSkippingFetchesSkippablesWithoutFullDiscovery if !mockOptimizationClient.GetSkippablesCalled { t.Fatal("expected planner to fetch suite skippables when full test discovery is unsupported") } + if runner.planReport.Skippables.TestSkippingLevel != settings.TestSkippingLevelSuite || + runner.planReport.Skippables.TIATests != 0 || + runner.planReport.Skippables.TIASuites != 1 { + t.Errorf("expected suite-level skippables to be reported separately, got %+v", runner.planReport.Skippables) + } expectedTestFiles := "src/b.test.ts\n" assertFileContent(t, constants.TestFilesOutputPath, expectedTestFiles) @@ -2067,8 +2072,8 @@ func TestTestPlanner_PreparePlanningData_DisabledTestManagementTestsAreSkipped(t t.Errorf("Expected attempt-to-fix file to be scheduled, got %v", runner.testFileWeights) } - if runner.planReport.SkippableTestsCount != 2 { - t.Errorf("Expected planner skip set to include TIA-skippable and disabled tests, got %d", runner.planReport.SkippableTestsCount) + if runner.planReport.Skippables.TIATests != 1 || runner.planReport.Skippables.TIASuites != 0 || runner.planReport.Skippables.DisabledTests != 1 { + t.Errorf("Expected planner report to split TIA and disabled skippables, got %+v", runner.planReport.Skippables) } } @@ -2259,8 +2264,8 @@ func TestTestPlanner_PreparePlanningData_TestManagementDoesNotKeepFullDiscoveryW t.Errorf("Expected fast discovery fallback to keep all discovered files, got %v", runner.testFileWeights) } - if runner.planReport.SkippableTestsCount != 2 { - t.Errorf("Expected planner skip set to include fetched disabled tests for reporting, got %d", runner.planReport.SkippableTestsCount) + if runner.planReport.Skippables.TIATests != 0 || runner.planReport.Skippables.TIASuites != 0 || runner.planReport.Skippables.DisabledTests != 2 { + t.Errorf("Expected planner report to split TIA and disabled skippables, got %+v", runner.planReport.Skippables) } } diff --git a/internal/planner/report.go b/internal/planner/report.go index 320c61b..0fe7fa3 100644 --- a/internal/planner/report.go +++ b/internal/planner/report.go @@ -25,7 +25,7 @@ func printPlanReport(w io.Writer, report planReport) { reportFprintln(w) printBackendDataReport(w, report) reportFprintln(w) - printPlanningReport(w, report.Planning) + printPlanningReport(w, report.Planning, report.Skippables) reportFprintln(w) printSplitReport(w, report.Split) reportFprintln(w) @@ -149,15 +149,16 @@ func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { func printBackendDataReport(w io.Writer, report planReport) { reportFprintln(w, "Backend data") reportFprintf(w, " Known tests: %s\n", formatKnownTests(report.DatadogSettings, report.KnownTests)) - reportFprintf(w, " Skippable tests for this run: %s\n", formatSkippableTests(report.DatadogSettings, report.SkippableTestsCount)) + reportFprintf(w, " TIA skippables returned: %s\n", formatTIASkippables(report.DatadogSettings, report.Skippables)) reportFprintf(w, " Managed flaky tests: %s\n", formatManagedFlakyTests(report.DatadogSettings, report.ManagedFlakyTests)) reportFprintf(w, " Test suite durations: %s\n", formatTestSuiteDurations(report.TestSuiteDurations)) } -func printPlanningReport(w io.Writer, report planningReport) { +func printPlanningReport(w io.Writer, report planningReport, skippables skippablesReport) { reportFprintln(w, "Planning") reportFprintf(w, " Test files discovered: %s\n", formatCount(report.TestFilesDiscovered)) reportFprintf(w, " Fully skipped files: %s\n", formatCount(report.FullySkippedFiles)) + reportFprintf(w, " Test Management disabled tests applied: %s\n", formatDisabledTestsApplied(skippables)) reportFprintf(w, " Test files to run: %s\n", formatCount(report.TestFilesToRun)) reportFprintf(w, " Duration source: %s known, %s default\n", formatCount(report.DurationSources.Known), @@ -290,11 +291,30 @@ func formatKnownTests(settings datadogSettingsReport, known knownTestsReport) st formatCount(known.Tests)) } -func formatSkippableTests(settings datadogSettingsReport, count int) string { - if settings.Available && !settings.TestSkipping { +func formatTIASkippables(datadogSettings datadogSettingsReport, skippables skippablesReport) string { + if datadogSettings.Available && !datadogSettings.TestSkipping { return "disabled" } - return formatCount(count) + if !skippables.Available { + return "not available" + } + switch skippables.TestSkippingLevel { + case settings.TestSkippingLevelTest: + return fmt.Sprintf("%s tests", formatCount(skippables.TIATests)) + case settings.TestSkippingLevelSuite: + return fmt.Sprintf("%s suites", formatCount(skippables.TIASuites)) + case "": + return "skipping mode not available" + default: + return fmt.Sprintf("%s tests, %s suites", formatCount(skippables.TIATests), formatCount(skippables.TIASuites)) + } +} + +func formatDisabledTestsApplied(skippables skippablesReport) string { + if !skippables.Available { + return "not available" + } + return formatCount(skippables.DisabledTests) } func formatManagedFlakyTests(settings datadogSettingsReport, managed managedFlakyTestsReport) string { diff --git a/internal/planner/report_data.go b/internal/planner/report_data.go index 1973a94..e07157c 100644 --- a/internal/planner/report_data.go +++ b/internal/planner/report_data.go @@ -94,6 +94,24 @@ func newManagedFlakyTestsReport(testManagementTests *api.TestManagementTestsResp return report } +type skippablesReport struct { + Available bool + TestSkippingLevel settings.TestSkippingLevel + TIATests int + TIASuites int + DisabledTests int +} + +func newSkippablesReport(skippables skippableMatcher, testSkippingLevel settings.TestSkippingLevel) skippablesReport { + return skippablesReport{ + Available: true, + TestSkippingLevel: testSkippingLevel, + TIATests: skippables.TIATestsCount(), + TIASuites: skippables.TIASuitesCount(), + DisabledTests: skippables.DisabledTestsCount(), + } +} + type testSuiteDurationsReport struct { Available bool Modules int @@ -144,7 +162,7 @@ type planReport struct { DDTestSettings *settings.Config DatadogSettings datadogSettingsReport KnownTests knownTestsReport - SkippableTestsCount int + Skippables skippablesReport ManagedFlakyTests managedFlakyTestsReport TestSuiteDurations testSuiteDurationsReport Planning planningReport diff --git a/internal/planner/report_test.go b/internal/planner/report_test.go index 9e32270..68e1e80 100644 --- a/internal/planner/report_test.go +++ b/internal/planner/report_test.go @@ -72,7 +72,12 @@ func TestPrintPlanReport_AllData(t *testing.T) { Suites: 1284, Tests: 18921, }, - SkippableTestsCount: 312, + Skippables: skippablesReport{ + Available: true, + TestSkippingLevel: settings.TestSkippingLevelSuite, + TIASuites: 312, + DisabledTests: 3, + }, ManagedFlakyTests: managedFlakyTestsReport{ Available: true, Total: 26, @@ -173,13 +178,14 @@ Datadog settings Backend data Known tests: 4 modules, 1,284 suites, 18,921 tests - Skippable tests for this run: 312 + TIA skippables returned: 312 suites Managed flaky tests: 26 total, 8 quarantined, 3 disabled, 5 attempt-to-fix Test suite durations: 3 modules, 1,491 suites Planning Test files discovered: 642 Fully skipped files: 118 + Test Management disabled tests applied: 3 Test files to run: 524 Duration source: 431 known, 90 default Estimated time saved: 38.40%% @@ -218,6 +224,12 @@ func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { if !strings.Contains(report, " Known tests: not available") { t.Errorf("expected missing known tests message, got:\n%s", report) } + if !strings.Contains(report, " TIA skippables returned: not available") { + t.Errorf("expected missing TIA skippables message, got:\n%s", report) + } + if !strings.Contains(report, " Test Management disabled tests applied: not available") { + t.Errorf("expected missing disabled tests message, got:\n%s", report) + } if !strings.Contains(report, " Managed flaky tests: not available") { t.Errorf("expected missing managed flaky tests message, got:\n%s", report) } @@ -314,7 +326,7 @@ func TestPrintPlanReport_DisabledFeatures(t *testing.T) { if !strings.Contains(report, " Known tests: disabled") { t.Errorf("expected disabled known tests, got:\n%s", report) } - if !strings.Contains(report, " Skippable tests for this run: disabled") { + if !strings.Contains(report, " TIA skippables returned: disabled") { t.Errorf("expected disabled skippable tests, got:\n%s", report) } if !strings.Contains(report, " Managed flaky tests: disabled") { @@ -373,6 +385,30 @@ func TestReportSummaries(t *testing.T) { } } +func TestFormatTIASkippablesReportsOnlyActiveCount(t *testing.T) { + settingsReport := datadogSettingsReport{Available: true, TestSkipping: true} + + testLevel := formatTIASkippables(settingsReport, skippablesReport{ + Available: true, + TestSkippingLevel: settings.TestSkippingLevelTest, + TIATests: 123, + TIASuites: 456, + }) + if testLevel != "123 tests" { + t.Fatalf("unexpected test-level TIA skippables report: %s", testLevel) + } + + suiteLevel := formatTIASkippables(settingsReport, skippablesReport{ + Available: true, + TestSkippingLevel: settings.TestSkippingLevelSuite, + TIATests: 123, + TIASuites: 456, + }) + if suiteLevel != "456 suites" { + t.Fatalf("unexpected suite-level TIA skippables report: %s", suiteLevel) + } +} + func TestFormatWorkerEnvKeys(t *testing.T) { report := formatWorkerEnvKeys("TOKEN=secret; RAILS_ENV = test ;BAD_NO_EQUALS;=NO_KEY;TOKEN=other") From ab3a9022f9bec7bc5bfa776a86149e0dbbc62a62 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 30 Jun 2026 11:30:28 +0200 Subject: [PATCH 4/7] Improve planner report narrative --- internal/planner/discovered_tests.go | 65 ++++- internal/planner/planner.go | 72 ++++-- internal/planner/planner_test.go | 36 ++- internal/planner/report.go | 150 ++++++++--- internal/planner/report_data.go | 142 +++++++++-- internal/planner/report_test.go | 364 +++++++++++++++++++++++++-- 6 files changed, 714 insertions(+), 115 deletions(-) diff --git a/internal/planner/discovered_tests.go b/internal/planner/discovered_tests.go index ca9bf26..9a387cc 100644 --- a/internal/planner/discovered_tests.go +++ b/internal/planner/discovered_tests.go @@ -17,6 +17,13 @@ func (tp *TestPlanner) resetDiscoveryResults() { tp.testFiles = make(map[string]struct{}) tp.suiteAggregates = make(map[testSuiteKey]testSuiteAggregate) tp.suitesBySourceFile = make(map[string][]testSuiteKey) + tp.discoveryMode = discoveryModeUnknown + tp.localDiscoveredSuites = 0 + tp.localDiscoveredTests = 0 + tp.tiaSkippableTestsApplied = 0 + tp.tiaSkippableSuitesApplied = make(map[testSuiteKey]struct{}) + tp.disabledTestsApplied = 0 + tp.unskippableMarkerSuitesForced = 0 } func (tp *TestPlanner) recordFullDiscoveryResults( @@ -65,12 +72,14 @@ func (tp *TestPlanner) recordFullDiscoveryResults( tp.testFiles[normalizedSourceFile] = struct{}{} } - if !skippableMatcher.Contains(test) { + match := skippableMatcher.Match(test) + if !match.Skipped() { slog.Debug("Test is not skipped", "test", test.DatadogTestId(), "sourceFile", test.SuiteSourceFile) recordRunnableTest(tp.suiteAggregates, test, normalizedSourceFile) } else { slog.Debug("Test will be skipped", "test", test.DatadogTestId(), "sourceFile", test.SuiteSourceFile) recordSkippedTest(tp.suiteAggregates, test, normalizedSourceFile) + tp.recordAppliedSkippable(match, testSuiteKey{Module: test.Module, Suite: test.Suite}) skippableTestsCount++ } } @@ -88,6 +97,19 @@ type skippableMatcher struct { disabledTests map[string]bool } +type skippableMatch int + +const ( + skippableMatchNone skippableMatch = iota + skippableMatchTIATest + skippableMatchTIASuite + skippableMatchDisabledTest +) + +func (m skippableMatch) Skipped() bool { + return m != skippableMatchNone +} + func newSkippableMatcher(skippables api.Skippables, disabledTests map[string]bool) skippableMatcher { if skippables.Tests == nil { skippables.Tests = api.SkippableTests{} @@ -107,10 +129,21 @@ func newSkippableMatcher(skippables api.Skippables, disabledTests map[string]boo } func (s skippableMatcher) Contains(test testoptimization.Test) bool { + return s.Match(test).Skipped() +} + +func (s skippableMatcher) Match(test testoptimization.Test) skippableMatch { suite := api.SkippableSuite{Module: test.Module, Suite: test.Suite} - return s.tiaSkippableTests[test.DatadogTestId()] || - s.tiaSkippableSuites[suite] || - s.disabledTests[test.FQN()] + if s.disabledTests[test.FQN()] { + return skippableMatchDisabledTest + } + if s.tiaSkippableSuites[suite] { + return skippableMatchTIASuite + } + if s.tiaSkippableTests[test.DatadogTestId()] { + return skippableMatchTIATest + } + return skippableMatchNone } func (s skippableMatcher) Count() int { @@ -133,6 +166,24 @@ func (s skippableMatcher) DisabledTestsCount() int { return len(s.disabledTests) } +func (tp *TestPlanner) recordAppliedSkippable(match skippableMatch, key testSuiteKey) { + switch match { + case skippableMatchTIATest: + tp.tiaSkippableTestsApplied++ + case skippableMatchTIASuite: + tp.recordAppliedTIASuite(key) + case skippableMatchDisabledTest: + tp.disabledTestsApplied++ + } +} + +func (tp *TestPlanner) recordAppliedTIASuite(key testSuiteKey) { + if tp.tiaSkippableSuitesApplied == nil { + tp.tiaSkippableSuitesApplied = make(map[testSuiteKey]struct{}) + } + tp.tiaSkippableSuitesApplied[key] = struct{}{} +} + func (tp *TestPlanner) recordFastDiscoveryFallbackFiles(discoveredTestFiles []string) error { excluder, err := discovery.NewExcluder(settings.GetTestsExcludePattern()) if err != nil { @@ -190,12 +241,13 @@ func (tp *TestPlanner) recordSuiteLevelSkippables(skippableMatcher skippableMatc } aggregate.NumTestsSkipped = aggregate.NumTests tp.suiteAggregates[key] = aggregate + tp.recordAppliedTIASuite(key) } } -func (tp *TestPlanner) keepUnskippableMarkerSuitesRunnable(testFramework framework.Framework) { +func (tp *TestPlanner) keepUnskippableMarkerSuitesRunnable(testFramework framework.Framework) int { if testFramework == nil { - return + return 0 } startTime := time.Now() @@ -224,6 +276,7 @@ func (tp *TestPlanner) keepUnskippableMarkerSuitesRunnable(testFramework framewo slog.Info("Checked unskippable marker suites", "duration", time.Since(startTime), "forceRunnableSuiteAggregatesCount", forceRunnableSuiteAggregatesCount) + return forceRunnableSuiteAggregatesCount } func (tp *TestPlanner) sourceFileForSuite(key testSuiteKey, testFramework framework.Framework) (string, bool) { diff --git a/internal/planner/planner.go b/internal/planner/planner.go index f355ec2..bcddcc9 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -69,22 +69,29 @@ func (p PlanInfo) IsZero() bool { } type TestPlanner struct { - testFiles map[string]struct{} - suiteAggregates map[testSuiteKey]testSuiteAggregate - suitesBySourceFile map[string][]testSuiteKey - testSuiteDurations map[string]map[string]api.TestSuiteDurationInfo - testFileWeights map[string]int - testFileDurationSources map[string]testFileDurationSource - skippablePercentage float64 - planReport planReport - planLoaded bool - runInfo runmetadata.RunInfo - planInfo PlanInfo - platformDetector platform.PlatformDetector - optimizationClient testOptimizationClient - newOptimizationClient func(testSkippingLevel settings.TestSkippingLevel) testOptimizationClient - ciProviderDetector environment.CIProviderDetector - reportWriter io.Writer + testFiles map[string]struct{} + suiteAggregates map[testSuiteKey]testSuiteAggregate + suitesBySourceFile map[string][]testSuiteKey + testSuiteDurations map[string]map[string]api.TestSuiteDurationInfo + testFileWeights map[string]int + testFileDurationSources map[string]testFileDurationSource + discoveryMode discoveryMode + localDiscoveredSuites int + localDiscoveredTests int + tiaSkippableTestsApplied int + tiaSkippableSuitesApplied map[testSuiteKey]struct{} + disabledTestsApplied int + unskippableMarkerSuitesForced int + skippablePercentage float64 + planReport planReport + planLoaded bool + runInfo runmetadata.RunInfo + planInfo PlanInfo + platformDetector platform.PlatformDetector + optimizationClient testOptimizationClient + newOptimizationClient func(testSkippingLevel settings.TestSkippingLevel) testOptimizationClient + ciProviderDetector environment.CIProviderDetector + reportWriter io.Writer } const ( @@ -172,14 +179,15 @@ func NewWithDependencies( func newTestPlannerWithDefaults() *TestPlanner { return &TestPlanner{ - testFiles: make(map[string]struct{}), - suiteAggregates: make(map[testSuiteKey]testSuiteAggregate), - suitesBySourceFile: make(map[string][]testSuiteKey), - testSuiteDurations: make(map[string]map[string]api.TestSuiteDurationInfo), - testFileWeights: make(map[string]int), - testFileDurationSources: make(map[string]testFileDurationSource), - skippablePercentage: 0.0, - reportWriter: os.Stderr, + testFiles: make(map[string]struct{}), + suiteAggregates: make(map[testSuiteKey]testSuiteAggregate), + suitesBySourceFile: make(map[string][]testSuiteKey), + testSuiteDurations: make(map[string]map[string]api.TestSuiteDurationInfo), + testFileWeights: make(map[string]int), + testFileDurationSources: make(map[string]testFileDurationSource), + tiaSkippableSuitesApplied: make(map[testSuiteKey]struct{}), + skippablePercentage: 0.0, + reportWriter: os.Stderr, } } @@ -433,6 +441,9 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { if err := tp.recordFullDiscoveryResults(discoveredTests, skipMatcher); err != nil { return err } + tp.discoveryMode = discoveryModeFull + tp.localDiscoveredSuites = len(tp.suiteAggregates) + tp.localDiscoveredTests = countSuiteAggregateTests(tp.suiteAggregates) // if we have data on which tests exist in the local repository, we will aggregate them // into a collection of testSuiteAggregate structs. // This collection is used to calculate the skippable percentage and the weighted test files. @@ -450,6 +461,9 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { if err := tp.recordFastDiscoveryFallbackFiles(discoveredTestFiles); err != nil { return err } + tp.discoveryMode = discoveryModeFast + tp.localDiscoveredSuites = 0 + tp.localDiscoveredTests = 0 tp.addDurationDataForFastDiscoveryFallback() if isSuiteLevelSkipping && tiaSkippingEnabled { tp.recordSuiteLevelSkippables(skipMatcher, testFramework) @@ -459,7 +473,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { "fastDiscoveredTestFilesCount", len(discoveredTestFiles)) } - tp.keepUnskippableMarkerSuitesRunnable(testFramework) + tp.unskippableMarkerSuitesForced = tp.keepUnskippableMarkerSuitesRunnable(testFramework) tp.suitesBySourceFile = indexSuitesBySourceFile(tp.suiteAggregates) tp.skippablePercentage = calculateSavedTimePercentage(tp.suiteAggregates) tp.testFileWeights = tp.calculateFileWeights() @@ -634,6 +648,14 @@ func calculateSavedTimePercentage(suiteAggregates map[testSuiteKey]testSuiteAggr return (totalDuration - estimatedDuration) / totalDuration * 100.0 } +func countSuiteAggregateTests(suiteAggregates map[testSuiteKey]testSuiteAggregate) int { + count := 0 + for _, aggregate := range suiteAggregates { + count += aggregate.NumTests + } + return count +} + func indexSuitesBySourceFile(suiteAggregates map[testSuiteKey]testSuiteAggregate) map[string][]testSuiteKey { sourceFileLookup := make(map[string][]testSuiteKey) for key, aggregate := range suiteAggregates { diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 093cd0d..622c4f6 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -739,11 +739,12 @@ func TestTestPlanner_Setup_WithParallelRunners(t *testing.T) { report := reportOutput.String() for _, expectedReportLine := range []string{ - "Split\n", - " Runners: 1\n", - " Expected wall time:", - " Imbalance:", - " Total estimated runtime:", + "Planning\n", + " Runner split\n", + " Estimated runtime:", + " Runners: 1", + " Expected wall time:", + " Imbalance:", } { if !strings.Contains(report, expectedReportLine) { t.Errorf("Expected report to contain %q, got report: %s", expectedReportLine, report) @@ -863,6 +864,21 @@ func TestTestPlanner_Plan_JestSuiteSkippingFetchesSkippablesWithoutFullDiscovery runner.planReport.Skippables.TIASuites != 1 { t.Errorf("expected suite-level skippables to be reported separately, got %+v", runner.planReport.Skippables) } + if runner.planReport.Planning.Discovery.Mode != discoveryModeFast || + runner.planReport.Planning.Discovery.TestFiles != 2 || + runner.planReport.Planning.Discovery.Suites != 0 { + t.Errorf("expected fast discovery report with files but no local suites, got %+v", runner.planReport.Planning.Discovery) + } + if runner.planReport.Planning.Durations.BackendDurationsApplied != 2 || + runner.planReport.Planning.Durations.BackendSuitesAdded != 2 || + runner.planReport.Planning.Durations.SuitesWithoutDurations != 0 { + t.Errorf("expected backend duration application report for fast discovery, got %+v", runner.planReport.Planning.Durations) + } + if runner.planReport.Planning.Skipping.TIASuites != 1 || + runner.planReport.Planning.Skipping.FullySkippedFiles != 1 || + runner.planReport.Planning.TestFilesToRun != 1 { + t.Errorf("expected suite skip application report, got planning=%+v", runner.planReport.Planning) + } expectedTestFiles := "src/b.test.ts\n" assertFileContent(t, constants.TestFilesOutputPath, expectedTestFiles) @@ -1018,6 +1034,16 @@ func TestTestPlanner_PreparePlanningData_RubySuiteModeForceFullDiscovery(t *test if guarded.NumTests != 1 || guarded.NumTestsSkipped != 0 { t.Fatalf("expected guarded suite to remain runnable, got %+v", guarded) } + if runner.planReport.Planning.Discovery.Mode != discoveryModeFull || + runner.planReport.Planning.Discovery.Suites != 4 || + runner.planReport.Planning.Discovery.Tests != 5 { + t.Errorf("expected full discovery suite/test report, got %+v", runner.planReport.Planning.Discovery) + } + if runner.planReport.Planning.Skipping.TIASuites != 3 || + runner.planReport.Planning.Skipping.UnskippableMarkerSuitesForced != 1 || + runner.planReport.Planning.Skipping.FullySkippedFiles != 1 { + t.Errorf("expected full discovery skipping application report, got %+v", runner.planReport.Planning.Skipping) + } } func TestTestPlanner_PreparePlanningData_ForceFullDiscoveryKeepsRunningWithNoTIASkippables(t *testing.T) { diff --git a/internal/planner/report.go b/internal/planner/report.go index 0fe7fa3..6fe42cb 100644 --- a/internal/planner/report.go +++ b/internal/planner/report.go @@ -25,9 +25,7 @@ func printPlanReport(w io.Writer, report planReport) { reportFprintln(w) printBackendDataReport(w, report) reportFprintln(w) - printPlanningReport(w, report.Planning, report.Skippables) - reportFprintln(w) - printSplitReport(w, report.Split) + printPlanningReport(w, report) reportFprintln(w) printLongSeparateRunnerSuitesReport(w, report.LongSeparateRunnerSuites) reportFprintln(w) @@ -154,24 +152,13 @@ func printBackendDataReport(w io.Writer, report planReport) { reportFprintf(w, " Test suite durations: %s\n", formatTestSuiteDurations(report.TestSuiteDurations)) } -func printPlanningReport(w io.Writer, report planningReport, skippables skippablesReport) { +func printPlanningReport(w io.Writer, report planReport) { reportFprintln(w, "Planning") - reportFprintf(w, " Test files discovered: %s\n", formatCount(report.TestFilesDiscovered)) - reportFprintf(w, " Fully skipped files: %s\n", formatCount(report.FullySkippedFiles)) - reportFprintf(w, " Test Management disabled tests applied: %s\n", formatDisabledTestsApplied(skippables)) - reportFprintf(w, " Test files to run: %s\n", formatCount(report.TestFilesToRun)) - reportFprintf(w, " Duration source: %s known, %s default\n", - formatCount(report.DurationSources.Known), - formatCount(report.DurationSources.Default)) - reportFprintf(w, " Estimated time saved: %.2f%%\n", report.EstimatedTimeSaved) -} - -func printSplitReport(w io.Writer, report splitScore) { - reportFprintln(w, "Split") - reportFprintf(w, " Runners: %s\n", formatCount(report.parallelRunners)) - reportFprintf(w, " Expected wall time: %s\n", formatDuration(report.wallTimeDuration())) - reportFprintf(w, " Imbalance: %s\n", formatDuration(report.imbalanceDuration())) - reportFprintf(w, " Total estimated runtime: %s\n", formatDuration(report.totalRuntimeDuration())) + printDiscoveryPlanningReport(w, report.Planning.Discovery) + printDurationEstimatesPlanningReport(w, report.Planning.Durations) + printSkippingPlanningReport(w, report.DatadogSettings, report.Skippables, report.Planning.Skipping) + printRunSetPlanningReport(w, report.Planning) + printRunnerSplitPlanningReport(w, report.Planning, report.Split) } func printLongSeparateRunnerSuitesReport(w io.Writer, suites []testSuiteTimingReport) { @@ -278,6 +265,106 @@ func reportFprintf(w io.Writer, format string, args ...any) { _, _ = fmt.Fprintf(w, format, args...) } +func printDiscoveryPlanningReport(w io.Writer, discovery discoveryReport) { + if !discovery.Available { + reportFprintln(w, " Discovery: not available") + return + } + + reportFprintln(w, " Discovery") + reportFprintf(w, " Method: %s\n", valueOrNotAvailable(string(discovery.Mode))) + reportFprintf(w, " Test files: %s\n", formatCount(discovery.TestFiles)) + switch discovery.Mode { + case discoveryModeFull: + reportFprintf(w, " Suites discovered: %s\n", formatCount(discovery.Suites)) + reportFprintf(w, " Tests discovered: %s\n", formatCount(discovery.Tests)) + } +} + +func printDurationEstimatesPlanningReport(w io.Writer, durations durationApplicationReport) { + if !durations.Available { + reportFprintln(w, " Duration estimates: not available") + return + } + + reportFprintln(w, " Duration estimates") + reportFprintf(w, " Backend durations used: %s\n", formatCountWithUnit(durations.BackendDurationsApplied, "suite", "suites")) + reportFprintf(w, " Default durations used: %s\n", formatCountWithUnit(durations.SuitesWithoutDurations, "suite", "suites")) + reportFprintf(w, " Backend-only suites added: %s\n", formatCount(durations.BackendSuitesAdded)) +} + +func printSkippingPlanningReport(w io.Writer, datadogSettings datadogSettingsReport, skippables skippablesReport, skipping skippingApplicationReport) { + if !skipping.Available { + reportFprintln(w, " Skipping: not available") + return + } + + reportFprintln(w, " Skipping") + reportFprintf(w, " TIA skippables applied: %s\n", formatAppliedTIASkippables(datadogSettings, skippables, skipping)) + reportFprintf(w, " Disabled tests applied: %s\n", formatAppliedDisabledTests(skippables, skipping)) + reportFprintf(w, " Suites marked unskippable: %s\n", formatCount(skipping.UnskippableMarkerSuitesForced)) + reportFprintf(w, " Files fully skipped: %s\n", formatCount(skipping.FullySkippedFiles)) +} + +func printRunSetPlanningReport(w io.Writer, planning planningReport) { + if !planning.Discovery.Available { + reportFprintln(w, " Run set: not available") + return + } + + reportFprintln(w, " Run set") + reportFprintf(w, " Test files to run: %s\n", formatCount(planning.TestFilesToRun)) + reportFprintf(w, " Estimated time saved: %.2f%%\n", planning.EstimatedTimeSaved) +} + +func printRunnerSplitPlanningReport(w io.Writer, planning planningReport, split splitScore) { + if split.parallelRunners <= 0 { + reportFprintln(w, " Runner split: not available") + return + } + + fullDuration := "not available" + if planning.Durations.ExpectedFullDuration > 0 { + fullDuration = formatDuration(planning.Durations.ExpectedFullDuration) + } + + reportFprintln(w, " Runner split") + reportFprintf(w, " Full runtime: %s\n", fullDuration) + reportFprintf(w, " Estimated runtime: %s\n", formatDuration(split.totalRuntimeDuration())) + reportFprintf(w, " Runners: %s\n", formatCount(split.parallelRunners)) + reportFprintf(w, " Expected wall time: %s\n", formatDuration(split.wallTimeDuration())) + reportFprintf(w, " Imbalance: %s\n", formatDuration(split.imbalanceDuration())) +} + +func formatAppliedTIASkippables(datadogSettings datadogSettingsReport, skippables skippablesReport, skipping skippingApplicationReport) string { + if datadogSettings.Available && !datadogSettings.TestSkipping { + return "disabled" + } + if !skippables.Available { + return "not available" + } + + switch skippables.TestSkippingLevel { + case settings.TestSkippingLevelTest: + return formatCountWithUnit(skipping.TIATests, "test", "tests") + case settings.TestSkippingLevelSuite: + return formatCountWithUnit(skipping.TIASuites, "suite", "suites") + case "": + return "mode not available" + default: + return fmt.Sprintf("%s, %s", + formatCountWithUnit(skipping.TIATests, "test", "tests"), + formatCountWithUnit(skipping.TIASuites, "suite", "suites")) + } +} + +func formatAppliedDisabledTests(skippables skippablesReport, skipping skippingApplicationReport) string { + if !skippables.Available { + return "not available" + } + return formatCountWithUnit(skipping.DisabledTests, "test", "tests") +} + func formatKnownTests(settings datadogSettingsReport, known knownTestsReport) string { if settings.Available && !settings.KnownTests { return "disabled" @@ -300,23 +387,18 @@ func formatTIASkippables(datadogSettings datadogSettingsReport, skippables skipp } switch skippables.TestSkippingLevel { case settings.TestSkippingLevelTest: - return fmt.Sprintf("%s tests", formatCount(skippables.TIATests)) + return formatCountWithUnit(skippables.TIATests, "test", "tests") case settings.TestSkippingLevelSuite: - return fmt.Sprintf("%s suites", formatCount(skippables.TIASuites)) + return formatCountWithUnit(skippables.TIASuites, "suite", "suites") case "": return "skipping mode not available" default: - return fmt.Sprintf("%s tests, %s suites", formatCount(skippables.TIATests), formatCount(skippables.TIASuites)) + return fmt.Sprintf("%s, %s", + formatCountWithUnit(skippables.TIATests, "test", "tests"), + formatCountWithUnit(skippables.TIASuites, "suite", "suites")) } } -func formatDisabledTestsApplied(skippables skippablesReport) string { - if !skippables.Available { - return "not available" - } - return formatCount(skippables.DisabledTests) -} - func formatManagedFlakyTests(settings datadogSettingsReport, managed managedFlakyTestsReport) string { if settings.Available && !settings.FlakyTestManagement { return "disabled" @@ -414,6 +496,14 @@ func formatCount(count int) string { return builder.String() } +func formatCountWithUnit(count int, singular string, plural string) string { + unit := plural + if count == 1 { + unit = singular + } + return formatCount(count) + " " + unit +} + func formatDuration(duration time.Duration) string { if duration < time.Millisecond { return duration.String() diff --git a/internal/planner/report_data.go b/internal/planner/report_data.go index e07157c..96244b6 100644 --- a/internal/planner/report_data.go +++ b/internal/planner/report_data.go @@ -4,6 +4,7 @@ import ( "slices" "time" + "github.com/DataDog/ddtest/internal/constants" "github.com/DataDog/ddtest/internal/runmetadata" "github.com/DataDog/ddtest/internal/settings" "github.com/DataDog/ddtest/internal/testoptimization/api" @@ -133,19 +134,6 @@ func newTestSuiteDurationsReport(testSuiteDurations *api.TestSuiteDurationsRespo return report } -type durationSourceReport struct { - Known int - Default int -} - -type planningReport struct { - TestFilesDiscovered int - FullySkippedFiles int - TestFilesToRun int - DurationSources durationSourceReport - EstimatedTimeSaved float64 -} - type testSuiteTimingReport struct { Runner int Module string @@ -156,6 +144,48 @@ type testSuiteTimingReport struct { DurationSource testFileDurationSource } +type discoveryMode string + +const ( + discoveryModeUnknown discoveryMode = "" + discoveryModeFast discoveryMode = "fast" + discoveryModeFull discoveryMode = "full" +) + +type discoveryReport struct { + Available bool + Mode discoveryMode + TestFiles int + Suites int + Tests int +} + +type durationApplicationReport struct { + Available bool + BackendDurationsApplied int + BackendSuitesAdded int + SuitesWithoutDurations int + FilesWithoutDurations int + ExpectedFullDuration time.Duration +} + +type skippingApplicationReport struct { + Available bool + TIATests int + TIASuites int + DisabledTests int + UnskippableMarkerSuitesForced int + FullySkippedFiles int +} + +type planningReport struct { + Discovery discoveryReport + Durations durationApplicationReport + Skipping skippingApplicationReport + TestFilesToRun int + EstimatedTimeSaved float64 +} + type planReport struct { RunInfo runmetadata.RunInfo PlanInfo PlanInfo @@ -178,25 +208,85 @@ func (tp *TestPlanner) newPlanningReport() planningReport { } return planningReport{ - TestFilesDiscovered: len(tp.testFiles), - FullySkippedFiles: fullySkippedFiles, - TestFilesToRun: len(tp.testFileWeights), - DurationSources: tp.durationSourceReport(), - EstimatedTimeSaved: tp.skippablePercentage, + Discovery: discoveryReport{ + Available: true, + Mode: tp.discoveryMode, + TestFiles: len(tp.testFiles), + Suites: tp.localDiscoveredSuites, + Tests: tp.localDiscoveredTests, + }, + Durations: durationApplicationReport{ + Available: true, + BackendDurationsApplied: tp.backendDurationApplicationsCount(), + BackendSuitesAdded: tp.backendSuitesAddedCount(), + SuitesWithoutDurations: tp.suitesWithoutBackendDurationsCount(), + FilesWithoutDurations: tp.filesWithoutBackendDurationsCount(), + ExpectedFullDuration: tp.expectedFullDuration(), + }, + Skipping: skippingApplicationReport{ + Available: true, + TIATests: tp.tiaSkippableTestsApplied, + TIASuites: len(tp.tiaSkippableSuitesApplied), + DisabledTests: tp.disabledTestsApplied, + UnskippableMarkerSuitesForced: tp.unskippableMarkerSuitesForced, + FullySkippedFiles: fullySkippedFiles, + }, + TestFilesToRun: len(tp.testFileWeights), + EstimatedTimeSaved: tp.skippablePercentage, } } -func (tp *TestPlanner) durationSourceReport() durationSourceReport { - var report durationSourceReport +func (tp *TestPlanner) backendDurationApplicationsCount() int { + count := 0 + for _, aggregate := range tp.suiteAggregates { + if aggregate.DurationSource == testFileDurationSourceKnown { + count++ + } + } + return count +} + +func (tp *TestPlanner) backendSuitesAddedCount() int { + count := len(tp.suiteAggregates) - tp.localDiscoveredSuites + if count < 0 { + return 0 + } + return count +} + +func (tp *TestPlanner) suitesWithoutBackendDurationsCount() int { + count := 0 + for _, aggregate := range tp.suiteAggregates { + if aggregate.DurationSource != testFileDurationSourceKnown { + count++ + } + } + return count +} + +func (tp *TestPlanner) filesWithoutBackendDurationsCount() int { + count := 0 for _, source := range tp.testFileDurationSources { - switch source { - case testFileDurationSourceKnown: - report.Known++ - default: - report.Default++ + if source != testFileDurationSourceKnown { + count++ } } - return report + return count +} + +func (tp *TestPlanner) expectedFullDuration() time.Duration { + var total float64 + for testFile := range tp.testFiles { + suiteKeys := tp.suitesBySourceFile[testFile] + if len(suiteKeys) == 0 { + total += float64(time.Duration(constants.DefaultTestFileWeight) * time.Millisecond) + continue + } + for _, key := range suiteKeys { + total += tp.suiteAggregates[key].TotalDuration + } + } + return durationFromNanoseconds(total) } func (tp *TestPlanner) longSeparateRunnerSuitesReport(parallelRunners int, split splitScore) []testSuiteTimingReport { diff --git a/internal/planner/report_test.go b/internal/planner/report_test.go index 68e1e80..1a97762 100644 --- a/internal/planner/report_test.go +++ b/internal/planner/report_test.go @@ -91,13 +91,29 @@ func TestPrintPlanReport_AllData(t *testing.T) { Suites: 1491, }, Planning: planningReport{ - TestFilesDiscovered: 642, - FullySkippedFiles: 118, - TestFilesToRun: 524, - DurationSources: durationSourceReport{ - Known: 431, - Default: 90, + Discovery: discoveryReport{ + Available: true, + Mode: discoveryModeFull, + TestFiles: 642, + Suites: 1284, + Tests: 18921, }, + Durations: durationApplicationReport{ + Available: true, + BackendDurationsApplied: 431, + BackendSuitesAdded: 12, + SuitesWithoutDurations: 90, + FilesWithoutDurations: 90, + ExpectedFullDuration: 37*time.Minute + 12*time.Second, + }, + Skipping: skippingApplicationReport{ + Available: true, + TIASuites: 312, + DisabledTests: 3, + UnskippableMarkerSuitesForced: 5, + FullySkippedFiles: 118, + }, + TestFilesToRun: 524, EstimatedTimeSaved: 38.4, }, Split: splitScore{ @@ -183,18 +199,29 @@ Backend data Test suite durations: 3 modules, 1,491 suites Planning - Test files discovered: 642 - Fully skipped files: 118 - Test Management disabled tests applied: 3 - Test files to run: 524 - Duration source: 431 known, 90 default - Estimated time saved: 38.40%% - -Split - Runners: 6 - Expected wall time: 4m12s - Imbalance: 11s - Total estimated runtime: 23m46s + Discovery + Method: full + Test files: 642 + Suites discovered: 1,284 + Tests discovered: 18,921 + Duration estimates + Backend durations used: 431 suites + Default durations used: 90 suites + Backend-only suites added: 12 + Skipping + TIA skippables applied: 312 suites + Disabled tests applied: 3 tests + Suites marked unskippable: 5 + Files fully skipped: 118 + Run set + Test files to run: 524 + Estimated time saved: 38.40%% + Runner split + Full runtime: 37m12s + Estimated runtime: 23m46s + Runners: 6 + Expected wall time: 4m12s + Imbalance: 11s Slow suites on dedicated runners ATTENTION: 1 dedicated runner @@ -209,6 +236,150 @@ Slow suites on dedicated runners } } +func TestPrintPlanReport_FastDiscovery(t *testing.T) { + var output strings.Builder + + printPlanReport(&output, planReport{ + RunInfo: runmetadata.RunInfo{ + Service: "checkout-api", + }, + PlanInfo: PlanInfo{ + Platform: "ruby", + }, + DatadogSettings: datadogSettingsReport{ + Available: true, + TestImpactAnalysis: true, + TestSkipping: true, + TestImpactCollection: true, + KnownTests: true, + }, + KnownTests: knownTestsReport{ + Available: true, + Modules: 1, + Suites: 10, + Tests: 125, + }, + Skippables: skippablesReport{ + Available: true, + TestSkippingLevel: settings.TestSkippingLevelSuite, + TIASuites: 8, + }, + TestSuiteDurations: testSuiteDurationsReport{ + Available: true, + Modules: 1, + Suites: 12, + }, + Planning: planningReport{ + Discovery: discoveryReport{ + Available: true, + Mode: discoveryModeFast, + TestFiles: 24, + Suites: 0, + }, + Durations: durationApplicationReport{ + Available: true, + BackendDurationsApplied: 12, + BackendSuitesAdded: 2, + SuitesWithoutDurations: 1, + }, + Skipping: skippingApplicationReport{ + Available: true, + TIASuites: 8, + FullySkippedFiles: 6, + }, + TestFilesToRun: 18, + EstimatedTimeSaved: 25, + }, + Split: splitScore{ + parallelRunners: 3, + wallTime: 90_000, + imbalance: 500, + totalRuntime: 210_000, + }, + SlowestTestSuitesOverall: []testSuiteTimingReport{ + { + Module: "rspec", + Suite: "Checkout::Order", + SourceFile: "spec/models/order_spec.rb", + TotalDuration: 2*time.Minute + 30*time.Second, + EstimatedDuration: 105 * time.Second, + DurationSource: testFileDurationSourceKnown, + }, + { + Module: "rspec", + Suite: "Checkout::Payment", + SourceFile: "spec/models/payment_spec.rb", + TotalDuration: time.Minute, + EstimatedDuration: 30 * time.Second, + DurationSource: testFileDurationSourceKnown, + }, + }, + }) + + expected := `+++ DDTest: plan report + +Run + Service: checkout-api + Repository: not available + Commit: not available + Branch: not available + Platform: ruby + OS tags: not available + Runtime tags: not available + +DDTest settings + Settings: not available + +Datadog settings + Test Impact Analysis: enabled + Test skipping: enabled + Test impact collection: enabled + Known tests: enabled + Early flake detection: disabled + Auto test retries: disabled + Flaky test management: disabled + +Backend data + Known tests: 1 modules, 10 suites, 125 tests + TIA skippables returned: 8 suites + Managed flaky tests: disabled + Test suite durations: 1 modules, 12 suites + +Planning + Discovery + Method: fast + Test files: 24 + Duration estimates + Backend durations used: 12 suites + Default durations used: 1 suite + Backend-only suites added: 2 + Skipping + TIA skippables applied: 8 suites + Disabled tests applied: 0 tests + Suites marked unskippable: 0 + Files fully skipped: 6 + Run set + Test files to run: 18 + Estimated time saved: 25.00% + Runner split + Full runtime: not available + Estimated runtime: 3m30s + Runners: 3 + Expected wall time: 1m30s + Imbalance: 500ms + +Slow suites on dedicated runners + None + +10 slowest test suites overall + 1. rspec / Checkout::Order (spec/models/order_spec.rb): historical duration 2m30s, estimated runtime 1m45s + 2. rspec / Checkout::Payment (spec/models/payment_spec.rb): historical duration 1m0s, estimated runtime 30s +` + if output.String() != expected { + t.Errorf("unexpected fast discovery report:\n%s", output.String()) + } +} + func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { var output strings.Builder @@ -221,21 +392,33 @@ func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { if !strings.Contains(report, " Settings: not available") { t.Errorf("expected missing settings message, got:\n%s", report) } - if !strings.Contains(report, " Known tests: not available") { - t.Errorf("expected missing known tests message, got:\n%s", report) + if !strings.Contains(report, "Planning\n Discovery: not available") { + t.Errorf("expected missing discovery message, got:\n%s", report) + } + if !strings.Contains(report, "Backend data\n Known tests: not available") { + t.Errorf("expected missing backend data message, got:\n%s", report) } if !strings.Contains(report, " TIA skippables returned: not available") { t.Errorf("expected missing TIA skippables message, got:\n%s", report) } - if !strings.Contains(report, " Test Management disabled tests applied: not available") { - t.Errorf("expected missing disabled tests message, got:\n%s", report) - } if !strings.Contains(report, " Managed flaky tests: not available") { t.Errorf("expected missing managed flaky tests message, got:\n%s", report) } if !strings.Contains(report, " Test suite durations: not available") { t.Errorf("expected missing test suite durations message, got:\n%s", report) } + if !strings.Contains(report, " Duration estimates: not available") { + t.Errorf("expected missing duration estimates message, got:\n%s", report) + } + if !strings.Contains(report, " Skipping: not available") { + t.Errorf("expected missing skipping message, got:\n%s", report) + } + if !strings.Contains(report, " Run set: not available") { + t.Errorf("expected missing run set message, got:\n%s", report) + } + if !strings.Contains(report, " Runner split: not available") { + t.Errorf("expected missing runner split message, got:\n%s", report) + } } func TestPrintPlanReport_DefaultSettings(t *testing.T) { @@ -409,6 +592,141 @@ func TestFormatTIASkippablesReportsOnlyActiveCount(t *testing.T) { } } +func TestReportFormattingVariants(t *testing.T) { + t.Run("suite labels", func(t *testing.T) { + tests := []struct { + name string + suite testSuiteTimingReport + want string + }{ + {name: "missing", suite: testSuiteTimingReport{}, want: "not available"}, + {name: "module only", suite: testSuiteTimingReport{Module: "rspec"}, want: "rspec"}, + {name: "suite only", suite: testSuiteTimingReport{Suite: "CartSuite"}, want: "CartSuite"}, + {name: "module and suite", suite: testSuiteTimingReport{Module: "rspec", Suite: "CartSuite"}, want: "rspec / CartSuite"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := formatSuiteLabel(tt.suite); got != tt.want { + t.Fatalf("formatSuiteLabel() = %q, want %q", got, tt.want) + } + }) + } + }) + + t.Run("basic values", func(t *testing.T) { + tests := []struct { + name string + got string + want string + }{ + {name: "plural dedicated runners", got: formatScheduledTestSuiteCount(2), want: "2 dedicated runners"}, + {name: "empty worker env", got: formatWorkerEnvKeys(" "), want: "not set"}, + {name: "platform only", got: formatPlatform("ruby", ""), want: "ruby"}, + {name: "framework only", got: formatPlatform("", "rspec"), want: "rspec"}, + {name: "empty setting", got: valueOrNotSet(""), want: "not set"}, + {name: "negative count", got: formatCount(-123456), want: "-123,456"}, + {name: "three digit group count", got: formatCount(123456), want: "123,456"}, + {name: "singular count unit", got: formatCountWithUnit(1, "suite", "suites"), want: "1 suite"}, + {name: "sub-millisecond duration", got: formatDuration(500 * time.Microsecond), want: "500µs"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.got != tt.want { + t.Fatalf("got %q, want %q", tt.got, tt.want) + } + }) + } + }) + + t.Run("ddtest setting fallback formatting", func(t *testing.T) { + unnamedField := reflect.StructField{Name: "CustomSetting"} + if got := formatDDTestSettingName(unnamedField); got != "CustomSetting" { + t.Fatalf("formatDDTestSettingName() = %q, want CustomSetting", got) + } + + field := reflect.StructField{Name: "CustomSetting", Tag: `mapstructure:"custom_setting"`} + value := reflect.ValueOf(struct{ Enabled bool }{Enabled: true}) + if got := formatDDTestSettingValue(field, value); got != "{true}" { + t.Fatalf("formatDDTestSettingValue() = %q, want {true}", got) + } + }) +} + +func TestSkippableReportFormattingVariants(t *testing.T) { + t.Run("applied TIA skippables", func(t *testing.T) { + tests := []struct { + name string + datadogSettings datadogSettingsReport + skippables skippablesReport + skipping skippingApplicationReport + want string + }{ + { + name: "disabled", + datadogSettings: datadogSettingsReport{Available: true, TestSkipping: false}, + skippables: skippablesReport{Available: true, TestSkippingLevel: settings.TestSkippingLevelTest}, + skipping: skippingApplicationReport{TIATests: 4}, + want: "disabled", + }, + { + name: "not available", + skipping: skippingApplicationReport{TIATests: 4}, + want: "not available", + }, + { + name: "test level", + skippables: skippablesReport{Available: true, TestSkippingLevel: settings.TestSkippingLevelTest}, + skipping: skippingApplicationReport{TIATests: 1}, + want: "1 test", + }, + { + name: "mode not available", + skippables: skippablesReport{Available: true}, + skipping: skippingApplicationReport{TIATests: 2}, + want: "mode not available", + }, + { + name: "mixed mode fallback", + skippables: skippablesReport{Available: true, TestSkippingLevel: settings.TestSkippingLevel("mixed")}, + skipping: skippingApplicationReport{TIATests: 2, TIASuites: 3}, + want: "2 tests, 3 suites", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := formatAppliedTIASkippables(tt.datadogSettings, tt.skippables, tt.skipping) + if got != tt.want { + t.Fatalf("formatAppliedTIASkippables() = %q, want %q", got, tt.want) + } + }) + } + }) + + t.Run("returned TIA skippables", func(t *testing.T) { + got := formatTIASkippables( + datadogSettingsReport{Available: true, TestSkipping: true}, + skippablesReport{Available: true}, + ) + if got != "skipping mode not available" { + t.Fatalf("formatTIASkippables() = %q, want skipping mode not available", got) + } + + got = formatTIASkippables( + datadogSettingsReport{Available: true, TestSkipping: true}, + skippablesReport{Available: true, TestSkippingLevel: settings.TestSkippingLevel("mixed"), TIATests: 1, TIASuites: 2}, + ) + if got != "1 test, 2 suites" { + t.Fatalf("formatTIASkippables() = %q, want mixed fallback", got) + } + }) + + t.Run("disabled tests", func(t *testing.T) { + if got := formatAppliedDisabledTests(skippablesReport{}, skippingApplicationReport{}); got != "not available" { + t.Fatalf("formatAppliedDisabledTests() = %q, want not available", got) + } + }) +} + func TestFormatWorkerEnvKeys(t *testing.T) { report := formatWorkerEnvKeys("TOKEN=secret; RAILS_ENV = test ;BAD_NO_EQUALS;=NO_KEY;TOKEN=other") From 0bebb65cf4596ffda9060dcc33f2540b5855144c Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 30 Jun 2026 11:42:31 +0200 Subject: [PATCH 5/7] Report discovery cache usage --- internal/planner/discovered_tests.go | 1 + internal/planner/discovery_cache.go | 20 ++++++++++++++------ internal/planner/discovery_cache_test.go | 15 ++++++++++++--- internal/planner/planner.go | 16 +++++++++++++++- internal/planner/planner_test.go | 6 ++++++ internal/planner/report.go | 14 ++++++++++++++ internal/planner/report_data.go | 8 ++++++++ internal/planner/report_test.go | 14 ++++++++++++++ 8 files changed, 84 insertions(+), 10 deletions(-) diff --git a/internal/planner/discovered_tests.go b/internal/planner/discovered_tests.go index 9a387cc..55c60a2 100644 --- a/internal/planner/discovered_tests.go +++ b/internal/planner/discovered_tests.go @@ -18,6 +18,7 @@ func (tp *TestPlanner) resetDiscoveryResults() { tp.suiteAggregates = make(map[testSuiteKey]testSuiteAggregate) tp.suitesBySourceFile = make(map[string][]testSuiteKey) tp.discoveryMode = discoveryModeUnknown + tp.discoveryCacheReport = discoveryCacheReport{} tp.localDiscoveredSuites = 0 tp.localDiscoveredTests = 0 tp.tiaSkippableTestsApplied = 0 diff --git a/internal/planner/discovery_cache.go b/internal/planner/discovery_cache.go index cc573fb..e1b1ea4 100644 --- a/internal/planner/discovery_cache.go +++ b/internal/planner/discovery_cache.go @@ -208,31 +208,39 @@ func readLastNonEmptyLine(filePath string) ([]byte, error) { } } -func (c discoveryCache) restore() ([]testoptimization.Test, bool) { +func (c discoveryCache) restore() ([]testoptimization.Test, discoveryCacheReport) { + report := discoveryCacheReport{ + Configured: settings.GetTestDiscoveryCache() != "", + } + c.importExternal() if err := c.validate(); err != nil { - if settings.GetTestDiscoveryCache() != "" { + report.Reason = err.Error() + if report.Configured { slog.Info("Cached test discovery not usable; full discovery will run", "reason", err) } else { slog.Debug("Cached test discovery not usable; full discovery will run", "reason", err) } - return nil, false + return nil, report } startTime := time.Now() tests, err := parseCachedDiscoveryTests(c.filePath) if err != nil { + report.Reason = err.Error() slog.Info("Cached test discovery could not be used; full discovery will run", "error", err) - return nil, false + return nil, report } if err := ensureDiscoveredTests(tests); err != nil { + report.Reason = err.Error() slog.Info("Cached test discovery could not be used; full discovery will run", "error", err) - return nil, false + return nil, report } + report.Used = true slog.Info("Cached test discovery succeeded", "duration", time.Since(startTime), "count", len(tests)) - return tests, true + return tests, report } func ensureDiscoveredTests(tests []testoptimization.Test) error { diff --git a/internal/planner/discovery_cache_test.go b/internal/planner/discovery_cache_test.go index 3a9255c..59d1438 100644 --- a/internal/planner/discovery_cache_test.go +++ b/internal/planner/discovery_cache_test.go @@ -192,6 +192,9 @@ func TestDiscoveryCacheHitUsesCachedTests(t *testing.T) { if len(mockFramework.DiscoverTestsFiles) != 0 { t.Fatalf("expected cache hit to avoid full discovery, got %d calls", len(mockFramework.DiscoverTestsFiles)) } + if !runner.planReport.Planning.Discovery.Cache.Used { + t.Fatalf("expected cache hit to be reported, got %+v", runner.planReport.Planning.Discovery.Cache) + } aggregate := runner.suiteAggregates[testSuiteKey{Module: "rspec", Suite: "Cart"}] if aggregate.NumTests != 1 || aggregate.NumTestsSkipped != 1 { t.Fatalf("cached aggregate = %+v, want one skipped test", aggregate) @@ -290,6 +293,9 @@ func TestDiscoveryCacheImportsExternalCacheBeforeValidation(t *testing.T) { if len(mockFramework.DiscoverTestsFiles) != 0 { t.Fatalf("expected imported cache to avoid full discovery, got %d calls", len(mockFramework.DiscoverTestsFiles)) } + if cacheReport := runner.planReport.Planning.Discovery.Cache; !cacheReport.Configured || !cacheReport.Used { + t.Fatalf("expected imported cache to be reported as configured and used, got %+v", cacheReport) + } if _, err := os.Stat(discovery.TestsFilePath); err != nil { t.Fatalf("expected imported cache at internal path: %v", err) } @@ -303,9 +309,12 @@ func TestDiscoveryCacheRestoreRejectsEmptyCache(t *testing.T) { writePlannerDiscoveryCache(t, "base-sha", "ruby", "rspec", pattern, "", nil) cache := newDiscoveryCache("ruby", &MockFramework{FrameworkName: "rspec", TestPatternValue: pattern}) - tests, ok := cache.restore() - if ok || tests != nil { - t.Fatalf("expected empty cache restore to fail, got ok=%v tests=%v", ok, tests) + tests, report := cache.restore() + if report.Used || tests != nil { + t.Fatalf("expected empty cache restore to fail, got report=%+v tests=%v", report, tests) + } + if report.Reason != "test discovery returned no tests" { + t.Fatalf("unexpected empty cache restore reason: %q", report.Reason) } } diff --git a/internal/planner/planner.go b/internal/planner/planner.go index bcddcc9..63f26e0 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -76,6 +76,7 @@ type TestPlanner struct { testFileWeights map[string]int testFileDurationSources map[string]testFileDurationSource discoveryMode discoveryMode + discoveryCacheReport discoveryCacheReport localDiscoveredSuites int localDiscoveredTests int tiaSkippableTestsApplied int @@ -378,17 +379,22 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { if !fullDiscoveryNeeded { if isSuiteLevelSkipping && !forceFullTestDiscovery { slog.Info("Suite-level skipping does not require full test discovery; using fast test file discovery fallback", "framework", testFramework.Name()) + tp.discoveryCacheReport = newDiscoveryCacheReport(false, "full discovery not required for suite-level skipping") return nil } if forceFullTestDiscovery && !fullTestDiscoverySupported { slog.Warn("Full test discovery was forced but is not supported by framework; using fast test file discovery fallback", "framework", testFramework.Name()) + tp.discoveryCacheReport = newDiscoveryCacheReport(false, "full discovery not supported by framework") return nil } slog.Info("Full test discovery is not supported by framework; using fast test file discovery fallback", "framework", testFramework.Name()) + tp.discoveryCacheReport = newDiscoveryCacheReport(false, "full discovery not supported by framework") return nil } - if res, ok := discoveryCache.restore(); ok { + res, cacheReport := discoveryCache.restore() + tp.discoveryCacheReport = cacheReport + if cacheReport.Used { discoveredTests = res fullDiscoverySucceeded = true return nil @@ -510,6 +516,14 @@ func discoverLocalTests(ctx context.Context, testFramework framework.Framework, return tests, nil } +func newDiscoveryCacheReport(used bool, reason string) discoveryCacheReport { + return discoveryCacheReport{ + Configured: settings.GetTestDiscoveryCache() != "", + Used: used, + Reason: reason, + } +} + func (tp *TestPlanner) fetchSkippables(tiaSkippingEnabled bool) skippableMatcher { startTime := time.Now() slog.Info("Fetching skippables from Datadog...") diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 622c4f6..7a43c53 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -818,6 +818,7 @@ func TestTestPlanner_Plan_JestSuiteSkippingFetchesSkippablesWithoutFullDiscovery t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MIN_PARALLELISM", "1") t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_MAX_PARALLELISM", "1") t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_REPORT_ENABLED", "false") + t.Setenv("DD_TEST_OPTIMIZATION_RUNNER_TEST_DISCOVERY_CACHE", filepath.Join(tempDir, "test-discovery-cache.json")) settings.Init() mockFramework := &MockFramework{ @@ -869,6 +870,11 @@ func TestTestPlanner_Plan_JestSuiteSkippingFetchesSkippablesWithoutFullDiscovery runner.planReport.Planning.Discovery.Suites != 0 { t.Errorf("expected fast discovery report with files but no local suites, got %+v", runner.planReport.Planning.Discovery) } + if cacheReport := runner.planReport.Planning.Discovery.Cache; !cacheReport.Configured || + cacheReport.Used || + cacheReport.Reason != "full discovery not required for suite-level skipping" { + t.Errorf("expected configured cache skip reason for fast discovery, got %+v", cacheReport) + } if runner.planReport.Planning.Durations.BackendDurationsApplied != 2 || runner.planReport.Planning.Durations.BackendSuitesAdded != 2 || runner.planReport.Planning.Durations.SuitesWithoutDurations != 0 { diff --git a/internal/planner/report.go b/internal/planner/report.go index 6fe42cb..c5e249e 100644 --- a/internal/planner/report.go +++ b/internal/planner/report.go @@ -274,6 +274,7 @@ func printDiscoveryPlanningReport(w io.Writer, discovery discoveryReport) { reportFprintln(w, " Discovery") reportFprintf(w, " Method: %s\n", valueOrNotAvailable(string(discovery.Mode))) reportFprintf(w, " Test files: %s\n", formatCount(discovery.TestFiles)) + reportFprintf(w, " Cache: %s\n", formatDiscoveryCache(discovery.Cache)) switch discovery.Mode { case discoveryModeFull: reportFprintf(w, " Suites discovered: %s\n", formatCount(discovery.Suites)) @@ -281,6 +282,19 @@ func printDiscoveryPlanningReport(w io.Writer, discovery discoveryReport) { } } +func formatDiscoveryCache(cache discoveryCacheReport) string { + if cache.Used { + return "used" + } + if !cache.Configured { + return "not configured" + } + if cache.Reason == "" { + return "not used" + } + return "not used (" + cache.Reason + ")" +} + func printDurationEstimatesPlanningReport(w io.Writer, durations durationApplicationReport) { if !durations.Available { reportFprintln(w, " Duration estimates: not available") diff --git a/internal/planner/report_data.go b/internal/planner/report_data.go index 96244b6..6c9a865 100644 --- a/internal/planner/report_data.go +++ b/internal/planner/report_data.go @@ -155,11 +155,18 @@ const ( type discoveryReport struct { Available bool Mode discoveryMode + Cache discoveryCacheReport TestFiles int Suites int Tests int } +type discoveryCacheReport struct { + Configured bool + Used bool + Reason string +} + type durationApplicationReport struct { Available bool BackendDurationsApplied int @@ -211,6 +218,7 @@ func (tp *TestPlanner) newPlanningReport() planningReport { Discovery: discoveryReport{ Available: true, Mode: tp.discoveryMode, + Cache: tp.discoveryCacheReport, TestFiles: len(tp.testFiles), Suites: tp.localDiscoveredSuites, Tests: tp.localDiscoveredTests, diff --git a/internal/planner/report_test.go b/internal/planner/report_test.go index 1a97762..817239b 100644 --- a/internal/planner/report_test.go +++ b/internal/planner/report_test.go @@ -94,6 +94,10 @@ func TestPrintPlanReport_AllData(t *testing.T) { Discovery: discoveryReport{ Available: true, Mode: discoveryModeFull, + Cache: discoveryCacheReport{ + Configured: true, + Used: true, + }, TestFiles: 642, Suites: 1284, Tests: 18921, @@ -202,6 +206,7 @@ Planning Discovery Method: full Test files: 642 + Cache: used Suites discovered: 1,284 Tests discovered: 18,921 Duration estimates @@ -273,6 +278,10 @@ func TestPrintPlanReport_FastDiscovery(t *testing.T) { Discovery: discoveryReport{ Available: true, Mode: discoveryModeFast, + Cache: discoveryCacheReport{ + Configured: true, + Reason: "full discovery not required for suite-level skipping", + }, TestFiles: 24, Suites: 0, }, @@ -349,6 +358,7 @@ Planning Discovery Method: fast Test files: 24 + Cache: not used (full discovery not required for suite-level skipping) Duration estimates Backend durations used: 12 suites Default durations used: 1 suite @@ -628,6 +638,10 @@ func TestReportFormattingVariants(t *testing.T) { {name: "three digit group count", got: formatCount(123456), want: "123,456"}, {name: "singular count unit", got: formatCountWithUnit(1, "suite", "suites"), want: "1 suite"}, {name: "sub-millisecond duration", got: formatDuration(500 * time.Microsecond), want: "500µs"}, + {name: "cache not configured", got: formatDiscoveryCache(discoveryCacheReport{}), want: "not configured"}, + {name: "cache used", got: formatDiscoveryCache(discoveryCacheReport{Configured: true, Used: true}), want: "used"}, + {name: "cache configured but not used", got: formatDiscoveryCache(discoveryCacheReport{Configured: true}), want: "not used"}, + {name: "cache configured but skipped with reason", got: formatDiscoveryCache(discoveryCacheReport{Configured: true, Reason: "full discovery not required"}), want: "not used (full discovery not required)"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 1e9c00c80314eb7099857a1f73a629856cb0387c Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 30 Jun 2026 11:51:28 +0200 Subject: [PATCH 6/7] Report planning operation durations --- internal/planner/discovered_tests.go | 1 + internal/planner/planner.go | 31 ++++++-- internal/planner/planner_test.go | 5 ++ internal/planner/report.go | 31 +++++++- internal/planner/report_data.go | 77 ++++++++++++------- internal/planner/report_test.go | 75 +++++++++++------- internal/testoptimization/testoptimization.go | 55 +++++++++++++ .../testoptimization_lifecycle_test.go | 4 + .../testoptimization/testoptimization_test.go | 12 +++ 9 files changed, 224 insertions(+), 67 deletions(-) diff --git a/internal/planner/discovered_tests.go b/internal/planner/discovered_tests.go index 55c60a2..5bbf30d 100644 --- a/internal/planner/discovered_tests.go +++ b/internal/planner/discovered_tests.go @@ -19,6 +19,7 @@ func (tp *TestPlanner) resetDiscoveryResults() { tp.suitesBySourceFile = make(map[string][]testSuiteKey) tp.discoveryMode = discoveryModeUnknown tp.discoveryCacheReport = discoveryCacheReport{} + tp.testDiscoveryDuration = 0 tp.localDiscoveredSuites = 0 tp.localDiscoveredTests = 0 tp.tiaSkippableTestsApplied = 0 diff --git a/internal/planner/planner.go b/internal/planner/planner.go index 63f26e0..c78b8cb 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -39,6 +39,7 @@ type testOptimizationClient interface { GetTestManagementTestsData() *api.TestManagementTestsResponseDataModules GetDisabledTests() map[string]bool GetTestSuiteDurations() *api.TestSuiteDurationsResponseData + OperationDurations() testoptimization.OperationDurations StoreCacheAndExit() } @@ -77,6 +78,7 @@ type TestPlanner struct { testFileDurationSources map[string]testFileDurationSource discoveryMode discoveryMode discoveryCacheReport discoveryCacheReport + testDiscoveryDuration time.Duration localDiscoveredSuites int localDiscoveredTests int tiaSkippableTestsApplied int @@ -316,6 +318,8 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { var fullDiscoveryErr error var fastDiscoveryErr error var tiaSkippingEnabled bool + var fullDiscoveryDuration time.Duration + var fastDiscoveryDuration time.Duration tp.resetDiscoveryResults() tp.testSuiteDurations = make(map[string]map[string]api.TestSuiteDurationInfo) @@ -340,7 +344,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } repositorySettings := tp.optimizationClient.GetSettings() - tp.planReport.DatadogSettings = newDatadogSettingsReport(repositorySettings) + tp.planReport.DatadogSettings = newDatadogSettingsReport(repositorySettings, tp.optimizationClient.OperationDurations().Settings) tiaSkippingEnabled = false if repositorySettings != nil { slog.Debug("Repository settings", "tia_enabled", repositorySettings.ItrEnabled, "tests_skipping", repositorySettings.TestsSkipping) @@ -358,7 +362,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } skipMatcher = tp.fetchSkippables(tiaSkippingEnabled) - tp.planReport.Skippables = newSkippablesReport(skipMatcher, testSkippingLevel) + tp.planReport.Skippables = newSkippablesReport(skipMatcher, testSkippingLevel, tp.optimizationClient.OperationDurations().Skippables) if tiaSkippingEnabled && skipMatcher.TIASkippablesCount() == 0 && !forceFullTestDiscovery { slog.Info("No TIA-skippable tests or suites found for this run, cancelling full test discovery") @@ -366,7 +370,10 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } testSuiteDurations := tp.optimizationClient.GetTestSuiteDurations() - tp.planReport.TestSuiteDurations = newTestSuiteDurationsReport(testSuiteDurations) + tp.planReport.TestSuiteDurations = newTestSuiteDurationsReport( + testSuiteDurations, + tp.optimizationClient.OperationDurations().TestSuiteDurations, + ) if testSuiteDurations != nil && testSuiteDurations.TestSuites != nil { tp.testSuiteDurations = testSuiteDurations.TestSuites } @@ -376,6 +383,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { // Goroutine 2: Tests discovery (respects context cancellation) g.Go(func() error { + fullDiscoveryStartTime := time.Now() if !fullDiscoveryNeeded { if isSuiteLevelSkipping && !forceFullTestDiscovery { slog.Info("Suite-level skipping does not require full test discovery; using fast test file discovery fallback", "framework", testFramework.Name()) @@ -397,6 +405,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { if cacheReport.Used { discoveredTests = res fullDiscoverySucceeded = true + fullDiscoveryDuration = time.Since(fullDiscoveryStartTime) return nil } @@ -410,6 +419,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { discoveryCache.store() discoveredTests = res fullDiscoverySucceeded = true + fullDiscoveryDuration = time.Since(fullDiscoveryStartTime) return nil }) @@ -431,7 +441,8 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } } discoveredTestFiles = res - slog.Info("Discovered test files (fast)", "duration", time.Since(startTime), "count", len(discoveredTestFiles)) + fastDiscoveryDuration = time.Since(startTime) + slog.Info("Discovered test files (fast)", "duration", fastDiscoveryDuration, "count", len(discoveredTestFiles)) return nil }) @@ -448,6 +459,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { return err } tp.discoveryMode = discoveryModeFull + tp.testDiscoveryDuration = fullDiscoveryDuration tp.localDiscoveredSuites = len(tp.suiteAggregates) tp.localDiscoveredTests = countSuiteAggregateTests(tp.suiteAggregates) // if we have data on which tests exist in the local repository, we will aggregate them @@ -468,6 +480,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { return err } tp.discoveryMode = discoveryModeFast + tp.testDiscoveryDuration = fastDiscoveryDuration tp.localDiscoveredSuites = 0 tp.localDiscoveredTests = 0 tp.addDurationDataForFastDiscoveryFallback() @@ -533,9 +546,15 @@ func (tp *TestPlanner) fetchSkippables(tiaSkippingEnabled bool) skippableMatcher skippables = tp.optimizationClient.GetSkippables() } - tp.planReport.KnownTests = newKnownTestsReport(tp.optimizationClient.GetKnownTests()) + tp.planReport.KnownTests = newKnownTestsReport( + tp.optimizationClient.GetKnownTests(), + tp.optimizationClient.OperationDurations().KnownTests, + ) testManagementTestsData := tp.optimizationClient.GetTestManagementTestsData() - tp.planReport.ManagedFlakyTests = newManagedFlakyTestsReport(testManagementTestsData) + tp.planReport.ManagedFlakyTests = newManagedFlakyTestsReport( + testManagementTestsData, + tp.optimizationClient.OperationDurations().TestManagementTests, + ) disabledTests := tp.optimizationClient.GetDisabledTests() slog.Info("Fetched skippables", diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 7a43c53..43e5433 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -364,6 +364,7 @@ type MockTestOptimizationClient struct { DisabledTests map[string]bool Durations map[string]map[string]api.TestSuiteDurationInfo DurationsCalled bool + OperationTimes testoptimization.OperationDurations ShutdownCalled bool Tags map[string]string } @@ -448,6 +449,10 @@ func (m *MockTestOptimizationClient) GetTestSuiteDurations() *api.TestSuiteDurat return &api.TestSuiteDurationsResponseData{TestSuites: m.Durations} } +func (m *MockTestOptimizationClient) OperationDurations() testoptimization.OperationDurations { + return m.OperationTimes +} + func (m *MockTestOptimizationClient) StoreCacheAndExit() { m.ShutdownCalled = true } diff --git a/internal/planner/report.go b/internal/planner/report.go index c5e249e..dad589b 100644 --- a/internal/planner/report.go +++ b/internal/planner/report.go @@ -132,9 +132,11 @@ func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { reportFprintln(w, "Datadog settings") if !report.Available { reportFprintln(w, " Settings: not available") + printFetchDuration(w, report.FetchDuration) return } + printFetchDuration(w, report.FetchDuration) reportFprintf(w, " Test Impact Analysis: %s\n", enabledWord(report.TestImpactAnalysis)) reportFprintf(w, " Test skipping: %s\n", enabledWord(report.TestSkipping)) reportFprintf(w, " Test impact collection: %s\n", enabledWord(report.TestImpactCollection)) @@ -146,10 +148,10 @@ func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { func printBackendDataReport(w io.Writer, report planReport) { reportFprintln(w, "Backend data") - reportFprintf(w, " Known tests: %s\n", formatKnownTests(report.DatadogSettings, report.KnownTests)) - reportFprintf(w, " TIA skippables returned: %s\n", formatTIASkippables(report.DatadogSettings, report.Skippables)) - reportFprintf(w, " Managed flaky tests: %s\n", formatManagedFlakyTests(report.DatadogSettings, report.ManagedFlakyTests)) - reportFprintf(w, " Test suite durations: %s\n", formatTestSuiteDurations(report.TestSuiteDurations)) + reportFprintf(w, " Known tests: %s\n", formatBackendDataValue(formatKnownTests(report.DatadogSettings, report.KnownTests), report.KnownTests.FetchDuration)) + reportFprintf(w, " TIA skippables returned: %s\n", formatBackendDataValue(formatTIASkippables(report.DatadogSettings, report.Skippables), report.Skippables.FetchDuration)) + reportFprintf(w, " Managed flaky tests: %s\n", formatBackendDataValue(formatManagedFlakyTests(report.DatadogSettings, report.ManagedFlakyTests), report.ManagedFlakyTests.FetchDuration)) + reportFprintf(w, " Test suite durations: %s\n", formatBackendDataValue(formatTestSuiteDurations(report.TestSuiteDurations), report.TestSuiteDurations.FetchDuration)) } func printPlanningReport(w io.Writer, report planReport) { @@ -275,6 +277,7 @@ func printDiscoveryPlanningReport(w io.Writer, discovery discoveryReport) { reportFprintf(w, " Method: %s\n", valueOrNotAvailable(string(discovery.Mode))) reportFprintf(w, " Test files: %s\n", formatCount(discovery.TestFiles)) reportFprintf(w, " Cache: %s\n", formatDiscoveryCache(discovery.Cache)) + reportFprintf(w, " Duration: %s\n", formatOptionalDuration(discovery.Duration)) switch discovery.Mode { case discoveryModeFull: reportFprintf(w, " Suites discovered: %s\n", formatCount(discovery.Suites)) @@ -295,6 +298,26 @@ func formatDiscoveryCache(cache discoveryCacheReport) string { return "not used (" + cache.Reason + ")" } +func printFetchDuration(w io.Writer, duration time.Duration) { + if duration > 0 { + reportFprintf(w, " Fetch duration: %s\n", formatDuration(duration)) + } +} + +func formatBackendDataValue(value string, duration time.Duration) string { + if duration <= 0 { + return value + } + return fmt.Sprintf("%s (fetched in %s)", value, formatDuration(duration)) +} + +func formatOptionalDuration(duration time.Duration) string { + if duration <= 0 { + return "not available" + } + return formatDuration(duration) +} + func printDurationEstimatesPlanningReport(w io.Writer, durations durationApplicationReport) { if !durations.Available { reportFprintln(w, " Duration estimates: not available") diff --git a/internal/planner/report_data.go b/internal/planner/report_data.go index 6c9a865..1490b10 100644 --- a/internal/planner/report_data.go +++ b/internal/planner/report_data.go @@ -12,6 +12,7 @@ import ( type datadogSettingsReport struct { Available bool + FetchDuration time.Duration TestImpactAnalysis bool TestSkipping bool TestImpactCollection bool @@ -21,12 +22,13 @@ type datadogSettingsReport struct { FlakyTestManagement bool } -func newDatadogSettingsReport(settings *api.SettingsResponseData) datadogSettingsReport { +func newDatadogSettingsReport(settings *api.SettingsResponseData, fetchDuration time.Duration) datadogSettingsReport { if settings == nil { - return datadogSettingsReport{} + return datadogSettingsReport{FetchDuration: fetchDuration} } return datadogSettingsReport{ Available: true, + FetchDuration: fetchDuration, TestImpactAnalysis: settings.ItrEnabled, TestSkipping: settings.TestsSkipping, TestImpactCollection: settings.CodeCoverage, @@ -38,20 +40,22 @@ func newDatadogSettingsReport(settings *api.SettingsResponseData) datadogSetting } type knownTestsReport struct { - Available bool - Modules int - Suites int - Tests int + Available bool + FetchDuration time.Duration + Modules int + Suites int + Tests int } -func newKnownTestsReport(knownTests *api.KnownTestsResponseData) knownTestsReport { +func newKnownTestsReport(knownTests *api.KnownTestsResponseData, fetchDuration time.Duration) knownTestsReport { if knownTests == nil { - return knownTestsReport{} + return knownTestsReport{FetchDuration: fetchDuration} } report := knownTestsReport{ - Available: true, - Modules: len(knownTests.Tests), + Available: true, + FetchDuration: fetchDuration, + Modules: len(knownTests.Tests), } for _, suites := range knownTests.Tests { report.Suites += len(suites) @@ -63,19 +67,23 @@ func newKnownTestsReport(knownTests *api.KnownTestsResponseData) knownTestsRepor } type managedFlakyTestsReport struct { - Available bool - Total int - Quarantined int - Disabled int - AttemptToFix int -} - -func newManagedFlakyTestsReport(testManagementTests *api.TestManagementTestsResponseDataModules) managedFlakyTestsReport { + Available bool + FetchDuration time.Duration + Total int + Quarantined int + Disabled int + AttemptToFix int +} + +func newManagedFlakyTestsReport( + testManagementTests *api.TestManagementTestsResponseDataModules, + fetchDuration time.Duration, +) managedFlakyTestsReport { if testManagementTests == nil { - return managedFlakyTestsReport{} + return managedFlakyTestsReport{FetchDuration: fetchDuration} } - report := managedFlakyTestsReport{Available: true} + report := managedFlakyTestsReport{Available: true, FetchDuration: fetchDuration} for _, suites := range testManagementTests.Modules { for _, tests := range suites.Suites { for _, test := range tests.Tests { @@ -97,15 +105,21 @@ func newManagedFlakyTestsReport(testManagementTests *api.TestManagementTestsResp type skippablesReport struct { Available bool + FetchDuration time.Duration TestSkippingLevel settings.TestSkippingLevel TIATests int TIASuites int DisabledTests int } -func newSkippablesReport(skippables skippableMatcher, testSkippingLevel settings.TestSkippingLevel) skippablesReport { +func newSkippablesReport( + skippables skippableMatcher, + testSkippingLevel settings.TestSkippingLevel, + fetchDuration time.Duration, +) skippablesReport { return skippablesReport{ Available: true, + FetchDuration: fetchDuration, TestSkippingLevel: testSkippingLevel, TIATests: skippables.TIATestsCount(), TIASuites: skippables.TIASuitesCount(), @@ -114,19 +128,24 @@ func newSkippablesReport(skippables skippableMatcher, testSkippingLevel settings } type testSuiteDurationsReport struct { - Available bool - Modules int - Suites int + Available bool + FetchDuration time.Duration + Modules int + Suites int } -func newTestSuiteDurationsReport(testSuiteDurations *api.TestSuiteDurationsResponseData) testSuiteDurationsReport { +func newTestSuiteDurationsReport( + testSuiteDurations *api.TestSuiteDurationsResponseData, + fetchDuration time.Duration, +) testSuiteDurationsReport { if testSuiteDurations == nil { - return testSuiteDurationsReport{} + return testSuiteDurationsReport{FetchDuration: fetchDuration} } report := testSuiteDurationsReport{ - Available: true, - Modules: len(testSuiteDurations.TestSuites), + Available: true, + FetchDuration: fetchDuration, + Modules: len(testSuiteDurations.TestSuites), } for _, suites := range testSuiteDurations.TestSuites { report.Suites += len(suites) @@ -156,6 +175,7 @@ type discoveryReport struct { Available bool Mode discoveryMode Cache discoveryCacheReport + Duration time.Duration TestFiles int Suites int Tests int @@ -219,6 +239,7 @@ func (tp *TestPlanner) newPlanningReport() planningReport { Available: true, Mode: tp.discoveryMode, Cache: tp.discoveryCacheReport, + Duration: tp.testDiscoveryDuration, TestFiles: len(tp.testFiles), Suites: tp.localDiscoveredSuites, Tests: tp.localDiscoveredTests, diff --git a/internal/planner/report_test.go b/internal/planner/report_test.go index 817239b..03392c3 100644 --- a/internal/planner/report_test.go +++ b/internal/planner/report_test.go @@ -58,6 +58,7 @@ func TestPrintPlanReport_AllData(t *testing.T) { }, DatadogSettings: datadogSettingsReport{ Available: true, + FetchDuration: 240 * time.Millisecond, TestImpactAnalysis: true, TestSkipping: true, TestImpactCollection: false, @@ -67,28 +68,32 @@ func TestPrintPlanReport_AllData(t *testing.T) { FlakyTestManagement: true, }, KnownTests: knownTestsReport{ - Available: true, - Modules: 4, - Suites: 1284, - Tests: 18921, + Available: true, + FetchDuration: 80 * time.Millisecond, + Modules: 4, + Suites: 1284, + Tests: 18921, }, Skippables: skippablesReport{ Available: true, + FetchDuration: 110 * time.Millisecond, TestSkippingLevel: settings.TestSkippingLevelSuite, TIASuites: 312, DisabledTests: 3, }, ManagedFlakyTests: managedFlakyTestsReport{ - Available: true, - Total: 26, - Quarantined: 8, - Disabled: 3, - AttemptToFix: 5, + Available: true, + FetchDuration: 90 * time.Millisecond, + Total: 26, + Quarantined: 8, + Disabled: 3, + AttemptToFix: 5, }, TestSuiteDurations: testSuiteDurationsReport{ - Available: true, - Modules: 3, - Suites: 1491, + Available: true, + FetchDuration: 140 * time.Millisecond, + Modules: 3, + Suites: 1491, }, Planning: planningReport{ Discovery: discoveryReport{ @@ -98,6 +103,7 @@ func TestPrintPlanReport_AllData(t *testing.T) { Configured: true, Used: true, }, + Duration: 3 * time.Second, TestFiles: 642, Suites: 1284, Tests: 18921, @@ -188,6 +194,7 @@ DDTest settings Report enabled: false Datadog settings + Fetch duration: 240ms Test Impact Analysis: enabled Test skipping: enabled Test impact collection: disabled @@ -197,16 +204,17 @@ Datadog settings Flaky test management: enabled Backend data - Known tests: 4 modules, 1,284 suites, 18,921 tests - TIA skippables returned: 312 suites - Managed flaky tests: 26 total, 8 quarantined, 3 disabled, 5 attempt-to-fix - Test suite durations: 3 modules, 1,491 suites + Known tests: 4 modules, 1,284 suites, 18,921 tests (fetched in 80ms) + TIA skippables returned: 312 suites (fetched in 110ms) + Managed flaky tests: 26 total, 8 quarantined, 3 disabled, 5 attempt-to-fix (fetched in 90ms) + Test suite durations: 3 modules, 1,491 suites (fetched in 140ms) Planning Discovery Method: full Test files: 642 Cache: used + Duration: 3s Suites discovered: 1,284 Tests discovered: 18,921 Duration estimates @@ -253,26 +261,30 @@ func TestPrintPlanReport_FastDiscovery(t *testing.T) { }, DatadogSettings: datadogSettingsReport{ Available: true, + FetchDuration: 50 * time.Millisecond, TestImpactAnalysis: true, TestSkipping: true, TestImpactCollection: true, KnownTests: true, }, KnownTests: knownTestsReport{ - Available: true, - Modules: 1, - Suites: 10, - Tests: 125, + Available: true, + FetchDuration: 20 * time.Millisecond, + Modules: 1, + Suites: 10, + Tests: 125, }, Skippables: skippablesReport{ Available: true, + FetchDuration: 30 * time.Millisecond, TestSkippingLevel: settings.TestSkippingLevelSuite, TIASuites: 8, }, TestSuiteDurations: testSuiteDurationsReport{ - Available: true, - Modules: 1, - Suites: 12, + Available: true, + FetchDuration: 40 * time.Millisecond, + Modules: 1, + Suites: 12, }, Planning: planningReport{ Discovery: discoveryReport{ @@ -282,6 +294,7 @@ func TestPrintPlanReport_FastDiscovery(t *testing.T) { Configured: true, Reason: "full discovery not required for suite-level skipping", }, + Duration: 120 * time.Millisecond, TestFiles: 24, Suites: 0, }, @@ -340,6 +353,7 @@ DDTest settings Settings: not available Datadog settings + Fetch duration: 50ms Test Impact Analysis: enabled Test skipping: enabled Test impact collection: enabled @@ -349,16 +363,17 @@ Datadog settings Flaky test management: disabled Backend data - Known tests: 1 modules, 10 suites, 125 tests - TIA skippables returned: 8 suites + Known tests: 1 modules, 10 suites, 125 tests (fetched in 20ms) + TIA skippables returned: 8 suites (fetched in 30ms) Managed flaky tests: disabled - Test suite durations: 1 modules, 12 suites + Test suite durations: 1 modules, 12 suites (fetched in 40ms) Planning Discovery Method: fast Test files: 24 Cache: not used (full discovery not required for suite-level skipping) + Duration: 120ms Duration estimates Backend durations used: 12 suites Default durations used: 1 suite @@ -538,7 +553,7 @@ func TestReportSummaries(t *testing.T) { "suite-c": []string{"test-d", "test-e"}, }, }, - }) + }, time.Millisecond) if known.Modules != 2 || known.Suites != 3 || known.Tests != 5 { t.Errorf("unexpected known test summary: %+v", known) } @@ -557,7 +572,7 @@ func TestReportSummaries(t *testing.T) { }, }, }, - }) + }, time.Millisecond) if managed.Total != 3 || managed.Quarantined != 1 || managed.Disabled != 1 || managed.AttemptToFix != 1 { t.Errorf("unexpected managed flaky test summary: %+v", managed) } @@ -572,7 +587,7 @@ func TestReportSummaries(t *testing.T) { "suite-c": {}, }, }, - }) + }, time.Millisecond) if durations.Modules != 2 || durations.Suites != 3 { t.Errorf("unexpected test suite durations summary: %+v", durations) } @@ -642,6 +657,8 @@ func TestReportFormattingVariants(t *testing.T) { {name: "cache used", got: formatDiscoveryCache(discoveryCacheReport{Configured: true, Used: true}), want: "used"}, {name: "cache configured but not used", got: formatDiscoveryCache(discoveryCacheReport{Configured: true}), want: "not used"}, {name: "cache configured but skipped with reason", got: formatDiscoveryCache(discoveryCacheReport{Configured: true, Reason: "full discovery not required"}), want: "not used (full discovery not required)"}, + {name: "backend data without duration", got: formatBackendDataValue("not available", 0), want: "not available"}, + {name: "optional duration missing", got: formatOptionalDuration(0), want: "not available"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/internal/testoptimization/testoptimization.go b/internal/testoptimization/testoptimization.go index de71f64..36e194e 100644 --- a/internal/testoptimization/testoptimization.go +++ b/internal/testoptimization/testoptimization.go @@ -32,6 +32,14 @@ const ( type testOptimizationCloseAction func() +type OperationDurations struct { + Settings time.Duration + KnownTests time.Duration + Skippables time.Duration + TestManagementTests time.Duration + TestSuiteDurations time.Duration +} + type searchCommitsResponse struct { LocalCommits []string RemoteCommits []string @@ -55,6 +63,9 @@ type TestOptimizationClient struct { skippables api.Skippables testSkippingLevel settings.TestSkippingLevel testManagementTests api.TestManagementTestsResponseDataModules + + operationDurationsMutex sync.Mutex + operationDurations OperationDurations } func NewTestOptimizationClient() *TestOptimizationClient { @@ -172,6 +183,13 @@ func (c *TestOptimizationClient) GetDisabledTests() map[string]bool { } func (c *TestOptimizationClient) GetTestSuiteDurations() *api.TestSuiteDurationsResponseData { + startTime := time.Now() + defer func() { + c.recordOperationDuration(func(durations *OperationDurations) { + durations.TestSuiteDurations = time.Since(startTime) + }) + }() + testOptimizationTransport := c.ensureAPITransport(autoDetectServiceName) if testOptimizationTransport == nil { return &api.TestSuiteDurationsResponseData{ @@ -181,6 +199,18 @@ func (c *TestOptimizationClient) GetTestSuiteDurations() *api.TestSuiteDurations return testOptimizationTransport.GetTestSuiteDurations() } +func (c *TestOptimizationClient) OperationDurations() OperationDurations { + c.operationDurationsMutex.Lock() + defer c.operationDurationsMutex.Unlock() + return c.operationDurations +} + +func (c *TestOptimizationClient) recordOperationDuration(update func(*OperationDurations)) { + c.operationDurationsMutex.Lock() + defer c.operationDurationsMutex.Unlock() + update(&c.operationDurations) +} + func (c *TestOptimizationClient) StoreCacheAndExit() { repositorySettings := c.GetSettings() if repositorySettings != nil { @@ -233,6 +263,13 @@ func traceDebugEnabled() bool { func (c *TestOptimizationClient) ensureSettingsInitialization(serviceName string) *api.SettingsResponseData { c.settingsOnce.Do(func() { + startTime := time.Now() + defer func() { + c.recordOperationDuration(func(durations *OperationDurations) { + durations.Settings = time.Since(startTime) + }) + }() + slog.Debug("testoptimization: initializing settings") defer slog.Debug("testoptimization: settings initialization complete") @@ -365,6 +402,12 @@ func (c *TestOptimizationClient) ensureTestOptimizationInitialized() { wg.Add(1) go func() { defer wg.Done() + startTime := time.Now() + defer func() { + c.recordOperationDuration(func(durations *OperationDurations) { + durations.KnownTests = time.Since(startTime) + }) + }() knownTests, err := c.apiTransport.GetKnownTests() if err != nil { slog.Error("testoptimization: error getting test optimization known tests data", "err", err.Error()) @@ -379,6 +422,12 @@ func (c *TestOptimizationClient) ensureTestOptimizationInitialized() { wg.Add(1) go func() { defer wg.Done() + startTime := time.Now() + defer func() { + c.recordOperationDuration(func(durations *OperationDurations) { + durations.Skippables = time.Since(startTime) + }) + }() correlationID, skippables, err := c.apiTransport.GetSkippableTests() if err != nil { slog.Error("testoptimization: error getting test optimization skippable tests", "err", err.Error()) @@ -396,6 +445,12 @@ func (c *TestOptimizationClient) ensureTestOptimizationInitialized() { wg.Add(1) go func() { defer wg.Done() + startTime := time.Now() + defer func() { + c.recordOperationDuration(func(durations *OperationDurations) { + durations.TestManagementTests = time.Since(startTime) + }) + }() testManagementTests, err := c.apiTransport.GetTestManagementTests() if err != nil { slog.Error("testoptimization: error getting test optimization test management tests", "err", err.Error()) diff --git a/internal/testoptimization/testoptimization_lifecycle_test.go b/internal/testoptimization/testoptimization_lifecycle_test.go index f27568d..75afb98 100644 --- a/internal/testoptimization/testoptimization_lifecycle_test.go +++ b/internal/testoptimization/testoptimization_lifecycle_test.go @@ -91,6 +91,10 @@ func TestTestOptimizationClientFeatureGetters(t *testing.T) { t.Fatalf("expected each feature endpoint once, got known=%d skippable=%d testManagement=%d", mockTransport.KnownTestsCalls, mockTransport.SkippableTestsCalls, mockTransport.TestManagementTestsCalls) } + operationDurations := client.OperationDurations() + if operationDurations.KnownTests <= 0 || operationDurations.Skippables <= 0 || operationDurations.TestManagementTests <= 0 { + t.Fatalf("expected feature endpoint durations, got %+v", operationDurations) + } ciTags := environment.GetCITags() if ciTags[constants.ItrCorrelationIDTag] != "correlation-id" { diff --git a/internal/testoptimization/testoptimization_test.go b/internal/testoptimization/testoptimization_test.go index 19e59f6..98b344e 100644 --- a/internal/testoptimization/testoptimization_test.go +++ b/internal/testoptimization/testoptimization_test.go @@ -258,6 +258,9 @@ func TestTestOptimizationClient_GetTestSuiteDurations(t *testing.T) { if result.TestSuites["rspec"]["Suite"].SourceFile != "spec/suite_spec.rb" { t.Fatalf("GetTestSuiteDurations() returned %#v, want %#v", result, durations) } + if client.OperationDurations().TestSuiteDurations <= 0 { + t.Fatal("GetTestSuiteDurations() should record operation duration") + } } func TestTestOptimizationClient_Initialize(t *testing.T) { @@ -298,6 +301,9 @@ func TestTestOptimizationClient_Initialize(t *testing.T) { if duration < 0 { t.Error("Initialize() duration should be non-negative") } + if client.OperationDurations().Settings <= 0 { + t.Error("Initialize() should record settings fetch duration") + } } func TestTestOptimizationClient_GetSkippables_NilResponse(t *testing.T) { @@ -370,6 +376,9 @@ func TestTestOptimizationClient_GetSkippables(t *testing.T) { if len(result) != len(expectedTests) { t.Errorf("Expected %d skippable tests, got %d", len(expectedTests), len(result)) } + if client.OperationDurations().Skippables <= 0 { + t.Error("GetSkippables() should record skippables fetch duration") + } } func TestTestOptimizationClient_StoreCacheAndExit(t *testing.T) { @@ -535,6 +544,9 @@ func TestTestOptimizationClient_GetDisabledTests(t *testing.T) { if mockAPIClient.TestManagementTestsCalls != 1 { t.Fatalf("expected test management tests to be fetched once, got %d", mockAPIClient.TestManagementTestsCalls) } + if client.OperationDurations().TestManagementTests <= 0 { + t.Fatal("GetDisabledTests() should record test management fetch duration") + } } func TestTestOptimizationClient_GetDisabledTestsDisabled(t *testing.T) { From a95edf6599cc018d07552da472d41140f679e8f4 Mon Sep 17 00:00:00 2001 From: Andrey Marchenko Date: Tue, 30 Jun 2026 16:28:12 +0200 Subject: [PATCH 7/7] Simplify plan report data flow --- internal/planner/discovered_tests.go | 83 ++++------ internal/planner/discovery_cache.go | 28 ++-- internal/planner/discovery_cache_test.go | 14 +- .../high_skippable_integration_test.go | 6 +- internal/planner/planner.go | 138 ++++++----------- internal/planner/planner_test.go | 133 ++++++++-------- internal/planner/report.go | 72 ++++++--- internal/planner/report_data.go | 130 +++++++++++----- internal/planner/report_test.go | 118 ++++++++------- .../planner/test_optimization_plan_cache.go | 72 +-------- .../test_optimization_plan_cache_test.go | 68 ++------- internal/runner/report.go | 20 +-- internal/runner/report_test.go | 2 +- internal/runner/runner.go | 18 +-- internal/runner/runner_test.go | 8 +- internal/settings/settings.go | 59 +++----- .../testoptimization/api/durations_api.go | 4 + .../api/durations_api_test.go | 3 + .../testoptimization/api/known_tests_api.go | 6 + .../testoptimization/api/raw_response_test.go | 14 ++ internal/testoptimization/api/settings_api.go | 6 + internal/testoptimization/api/skippable.go | 13 ++ .../testoptimization/api/skippable_test.go | 7 + .../api/test_management_tests_api.go | 6 + internal/testoptimization/api/transport.go | 14 ++ internal/testoptimization/testoptimization.go | 142 +++++------------- .../testoptimization_lifecycle_test.go | 4 - .../testoptimization/testoptimization_test.go | 78 ++++++++-- 28 files changed, 616 insertions(+), 650 deletions(-) diff --git a/internal/planner/discovered_tests.go b/internal/planner/discovered_tests.go index 5bbf30d..ec3f771 100644 --- a/internal/planner/discovered_tests.go +++ b/internal/planner/discovered_tests.go @@ -17,15 +17,7 @@ func (tp *TestPlanner) resetDiscoveryResults() { tp.testFiles = make(map[string]struct{}) tp.suiteAggregates = make(map[testSuiteKey]testSuiteAggregate) tp.suitesBySourceFile = make(map[string][]testSuiteKey) - tp.discoveryMode = discoveryModeUnknown - tp.discoveryCacheReport = discoveryCacheReport{} - tp.testDiscoveryDuration = 0 - tp.localDiscoveredSuites = 0 - tp.localDiscoveredTests = 0 - tp.tiaSkippableTestsApplied = 0 - tp.tiaSkippableSuitesApplied = make(map[testSuiteKey]struct{}) - tp.disabledTestsApplied = 0 - tp.unskippableMarkerSuitesForced = 0 + tp.reportStats = newPlanningReportStats() } func (tp *TestPlanner) recordFullDiscoveryResults( @@ -55,7 +47,6 @@ func (tp *TestPlanner) recordFullDiscoveryResults( break } } - slog.Debug("Skippable matcher size", "size", skippableMatcher.Count()) } skippableTestsCount := 0 @@ -81,7 +72,7 @@ func (tp *TestPlanner) recordFullDiscoveryResults( } else { slog.Debug("Test will be skipped", "test", test.DatadogTestId(), "sourceFile", test.SuiteSourceFile) recordSkippedTest(tp.suiteAggregates, test, normalizedSourceFile) - tp.recordAppliedSkippable(match, testSuiteKey{Module: test.Module, Suite: test.Suite}) + tp.recordAppliedSkippable(match) skippableTestsCount++ } } @@ -99,17 +90,22 @@ type skippableMatcher struct { disabledTests map[string]bool } -type skippableMatch int +type skippableMatch struct { + kind skippableMatchKind + suiteKey testSuiteKey +} + +type skippableMatchKind int const ( - skippableMatchNone skippableMatch = iota + skippableMatchNone skippableMatchKind = iota skippableMatchTIATest skippableMatchTIASuite skippableMatchDisabledTest ) func (m skippableMatch) Skipped() bool { - return m != skippableMatchNone + return m.kind != skippableMatchNone } func newSkippableMatcher(skippables api.Skippables, disabledTests map[string]bool) skippableMatcher { @@ -130,62 +126,40 @@ func newSkippableMatcher(skippables api.Skippables, disabledTests map[string]boo } } -func (s skippableMatcher) Contains(test testoptimization.Test) bool { - return s.Match(test).Skipped() -} - func (s skippableMatcher) Match(test testoptimization.Test) skippableMatch { suite := api.SkippableSuite{Module: test.Module, Suite: test.Suite} if s.disabledTests[test.FQN()] { - return skippableMatchDisabledTest + return skippableMatch{kind: skippableMatchDisabledTest} } if s.tiaSkippableSuites[suite] { - return skippableMatchTIASuite + return skippableMatch{ + kind: skippableMatchTIASuite, + suiteKey: testSuiteKey{Module: test.Module, Suite: test.Suite}, + } } if s.tiaSkippableTests[test.DatadogTestId()] { - return skippableMatchTIATest + return skippableMatch{kind: skippableMatchTIATest} } - return skippableMatchNone -} - -func (s skippableMatcher) Count() int { - return s.TIASkippablesCount() + len(s.disabledTests) + return skippableMatch{} } func (s skippableMatcher) TIASkippablesCount() int { return len(s.tiaSkippableTests) + len(s.tiaSkippableSuites) } -func (s skippableMatcher) TIATestsCount() int { - return len(s.tiaSkippableTests) -} - -func (s skippableMatcher) TIASuitesCount() int { - return len(s.tiaSkippableSuites) -} - -func (s skippableMatcher) DisabledTestsCount() int { - return len(s.disabledTests) -} - -func (tp *TestPlanner) recordAppliedSkippable(match skippableMatch, key testSuiteKey) { - switch match { +func (tp *TestPlanner) recordAppliedSkippable(match skippableMatch) { + switch match.kind { case skippableMatchTIATest: - tp.tiaSkippableTestsApplied++ + tp.reportStats.tiaSkippableTestsApplied++ case skippableMatchTIASuite: - tp.recordAppliedTIASuite(key) + // Full discovery sees suite-level skippables once per test, so keep a set + // and report the number of suites rather than the number of skipped tests. + tp.reportStats.uniqueTIASkippableSuitesApplied[match.suiteKey] = struct{}{} case skippableMatchDisabledTest: - tp.disabledTestsApplied++ + tp.reportStats.disabledTestsApplied++ } } -func (tp *TestPlanner) recordAppliedTIASuite(key testSuiteKey) { - if tp.tiaSkippableSuitesApplied == nil { - tp.tiaSkippableSuitesApplied = make(map[testSuiteKey]struct{}) - } - tp.tiaSkippableSuitesApplied[key] = struct{}{} -} - func (tp *TestPlanner) recordFastDiscoveryFallbackFiles(discoveredTestFiles []string) error { excluder, err := discovery.NewExcluder(settings.GetTestsExcludePattern()) if err != nil { @@ -243,13 +217,14 @@ func (tp *TestPlanner) recordSuiteLevelSkippables(skippableMatcher skippableMatc } aggregate.NumTestsSkipped = aggregate.NumTests tp.suiteAggregates[key] = aggregate - tp.recordAppliedTIASuite(key) + tp.reportStats.uniqueTIASkippableSuitesApplied[key] = struct{}{} } } -func (tp *TestPlanner) keepUnskippableMarkerSuitesRunnable(testFramework framework.Framework) int { +func (tp *TestPlanner) keepUnskippableMarkerSuitesRunnable(testFramework framework.Framework) { if testFramework == nil { - return 0 + tp.reportStats.unskippableMarkerSuitesForced = 0 + return } startTime := time.Now() @@ -278,7 +253,7 @@ func (tp *TestPlanner) keepUnskippableMarkerSuitesRunnable(testFramework framewo slog.Info("Checked unskippable marker suites", "duration", time.Since(startTime), "forceRunnableSuiteAggregatesCount", forceRunnableSuiteAggregatesCount) - return forceRunnableSuiteAggregatesCount + tp.reportStats.unskippableMarkerSuitesForced = forceRunnableSuiteAggregatesCount } func (tp *TestPlanner) sourceFileForSuite(key testSuiteKey, testFramework framework.Framework) (string, bool) { diff --git a/internal/planner/discovery_cache.go b/internal/planner/discovery_cache.go index e1b1ea4..2532683 100644 --- a/internal/planner/discovery_cache.go +++ b/internal/planner/discovery_cache.go @@ -53,6 +53,12 @@ type discoveryCache struct { filePath string } +type discoveryCacheResult struct { + Configured bool + Used bool + NotUsedReason string +} + var discoveryCacheGitOutput = func(args ...string) ([]byte, error) { return exec.Command("git", args...).Output() } @@ -208,39 +214,39 @@ func readLastNonEmptyLine(filePath string) ([]byte, error) { } } -func (c discoveryCache) restore() ([]testoptimization.Test, discoveryCacheReport) { - report := discoveryCacheReport{ +func (c discoveryCache) restore() ([]testoptimization.Test, discoveryCacheResult) { + result := discoveryCacheResult{ Configured: settings.GetTestDiscoveryCache() != "", } c.importExternal() if err := c.validate(); err != nil { - report.Reason = err.Error() - if report.Configured { + result.NotUsedReason = err.Error() + if result.Configured { slog.Info("Cached test discovery not usable; full discovery will run", "reason", err) } else { slog.Debug("Cached test discovery not usable; full discovery will run", "reason", err) } - return nil, report + return nil, result } startTime := time.Now() tests, err := parseCachedDiscoveryTests(c.filePath) if err != nil { - report.Reason = err.Error() + result.NotUsedReason = err.Error() slog.Info("Cached test discovery could not be used; full discovery will run", "error", err) - return nil, report + return nil, result } if err := ensureDiscoveredTests(tests); err != nil { - report.Reason = err.Error() + result.NotUsedReason = err.Error() slog.Info("Cached test discovery could not be used; full discovery will run", "error", err) - return nil, report + return nil, result } - report.Used = true + result.Used = true slog.Info("Cached test discovery succeeded", "duration", time.Since(startTime), "count", len(tests)) - return tests, report + return tests, result } func ensureDiscoveredTests(tests []testoptimization.Test) error { diff --git a/internal/planner/discovery_cache_test.go b/internal/planner/discovery_cache_test.go index 59d1438..63de691 100644 --- a/internal/planner/discovery_cache_test.go +++ b/internal/planner/discovery_cache_test.go @@ -192,8 +192,9 @@ func TestDiscoveryCacheHitUsesCachedTests(t *testing.T) { if len(mockFramework.DiscoverTestsFiles) != 0 { t.Fatalf("expected cache hit to avoid full discovery, got %d calls", len(mockFramework.DiscoverTestsFiles)) } - if !runner.planReport.Planning.Discovery.Cache.Used { - t.Fatalf("expected cache hit to be reported, got %+v", runner.planReport.Planning.Discovery.Cache) + report := runner.newPlanReportData(splitScore{}) + if !report.Planning.Discovery.Cache.Used { + t.Fatalf("expected cache hit to be reported, got %+v", report.Planning.Discovery.Cache) } aggregate := runner.suiteAggregates[testSuiteKey{Module: "rspec", Suite: "Cart"}] if aggregate.NumTests != 1 || aggregate.NumTestsSkipped != 1 { @@ -293,8 +294,9 @@ func TestDiscoveryCacheImportsExternalCacheBeforeValidation(t *testing.T) { if len(mockFramework.DiscoverTestsFiles) != 0 { t.Fatalf("expected imported cache to avoid full discovery, got %d calls", len(mockFramework.DiscoverTestsFiles)) } - if cacheReport := runner.planReport.Planning.Discovery.Cache; !cacheReport.Configured || !cacheReport.Used { - t.Fatalf("expected imported cache to be reported as configured and used, got %+v", cacheReport) + report := runner.newPlanReportData(splitScore{}) + if cacheResult := report.Planning.Discovery.Cache; !cacheResult.Configured || !cacheResult.Used { + t.Fatalf("expected imported cache to be reported as configured and used, got %+v", cacheResult) } if _, err := os.Stat(discovery.TestsFilePath); err != nil { t.Fatalf("expected imported cache at internal path: %v", err) @@ -313,8 +315,8 @@ func TestDiscoveryCacheRestoreRejectsEmptyCache(t *testing.T) { if report.Used || tests != nil { t.Fatalf("expected empty cache restore to fail, got report=%+v tests=%v", report, tests) } - if report.Reason != "test discovery returned no tests" { - t.Fatalf("unexpected empty cache restore reason: %q", report.Reason) + if report.NotUsedReason != "test discovery returned no tests" { + t.Fatalf("unexpected empty cache restore reason: %q", report.NotUsedReason) } } diff --git a/internal/planner/high_skippable_integration_test.go b/internal/planner/high_skippable_integration_test.go index 308db2f..99ee041 100644 --- a/internal/planner/high_skippable_integration_test.go +++ b/internal/planner/high_skippable_integration_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "slices" + "strconv" "strings" "testing" @@ -93,10 +94,7 @@ func TestTestPlanner_Plan_HighSkippableIntegrationSelectsExpectedRunnerCountAndR } expectedParallelRunners := fixture.ExpectedParallelRunners - if runner.planReport.Split.parallelRunners != expectedParallelRunners { - t.Fatalf("Plan() selected %d runners, expected %d", runner.planReport.Split.parallelRunners, expectedParallelRunners) - } - assertFileContent(t, constants.ParallelRunnersOutputPath, "1") + assertFileContent(t, constants.ParallelRunnersOutputPath, strconv.Itoa(expectedParallelRunners)) testFiles := readTestPlanLines(t, constants.TestFilesOutputPath) if !slices.Equal(testFiles, expectedRunnableFiles) { diff --git a/internal/planner/planner.go b/internal/planner/planner.go index c78b8cb..34d2230 100644 --- a/internal/planner/planner.go +++ b/internal/planner/planner.go @@ -27,7 +27,7 @@ import ( type Planner interface { Plan(ctx context.Context) error - LoadPlan() (PlanInfo, error) + LoadPlan() (PlanMetadata, error) DistributeTestFiles(testFiles []string, parallelRunners int) [][]string } @@ -39,11 +39,11 @@ type testOptimizationClient interface { GetTestManagementTestsData() *api.TestManagementTestsResponseDataModules GetDisabledTests() map[string]bool GetTestSuiteDurations() *api.TestSuiteDurationsResponseData - OperationDurations() testoptimization.OperationDurations + BackendRequestTimings() testoptimization.BackendRequestTimings StoreCacheAndExit() } -type PlanInfo struct { +type PlanMetadata struct { Platform string `json:"platform"` Framework string `json:"framework"` TestSkippingLevel string `json:"testSkippingLevel,omitempty"` @@ -51,8 +51,8 @@ type PlanInfo struct { RuntimeTags map[string]string `json:"runtimeTags"` } -func NewPlanInfo(tags map[string]string, platformName, frameworkName string, testSkippingLevel settings.TestSkippingLevel) PlanInfo { - return PlanInfo{ +func NewPlanMetadata(tags map[string]string, platformName, frameworkName string, testSkippingLevel settings.TestSkippingLevel) PlanMetadata { + return PlanMetadata{ Platform: platformName, Framework: frameworkName, TestSkippingLevel: testSkippingLevel.String(), @@ -61,7 +61,7 @@ func NewPlanInfo(tags map[string]string, platformName, frameworkName string, tes } } -func (p PlanInfo) IsZero() bool { +func (p PlanMetadata) IsZero() bool { return p.Platform == "" && p.Framework == "" && p.TestSkippingLevel == "" && @@ -70,31 +70,22 @@ func (p PlanInfo) IsZero() bool { } type TestPlanner struct { - testFiles map[string]struct{} - suiteAggregates map[testSuiteKey]testSuiteAggregate - suitesBySourceFile map[string][]testSuiteKey - testSuiteDurations map[string]map[string]api.TestSuiteDurationInfo - testFileWeights map[string]int - testFileDurationSources map[string]testFileDurationSource - discoveryMode discoveryMode - discoveryCacheReport discoveryCacheReport - testDiscoveryDuration time.Duration - localDiscoveredSuites int - localDiscoveredTests int - tiaSkippableTestsApplied int - tiaSkippableSuitesApplied map[testSuiteKey]struct{} - disabledTestsApplied int - unskippableMarkerSuitesForced int - skippablePercentage float64 - planReport planReport - planLoaded bool - runInfo runmetadata.RunInfo - planInfo PlanInfo - platformDetector platform.PlatformDetector - optimizationClient testOptimizationClient - newOptimizationClient func(testSkippingLevel settings.TestSkippingLevel) testOptimizationClient - ciProviderDetector environment.CIProviderDetector - reportWriter io.Writer + testFiles map[string]struct{} + suiteAggregates map[testSuiteKey]testSuiteAggregate + suitesBySourceFile map[string][]testSuiteKey + testSuiteDurations map[string]map[string]api.TestSuiteDurationInfo + testFileWeights map[string]int + testFileDurationSources map[string]testFileDurationSource + reportStats planningReportStats + skippablePercentage float64 + planLoaded bool + runInfo runmetadata.RunInfo + planMetadata PlanMetadata + platformDetector platform.PlatformDetector + optimizationClient testOptimizationClient + newOptimizationClient func(testSkippingLevel settings.TestSkippingLevel) testOptimizationClient + ciProviderDetector environment.CIProviderDetector + reportWriter io.Writer } const ( @@ -182,15 +173,15 @@ func NewWithDependencies( func newTestPlannerWithDefaults() *TestPlanner { return &TestPlanner{ - testFiles: make(map[string]struct{}), - suiteAggregates: make(map[testSuiteKey]testSuiteAggregate), - suitesBySourceFile: make(map[string][]testSuiteKey), - testSuiteDurations: make(map[string]map[string]api.TestSuiteDurationInfo), - testFileWeights: make(map[string]int), - testFileDurationSources: make(map[string]testFileDurationSource), - tiaSkippableSuitesApplied: make(map[testSuiteKey]struct{}), - skippablePercentage: 0.0, - reportWriter: os.Stderr, + testFiles: make(map[string]struct{}), + suiteAggregates: make(map[testSuiteKey]testSuiteAggregate), + suitesBySourceFile: make(map[string][]testSuiteKey), + testSuiteDurations: make(map[string]map[string]api.TestSuiteDurationInfo), + testFileWeights: make(map[string]int), + testFileDurationSources: make(map[string]testFileDurationSource), + reportStats: newPlanningReportStats(), + skippablePercentage: 0.0, + reportWriter: os.Stderr, } } @@ -245,13 +236,8 @@ func (tp *TestPlanner) Plan(ctx context.Context) error { return fmt.Errorf("failed to create test splits: %w", err) } - tp.planReport.Split = parallelRunnerSplit - if settings.GetReportEnabled() { - tp.planReport.DDTestSettings = settings.Get() - tp.planReport.LongSeparateRunnerSuites = tp.longSeparateRunnerSuitesReport(parallelRunners, parallelRunnerSplit) - tp.planReport.SlowestTestSuitesOverall = tp.slowestTestSuitesOverallReport(slowestTestSuitesReportLimit) - printPlanReport(tp.reportWriter, tp.planReport) + printPlanReport(tp.reportWriter, tp, parallelRunnerSplit) } tp.planLoaded = true @@ -298,7 +284,7 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { strictDiscovery := settings.GetStrictDiscovery() fullDiscoveryNeeded := fullTestDiscoverySupported && (isTestLevelSkipping || forceFullTestDiscovery) tp.runInfo = runmetadata.New(environment.GetCITags()) - tp.planInfo = NewPlanInfo(tags, detectedPlatform.Name(), testFramework.Name(), testSkippingLevel) + tp.planMetadata = NewPlanMetadata(tags, detectedPlatform.Name(), testFramework.Name(), testSkippingLevel) if tp.optimizationClient == nil { if tp.newOptimizationClient == nil { return fmt.Errorf("failed to create optimization client: missing client factory") @@ -320,6 +306,9 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { var tiaSkippingEnabled bool var fullDiscoveryDuration time.Duration var fastDiscoveryDuration time.Duration + var selectedDiscoveryMode discoveryMode + var selectedDiscoveryDuration time.Duration + var cacheResult discoveryCacheResult tp.resetDiscoveryResults() tp.testSuiteDurations = make(map[string]map[string]api.TestSuiteDurationInfo) @@ -344,7 +333,6 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } repositorySettings := tp.optimizationClient.GetSettings() - tp.planReport.DatadogSettings = newDatadogSettingsReport(repositorySettings, tp.optimizationClient.OperationDurations().Settings) tiaSkippingEnabled = false if repositorySettings != nil { slog.Debug("Repository settings", "tia_enabled", repositorySettings.ItrEnabled, "tests_skipping", repositorySettings.TestsSkipping) @@ -362,7 +350,6 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } skipMatcher = tp.fetchSkippables(tiaSkippingEnabled) - tp.planReport.Skippables = newSkippablesReport(skipMatcher, testSkippingLevel, tp.optimizationClient.OperationDurations().Skippables) if tiaSkippingEnabled && skipMatcher.TIASkippablesCount() == 0 && !forceFullTestDiscovery { slog.Info("No TIA-skippable tests or suites found for this run, cancelling full test discovery") @@ -370,10 +357,6 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { } testSuiteDurations := tp.optimizationClient.GetTestSuiteDurations() - tp.planReport.TestSuiteDurations = newTestSuiteDurationsReport( - testSuiteDurations, - tp.optimizationClient.OperationDurations().TestSuiteDurations, - ) if testSuiteDurations != nil && testSuiteDurations.TestSuites != nil { tp.testSuiteDurations = testSuiteDurations.TestSuites } @@ -387,22 +370,19 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { if !fullDiscoveryNeeded { if isSuiteLevelSkipping && !forceFullTestDiscovery { slog.Info("Suite-level skipping does not require full test discovery; using fast test file discovery fallback", "framework", testFramework.Name()) - tp.discoveryCacheReport = newDiscoveryCacheReport(false, "full discovery not required for suite-level skipping") return nil } if forceFullTestDiscovery && !fullTestDiscoverySupported { slog.Warn("Full test discovery was forced but is not supported by framework; using fast test file discovery fallback", "framework", testFramework.Name()) - tp.discoveryCacheReport = newDiscoveryCacheReport(false, "full discovery not supported by framework") return nil } slog.Info("Full test discovery is not supported by framework; using fast test file discovery fallback", "framework", testFramework.Name()) - tp.discoveryCacheReport = newDiscoveryCacheReport(false, "full discovery not supported by framework") return nil } - res, cacheReport := discoveryCache.restore() - tp.discoveryCacheReport = cacheReport - if cacheReport.Used { + res, restoredCacheResult := discoveryCache.restore() + cacheResult = restoredCacheResult + if restoredCacheResult.Used { discoveredTests = res fullDiscoverySucceeded = true fullDiscoveryDuration = time.Since(fullDiscoveryStartTime) @@ -458,10 +438,8 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { if err := tp.recordFullDiscoveryResults(discoveredTests, skipMatcher); err != nil { return err } - tp.discoveryMode = discoveryModeFull - tp.testDiscoveryDuration = fullDiscoveryDuration - tp.localDiscoveredSuites = len(tp.suiteAggregates) - tp.localDiscoveredTests = countSuiteAggregateTests(tp.suiteAggregates) + selectedDiscoveryMode = discoveryModeFull + selectedDiscoveryDuration = fullDiscoveryDuration // if we have data on which tests exist in the local repository, we will aggregate them // into a collection of testSuiteAggregate structs. // This collection is used to calculate the skippable percentage and the weighted test files. @@ -479,10 +457,8 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { if err := tp.recordFastDiscoveryFallbackFiles(discoveredTestFiles); err != nil { return err } - tp.discoveryMode = discoveryModeFast - tp.testDiscoveryDuration = fastDiscoveryDuration - tp.localDiscoveredSuites = 0 - tp.localDiscoveredTests = 0 + selectedDiscoveryMode = discoveryModeFast + selectedDiscoveryDuration = fastDiscoveryDuration tp.addDurationDataForFastDiscoveryFallback() if isSuiteLevelSkipping && tiaSkippingEnabled { tp.recordSuiteLevelSkippables(skipMatcher, testFramework) @@ -492,14 +468,12 @@ func (tp *TestPlanner) PreparePlanningData(ctx context.Context) error { "fastDiscoveredTestFilesCount", len(discoveredTestFiles)) } - tp.unskippableMarkerSuitesForced = tp.keepUnskippableMarkerSuitesRunnable(testFramework) + tp.keepUnskippableMarkerSuitesRunnable(testFramework) tp.suitesBySourceFile = indexSuitesBySourceFile(tp.suiteAggregates) tp.skippablePercentage = calculateSavedTimePercentage(tp.suiteAggregates) tp.testFileWeights = tp.calculateFileWeights() - tp.planReport.RunInfo = tp.runInfo - tp.planReport.PlanInfo = tp.planInfo - tp.planReport.Planning = tp.newPlanningReport() + tp.recordDiscoveryReport(selectedDiscoveryMode, cacheResult, selectedDiscoveryDuration) slog.Info("Test files prepared", "testFilesCount", len(tp.testFiles)) @@ -529,14 +503,6 @@ func discoverLocalTests(ctx context.Context, testFramework framework.Framework, return tests, nil } -func newDiscoveryCacheReport(used bool, reason string) discoveryCacheReport { - return discoveryCacheReport{ - Configured: settings.GetTestDiscoveryCache() != "", - Used: used, - Reason: reason, - } -} - func (tp *TestPlanner) fetchSkippables(tiaSkippingEnabled bool) skippableMatcher { startTime := time.Now() slog.Info("Fetching skippables from Datadog...") @@ -546,24 +512,16 @@ func (tp *TestPlanner) fetchSkippables(tiaSkippingEnabled bool) skippableMatcher skippables = tp.optimizationClient.GetSkippables() } - tp.planReport.KnownTests = newKnownTestsReport( - tp.optimizationClient.GetKnownTests(), - tp.optimizationClient.OperationDurations().KnownTests, - ) - testManagementTestsData := tp.optimizationClient.GetTestManagementTestsData() - tp.planReport.ManagedFlakyTests = newManagedFlakyTestsReport( - testManagementTestsData, - tp.optimizationClient.OperationDurations().TestManagementTests, - ) - disabledTests := tp.optimizationClient.GetDisabledTests() + matcher := newSkippableMatcher(skippables, disabledTests) + slog.Info("Fetched skippables", "duration", time.Since(startTime), "tiaSkippableTestsCount", len(skippables.Tests), "tiaSkippableSuitesCount", len(skippables.Suites), "disabledTestsCount", len(disabledTests)) - return newSkippableMatcher(skippables, disabledTests) + return matcher } func (tp *TestPlanner) estimateDiscoveredSuiteDurations() { diff --git a/internal/planner/planner_test.go b/internal/planner/planner_test.go index 43e5433..b0b59e2 100644 --- a/internal/planner/planner_test.go +++ b/internal/planner/planner_test.go @@ -354,19 +354,19 @@ func (m *longRunningDiscoveryFramework) DiscoverTests(ctx context.Context, testF // MockTestOptimizationClient mocks the test optimization client type MockTestOptimizationClient struct { - InitializeCalled bool - InitializeErr error - Settings *api.SettingsResponseData - Skippables api.Skippables - GetSkippablesCalled bool - KnownTests *api.KnownTestsResponseData - TestManagementTests *api.TestManagementTestsResponseDataModules - DisabledTests map[string]bool - Durations map[string]map[string]api.TestSuiteDurationInfo - DurationsCalled bool - OperationTimes testoptimization.OperationDurations - ShutdownCalled bool - Tags map[string]string + InitializeCalled bool + InitializeErr error + Settings *api.SettingsResponseData + Skippables api.Skippables + GetSkippablesCalled bool + KnownTests *api.KnownTestsResponseData + TestManagementTests *api.TestManagementTestsResponseDataModules + DisabledTests map[string]bool + Durations map[string]map[string]api.TestSuiteDurationInfo + BackendRequestTimingValues testoptimization.BackendRequestTimings + DurationsCalled bool + ShutdownCalled bool + Tags map[string]string } func testSkippables(tests map[string]bool) api.Skippables { @@ -392,6 +392,9 @@ func (m *MockTestOptimizationClient) GetSettings() *api.SettingsResponseData { func (m *MockTestOptimizationClient) GetSkippables() api.Skippables { m.GetSkippablesCalled = true + if m.Settings != nil && !m.Settings.TestsSkipping { + return api.NewSkippables() + } skippables := m.Skippables if skippables.Tests == nil { skippables.Tests = api.SkippableTests{} @@ -399,6 +402,7 @@ func (m *MockTestOptimizationClient) GetSkippables() api.Skippables { if skippables.Suites == nil { skippables.Suites = api.SkippableSuites{} } + m.Skippables = skippables return skippables } @@ -449,8 +453,8 @@ func (m *MockTestOptimizationClient) GetTestSuiteDurations() *api.TestSuiteDurat return &api.TestSuiteDurationsResponseData{TestSuites: m.Durations} } -func (m *MockTestOptimizationClient) OperationDurations() testoptimization.OperationDurations { - return m.OperationTimes +func (m *MockTestOptimizationClient) BackendRequestTimings() testoptimization.BackendRequestTimings { + return m.BackendRequestTimingValues } func (m *MockTestOptimizationClient) StoreCacheAndExit() { @@ -865,30 +869,29 @@ func TestTestPlanner_Plan_JestSuiteSkippingFetchesSkippablesWithoutFullDiscovery if !mockOptimizationClient.GetSkippablesCalled { t.Fatal("expected planner to fetch suite skippables when full test discovery is unsupported") } - if runner.planReport.Skippables.TestSkippingLevel != settings.TestSkippingLevelSuite || - runner.planReport.Skippables.TIATests != 0 || - runner.planReport.Skippables.TIASuites != 1 { - t.Errorf("expected suite-level skippables to be reported separately, got %+v", runner.planReport.Skippables) + report := runner.newPlanReportData(splitScore{}) + if report.Skippables.TestSkippingLevel != settings.TestSkippingLevelSuite || + report.Skippables.TIATests != 0 || + report.Skippables.TIASuites != 1 { + t.Errorf("expected suite-level skippables to be reported separately, got %+v", report.Skippables) } - if runner.planReport.Planning.Discovery.Mode != discoveryModeFast || - runner.planReport.Planning.Discovery.TestFiles != 2 || - runner.planReport.Planning.Discovery.Suites != 0 { - t.Errorf("expected fast discovery report with files but no local suites, got %+v", runner.planReport.Planning.Discovery) + if report.Planning.Discovery.Mode != discoveryModeFast || + report.Planning.Discovery.TestFiles != 2 || + report.Planning.Discovery.Suites != 0 { + t.Errorf("expected fast discovery report with files but no local suites, got %+v", report.Planning.Discovery) } - if cacheReport := runner.planReport.Planning.Discovery.Cache; !cacheReport.Configured || - cacheReport.Used || - cacheReport.Reason != "full discovery not required for suite-level skipping" { - t.Errorf("expected configured cache skip reason for fast discovery, got %+v", cacheReport) + if cacheResult := report.Planning.Discovery.Cache; cacheResult != (discoveryCacheResult{}) { + t.Errorf("expected cache result to be empty when full discovery is not applicable, got %+v", cacheResult) } - if runner.planReport.Planning.Durations.BackendDurationsApplied != 2 || - runner.planReport.Planning.Durations.BackendSuitesAdded != 2 || - runner.planReport.Planning.Durations.SuitesWithoutDurations != 0 { - t.Errorf("expected backend duration application report for fast discovery, got %+v", runner.planReport.Planning.Durations) + if report.Planning.Durations.BackendDurationsApplied != 2 || + report.Planning.Durations.BackendSuitesAdded != 2 || + report.Planning.Durations.SuitesWithoutDurations != 0 { + t.Errorf("expected backend duration application report for fast discovery, got %+v", report.Planning.Durations) } - if runner.planReport.Planning.Skipping.TIASuites != 1 || - runner.planReport.Planning.Skipping.FullySkippedFiles != 1 || - runner.planReport.Planning.TestFilesToRun != 1 { - t.Errorf("expected suite skip application report, got planning=%+v", runner.planReport.Planning) + if report.Planning.Skipping.TIASuites != 1 || + report.Planning.Skipping.FullySkippedFiles != 1 || + report.Planning.TestFilesToRun != 1 { + t.Errorf("expected suite skip application report, got planning=%+v", report.Planning) } expectedTestFiles := "src/b.test.ts\n" @@ -962,8 +965,8 @@ func TestTestPlanner_PreparePlanningData_RubySuiteModeSkipsFullDiscoveryAndSkips if _, ok := runner.testFileWeights["spec/models/payment_spec.rb"]; !ok { t.Fatalf("expected runnable suite file to remain, got weights: %+v", runner.testFileWeights) } - if runner.planInfo.TestSkippingLevel != settings.TestSkippingLevelSuite.String() { - t.Fatalf("plan test skipping level = %q, want suite", runner.planInfo.TestSkippingLevel) + if runner.planMetadata.TestSkippingLevel != settings.TestSkippingLevelSuite.String() { + t.Fatalf("plan test skipping level = %q, want suite", runner.planMetadata.TestSkippingLevel) } } @@ -1045,15 +1048,16 @@ func TestTestPlanner_PreparePlanningData_RubySuiteModeForceFullDiscovery(t *test if guarded.NumTests != 1 || guarded.NumTestsSkipped != 0 { t.Fatalf("expected guarded suite to remain runnable, got %+v", guarded) } - if runner.planReport.Planning.Discovery.Mode != discoveryModeFull || - runner.planReport.Planning.Discovery.Suites != 4 || - runner.planReport.Planning.Discovery.Tests != 5 { - t.Errorf("expected full discovery suite/test report, got %+v", runner.planReport.Planning.Discovery) + report := runner.newPlanReportData(splitScore{}) + if report.Planning.Discovery.Mode != discoveryModeFull || + report.Planning.Discovery.Suites != 4 || + report.Planning.Discovery.Tests != 5 { + t.Errorf("expected full discovery suite/test report, got %+v", report.Planning.Discovery) } - if runner.planReport.Planning.Skipping.TIASuites != 3 || - runner.planReport.Planning.Skipping.UnskippableMarkerSuitesForced != 1 || - runner.planReport.Planning.Skipping.FullySkippedFiles != 1 { - t.Errorf("expected full discovery skipping application report, got %+v", runner.planReport.Planning.Skipping) + if report.Planning.Skipping.TIASuites != 3 || + report.Planning.Skipping.UnskippableMarkerSuitesForced != 1 || + report.Planning.Skipping.FullySkippedFiles != 1 { + t.Errorf("expected full discovery skipping application report, got %+v", report.Planning.Skipping) } } @@ -2018,10 +2022,11 @@ func TestTestPlanner_PreparePlanningData_Success(t *testing.T) { if !mockOptimizationClient.DurationsCalled { t.Error("PreparePlanningData() should fetch test suite durations") } - if !runner.planReport.TestSuiteDurations.Available || - runner.planReport.TestSuiteDurations.Modules != 1 || - runner.planReport.TestSuiteDurations.Suites != 1 { - t.Errorf("Expected plan report to summarize fetched durations, got %+v", runner.planReport.TestSuiteDurations) + report := runner.newPlanReportData(splitScore{}) + if !report.TestSuiteDurations.Available || + report.TestSuiteDurations.Modules != 1 || + report.TestSuiteDurations.Suites != 1 { + t.Errorf("Expected plan report to summarize fetched durations, got %+v", report.TestSuiteDurations) } } @@ -2109,8 +2114,10 @@ func TestTestPlanner_PreparePlanningData_DisabledTestManagementTestsAreSkipped(t t.Errorf("Expected attempt-to-fix file to be scheduled, got %v", runner.testFileWeights) } - if runner.planReport.Skippables.TIATests != 1 || runner.planReport.Skippables.TIASuites != 0 || runner.planReport.Skippables.DisabledTests != 1 { - t.Errorf("Expected planner report to split TIA and disabled skippables, got %+v", runner.planReport.Skippables) + report := runner.newPlanReportData(splitScore{}) + if report.Skippables.TIATests != 1 || report.Skippables.TIASuites != 0 || report.Planning.Skipping.DisabledTests != 1 { + t.Errorf("Expected planner report to split TIA skippables and disabled tests, got skippables=%+v planning=%+v", + report.Skippables, report.Planning.Skipping) } } @@ -2301,8 +2308,10 @@ func TestTestPlanner_PreparePlanningData_TestManagementDoesNotKeepFullDiscoveryW t.Errorf("Expected fast discovery fallback to keep all discovered files, got %v", runner.testFileWeights) } - if runner.planReport.Skippables.TIATests != 0 || runner.planReport.Skippables.TIASuites != 0 || runner.planReport.Skippables.DisabledTests != 2 { - t.Errorf("Expected planner report to split TIA and disabled skippables, got %+v", runner.planReport.Skippables) + report := runner.newPlanReportData(splitScore{}) + if report.Skippables.TIATests != 0 || report.Skippables.TIASuites != 0 || report.Planning.Skipping.DisabledTests != 0 { + t.Errorf("Expected planner report to split TIA skippables and disabled tests, got skippables=%+v planning=%+v", + report.Skippables, report.Planning.Skipping) } } @@ -2491,10 +2500,11 @@ func TestTestPlanner_PreparePlanningData_EmptyDurationsContinues(t *testing.T) { if len(runner.testSuiteDurations) != 0 { t.Errorf("Expected empty in-memory test suite durations on empty response, got %v", runner.testSuiteDurations) } - if !runner.planReport.TestSuiteDurations.Available || - runner.planReport.TestSuiteDurations.Modules != 0 || - runner.planReport.TestSuiteDurations.Suites != 0 { - t.Errorf("Expected plan report to summarize empty durations response, got %+v", runner.planReport.TestSuiteDurations) + report := runner.newPlanReportData(splitScore{}) + if !report.TestSuiteDurations.Available || + report.TestSuiteDurations.Modules != 0 || + report.TestSuiteDurations.Suites != 0 { + t.Errorf("Expected plan report to summarize empty durations response, got %+v", report.TestSuiteDurations) } } @@ -2545,10 +2555,11 @@ func TestTestPlanner_PreparePlanningData_NonEmptyDurationsUsesP50ForMatchingSuit if len(runner.testSuiteDurations) != 1 { t.Fatalf("Expected stored durations data, got %v", runner.testSuiteDurations) } - if !runner.planReport.TestSuiteDurations.Available || - runner.planReport.TestSuiteDurations.Modules != 1 || - runner.planReport.TestSuiteDurations.Suites != 2 { - t.Errorf("Expected plan report to summarize fetched durations, got %+v", runner.planReport.TestSuiteDurations) + report := runner.newPlanReportData(splitScore{}) + if !report.TestSuiteDurations.Available || + report.TestSuiteDurations.Modules != 1 || + report.TestSuiteDurations.Suites != 2 { + t.Errorf("Expected plan report to summarize fetched durations, got %+v", report.TestSuiteDurations) } if _, ok := runner.testFiles["spec/file1_test.rb"]; !ok { diff --git a/internal/planner/report.go b/internal/planner/report.go index dad589b..a521cda 100644 --- a/internal/planner/report.go +++ b/internal/planner/report.go @@ -14,10 +14,14 @@ import ( "github.com/DataDog/ddtest/internal/settings" ) -func printPlanReport(w io.Writer, report planReport) { +func printPlanReport(w io.Writer, tp *TestPlanner, split splitScore) { + printPlanReportData(w, tp.newPlanReportData(split)) +} + +func printPlanReportData(w io.Writer, report PlanReportData) { reportFprintln(w, "+++ DDTest: plan report") reportFprintln(w) - printRunInfoReport(w, report.RunInfo, report.PlanInfo) + printRunInfoReport(w, report.RunInfo, report.PlanMetadata) reportFprintln(w) printDDTestSettingsReport(w, report.DDTestSettings) reportFprintln(w) @@ -32,15 +36,15 @@ func printPlanReport(w io.Writer, report planReport) { printSlowestTestSuitesOverallReport(w, report.SlowestTestSuitesOverall) } -func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planInfo PlanInfo) { +func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planMetadata PlanMetadata) { reportFprintln(w, "Run") reportFprintf(w, " Service: %s\n", valueOrNotAvailable(runInfo.Service)) reportFprintf(w, " Repository: %s\n", valueOrNotAvailable(runInfo.Repository)) reportFprintf(w, " Commit: %s\n", valueOrNotAvailable(runInfo.Commit)) reportFprintf(w, " Branch: %s\n", valueOrNotAvailable(runInfo.Branch)) - reportFprintf(w, " Platform: %s\n", formatPlatform(planInfo.Platform, planInfo.Framework)) - reportFprintf(w, " OS tags: %s\n", formatTagList(planInfo.OSTags, constants.OSPlatform, constants.OSArchitecture, constants.OSVersion)) - reportFprintf(w, " Runtime tags: %s\n", formatTagList(planInfo.RuntimeTags, constants.RuntimeName, constants.RuntimeVersion)) + reportFprintf(w, " Platform: %s\n", formatPlatform(planMetadata.Platform, planMetadata.Framework)) + reportFprintf(w, " OS tags: %s\n", formatTagList(planMetadata.OSTags, constants.OSPlatform, constants.OSArchitecture, constants.OSVersion)) + reportFprintf(w, " Runtime tags: %s\n", formatTagList(planMetadata.RuntimeTags, constants.RuntimeName, constants.RuntimeVersion)) } func printDDTestSettingsReport(w io.Writer, config *settings.Config) { @@ -56,7 +60,7 @@ func printDDTestSettingsReport(w io.Writer, config *settings.Config) { } func printChangedDDTestSettings(w io.Writer, config *settings.Config) bool { - defaults := settings.DefaultConfig() + defaults := defaultDDTestSettings() configValue := reflect.ValueOf(*config) defaultsValue := reflect.ValueOf(defaults) configType := configValue.Type() @@ -77,14 +81,13 @@ func printChangedDDTestSettings(w io.Writer, config *settings.Config) bool { } func formatDDTestSettingName(field reflect.StructField) string { - if label := field.Tag.Get("report"); label != "" { - return label - } - key := configFieldKey(field) if key == "" { return field.Name } + if key == "parallel_runner_overhead" { + return "CI job overhead" + } words := strings.Split(key, "_") for i, word := range words { @@ -128,6 +131,20 @@ func configFieldKey(field reflect.StructField) string { return key } +func defaultDDTestSettings() settings.Config { + return settings.Config{ + Platform: "ruby", + Framework: "rspec", + MinParallelism: settings.DefaultParallelism(), + MaxParallelism: settings.DefaultParallelism(), + ParallelRunnerOverhead: settings.DefaultParallelRunnerOverhead(), + CiNode: -1, + CiNodeWorkers: 1, + TestSkippingLevel: settings.TestSkippingLevelTest, + ReportEnabled: true, + } +} + func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { reportFprintln(w, "Datadog settings") if !report.Available { @@ -146,7 +163,7 @@ func printDatadogSettingsReport(w io.Writer, report datadogSettingsReport) { reportFprintf(w, " Flaky test management: %s\n", enabledWord(report.FlakyTestManagement)) } -func printBackendDataReport(w io.Writer, report planReport) { +func printBackendDataReport(w io.Writer, report PlanReportData) { reportFprintln(w, "Backend data") reportFprintf(w, " Known tests: %s\n", formatBackendDataValue(formatKnownTests(report.DatadogSettings, report.KnownTests), report.KnownTests.FetchDuration)) reportFprintf(w, " TIA skippables returned: %s\n", formatBackendDataValue(formatTIASkippables(report.DatadogSettings, report.Skippables), report.Skippables.FetchDuration)) @@ -154,11 +171,11 @@ func printBackendDataReport(w io.Writer, report planReport) { reportFprintf(w, " Test suite durations: %s\n", formatBackendDataValue(formatTestSuiteDurations(report.TestSuiteDurations), report.TestSuiteDurations.FetchDuration)) } -func printPlanningReport(w io.Writer, report planReport) { +func printPlanningReport(w io.Writer, report PlanReportData) { reportFprintln(w, "Planning") printDiscoveryPlanningReport(w, report.Planning.Discovery) printDurationEstimatesPlanningReport(w, report.Planning.Durations) - printSkippingPlanningReport(w, report.DatadogSettings, report.Skippables, report.Planning.Skipping) + printSkippingPlanningReport(w, report.DatadogSettings, report.Skippables, report.ManagedFlakyTests, report.Planning.Skipping) printRunSetPlanningReport(w, report.Planning) printRunnerSplitPlanningReport(w, report.Planning, report.Split) } @@ -276,7 +293,9 @@ func printDiscoveryPlanningReport(w io.Writer, discovery discoveryReport) { reportFprintln(w, " Discovery") reportFprintf(w, " Method: %s\n", valueOrNotAvailable(string(discovery.Mode))) reportFprintf(w, " Test files: %s\n", formatCount(discovery.TestFiles)) - reportFprintf(w, " Cache: %s\n", formatDiscoveryCache(discovery.Cache)) + if discovery.Cache.Configured || discovery.Cache.Used || discovery.Cache.NotUsedReason != "" { + reportFprintf(w, " Cache: %s\n", formatDiscoveryCache(discovery.Cache)) + } reportFprintf(w, " Duration: %s\n", formatOptionalDuration(discovery.Duration)) switch discovery.Mode { case discoveryModeFull: @@ -285,17 +304,17 @@ func printDiscoveryPlanningReport(w io.Writer, discovery discoveryReport) { } } -func formatDiscoveryCache(cache discoveryCacheReport) string { +func formatDiscoveryCache(cache discoveryCacheResult) string { if cache.Used { return "used" } if !cache.Configured { return "not configured" } - if cache.Reason == "" { + if cache.NotUsedReason == "" { return "not used" } - return "not used (" + cache.Reason + ")" + return "not used (" + cache.NotUsedReason + ")" } func printFetchDuration(w io.Writer, duration time.Duration) { @@ -330,7 +349,13 @@ func printDurationEstimatesPlanningReport(w io.Writer, durations durationApplica reportFprintf(w, " Backend-only suites added: %s\n", formatCount(durations.BackendSuitesAdded)) } -func printSkippingPlanningReport(w io.Writer, datadogSettings datadogSettingsReport, skippables skippablesReport, skipping skippingApplicationReport) { +func printSkippingPlanningReport( + w io.Writer, + datadogSettings datadogSettingsReport, + skippables skippablesReport, + managed managedFlakyTestsReport, + skipping skippingApplicationReport, +) { if !skipping.Available { reportFprintln(w, " Skipping: not available") return @@ -338,7 +363,7 @@ func printSkippingPlanningReport(w io.Writer, datadogSettings datadogSettingsRep reportFprintln(w, " Skipping") reportFprintf(w, " TIA skippables applied: %s\n", formatAppliedTIASkippables(datadogSettings, skippables, skipping)) - reportFprintf(w, " Disabled tests applied: %s\n", formatAppliedDisabledTests(skippables, skipping)) + reportFprintf(w, " Disabled tests applied: %s\n", formatAppliedDisabledTests(datadogSettings, managed, skipping)) reportFprintf(w, " Suites marked unskippable: %s\n", formatCount(skipping.UnskippableMarkerSuitesForced)) reportFprintf(w, " Files fully skipped: %s\n", formatCount(skipping.FullySkippedFiles)) } @@ -395,8 +420,11 @@ func formatAppliedTIASkippables(datadogSettings datadogSettingsReport, skippable } } -func formatAppliedDisabledTests(skippables skippablesReport, skipping skippingApplicationReport) string { - if !skippables.Available { +func formatAppliedDisabledTests(datadogSettings datadogSettingsReport, managed managedFlakyTestsReport, skipping skippingApplicationReport) string { + if datadogSettings.Available && !datadogSettings.FlakyTestManagement { + return "disabled" + } + if !managed.Available { return "not available" } return formatCountWithUnit(skipping.DisabledTests, "test", "tests") diff --git a/internal/planner/report_data.go b/internal/planner/report_data.go index 1490b10..20d05b6 100644 --- a/internal/planner/report_data.go +++ b/internal/planner/report_data.go @@ -22,7 +22,29 @@ type datadogSettingsReport struct { FlakyTestManagement bool } -func newDatadogSettingsReport(settings *api.SettingsResponseData, fetchDuration time.Duration) datadogSettingsReport { +func addBackendDataReports(report PlanReportData, client testOptimizationClient) PlanReportData { + timings := client.BackendRequestTimings() + + report.DatadogSettings = reportDatadogSettings(client.GetSettings(), timings.Settings) + report.KnownTests = reportKnownTests(client.GetKnownTests(), timings.KnownTests) + report.Skippables = reportSkippables( + client.GetSkippables(), + settings.TestSkippingLevel(report.PlanMetadata.TestSkippingLevel), + timings.Skippables, + ) + report.ManagedFlakyTests = reportManagedFlakyTests( + client.GetTestManagementTestsData(), + timings.TestManagementTests, + ) + report.TestSuiteDurations = reportTestSuiteDurations( + client.GetTestSuiteDurations(), + timings.TestSuiteDurations, + ) + + return report +} + +func reportDatadogSettings(settings *api.SettingsResponseData, fetchDuration time.Duration) datadogSettingsReport { if settings == nil { return datadogSettingsReport{FetchDuration: fetchDuration} } @@ -47,7 +69,7 @@ type knownTestsReport struct { Tests int } -func newKnownTestsReport(knownTests *api.KnownTestsResponseData, fetchDuration time.Duration) knownTestsReport { +func reportKnownTests(knownTests *api.KnownTestsResponseData, fetchDuration time.Duration) knownTestsReport { if knownTests == nil { return knownTestsReport{FetchDuration: fetchDuration} } @@ -75,7 +97,7 @@ type managedFlakyTestsReport struct { AttemptToFix int } -func newManagedFlakyTestsReport( +func reportManagedFlakyTests( testManagementTests *api.TestManagementTestsResponseDataModules, fetchDuration time.Duration, ) managedFlakyTestsReport { @@ -109,21 +131,21 @@ type skippablesReport struct { TestSkippingLevel settings.TestSkippingLevel TIATests int TIASuites int - DisabledTests int } -func newSkippablesReport( - skippables skippableMatcher, +func reportSkippables( + skippables api.Skippables, testSkippingLevel settings.TestSkippingLevel, fetchDuration time.Duration, ) skippablesReport { return skippablesReport{ - Available: true, + Available: skippables.Tests != nil || + skippables.Suites != nil || + fetchDuration > 0, FetchDuration: fetchDuration, TestSkippingLevel: testSkippingLevel, - TIATests: skippables.TIATestsCount(), - TIASuites: skippables.TIASuitesCount(), - DisabledTests: skippables.DisabledTestsCount(), + TIATests: len(skippables.Tests), + TIASuites: len(skippables.Suites), } } @@ -134,7 +156,7 @@ type testSuiteDurationsReport struct { Suites int } -func newTestSuiteDurationsReport( +func reportTestSuiteDurations( testSuiteDurations *api.TestSuiteDurationsResponseData, fetchDuration time.Duration, ) testSuiteDurationsReport { @@ -166,27 +188,20 @@ type testSuiteTimingReport struct { type discoveryMode string const ( - discoveryModeUnknown discoveryMode = "" - discoveryModeFast discoveryMode = "fast" - discoveryModeFull discoveryMode = "full" + discoveryModeFast discoveryMode = "fast" + discoveryModeFull discoveryMode = "full" ) type discoveryReport struct { Available bool Mode discoveryMode - Cache discoveryCacheReport + Cache discoveryCacheResult Duration time.Duration TestFiles int Suites int Tests int } -type discoveryCacheReport struct { - Configured bool - Used bool - Reason string -} - type durationApplicationReport struct { Available bool BackendDurationsApplied int @@ -213,9 +228,9 @@ type planningReport struct { EstimatedTimeSaved float64 } -type planReport struct { +type PlanReportData struct { RunInfo runmetadata.RunInfo - PlanInfo PlanInfo + PlanMetadata PlanMetadata DDTestSettings *settings.Config DatadogSettings datadogSettingsReport KnownTests knownTestsReport @@ -228,36 +243,78 @@ type planReport struct { Split splitScore } +type planningReportStats struct { + discoveryMode discoveryMode + discoveryCache discoveryCacheResult + discoveryDuration time.Duration + tiaSkippableTestsApplied int + uniqueTIASkippableSuitesApplied map[testSuiteKey]struct{} + disabledTestsApplied int + unskippableMarkerSuitesForced int +} + +func newPlanningReportStats() planningReportStats { + return planningReportStats{ + uniqueTIASkippableSuitesApplied: make(map[testSuiteKey]struct{}), + } +} + +func (tp *TestPlanner) newPlanReportData(split splitScore) PlanReportData { + report := PlanReportData{ + RunInfo: tp.runInfo, + PlanMetadata: tp.planMetadata, + DDTestSettings: settings.Get(), + Planning: tp.newPlanningReport(), + LongSeparateRunnerSuites: tp.longSeparateRunnerSuitesReport(split.parallelRunners, split), + SlowestTestSuitesOverall: tp.slowestTestSuitesOverallReport(slowestTestSuitesReportLimit), + Split: split, + } + return addBackendDataReports(report, tp.optimizationClient) +} + +func (tp *TestPlanner) recordDiscoveryReport(mode discoveryMode, cache discoveryCacheResult, duration time.Duration) { + tp.reportStats.discoveryMode = mode + tp.reportStats.discoveryCache = cache + tp.reportStats.discoveryDuration = duration +} + func (tp *TestPlanner) newPlanningReport() planningReport { fullySkippedFiles := len(tp.testFiles) - len(tp.testFileWeights) if fullySkippedFiles < 0 { fullySkippedFiles = 0 } + discoveredSuites := 0 + discoveredTests := 0 + mode := tp.reportStats.discoveryMode + if mode == discoveryModeFull { + discoveredSuites = len(tp.suiteAggregates) + discoveredTests = countSuiteAggregateTests(tp.suiteAggregates) + } return planningReport{ Discovery: discoveryReport{ Available: true, - Mode: tp.discoveryMode, - Cache: tp.discoveryCacheReport, - Duration: tp.testDiscoveryDuration, + Mode: mode, + Cache: tp.reportStats.discoveryCache, + Duration: tp.reportStats.discoveryDuration, TestFiles: len(tp.testFiles), - Suites: tp.localDiscoveredSuites, - Tests: tp.localDiscoveredTests, + Suites: discoveredSuites, + Tests: discoveredTests, }, Durations: durationApplicationReport{ Available: true, BackendDurationsApplied: tp.backendDurationApplicationsCount(), - BackendSuitesAdded: tp.backendSuitesAddedCount(), + BackendSuitesAdded: tp.backendSuitesAddedCount(mode), SuitesWithoutDurations: tp.suitesWithoutBackendDurationsCount(), FilesWithoutDurations: tp.filesWithoutBackendDurationsCount(), ExpectedFullDuration: tp.expectedFullDuration(), }, Skipping: skippingApplicationReport{ Available: true, - TIATests: tp.tiaSkippableTestsApplied, - TIASuites: len(tp.tiaSkippableSuitesApplied), - DisabledTests: tp.disabledTestsApplied, - UnskippableMarkerSuitesForced: tp.unskippableMarkerSuitesForced, + TIATests: tp.reportStats.tiaSkippableTestsApplied, + TIASuites: len(tp.reportStats.uniqueTIASkippableSuitesApplied), + DisabledTests: tp.reportStats.disabledTestsApplied, + UnskippableMarkerSuitesForced: tp.reportStats.unskippableMarkerSuitesForced, FullySkippedFiles: fullySkippedFiles, }, TestFilesToRun: len(tp.testFileWeights), @@ -275,12 +332,11 @@ func (tp *TestPlanner) backendDurationApplicationsCount() int { return count } -func (tp *TestPlanner) backendSuitesAddedCount() int { - count := len(tp.suiteAggregates) - tp.localDiscoveredSuites - if count < 0 { +func (tp *TestPlanner) backendSuitesAddedCount(mode discoveryMode) int { + if mode == discoveryModeFull { return 0 } - return count + return len(tp.suiteAggregates) } func (tp *TestPlanner) suitesWithoutBackendDurationsCount() int { diff --git a/internal/planner/report_test.go b/internal/planner/report_test.go index 03392c3..654085d 100644 --- a/internal/planner/report_test.go +++ b/internal/planner/report_test.go @@ -9,6 +9,7 @@ import ( "github.com/DataDog/ddtest/internal/runmetadata" "github.com/DataDog/ddtest/internal/settings" + "github.com/DataDog/ddtest/internal/testoptimization" "github.com/DataDog/ddtest/internal/testoptimization/api" ) @@ -17,14 +18,14 @@ func TestPrintPlanReport_AllData(t *testing.T) { minParallelism := settings.DefaultParallelism() + 1 maxParallelism := settings.DefaultParallelism() + 2 - printPlanReport(&output, planReport{ + printPlanReportData(&output, PlanReportData{ RunInfo: runmetadata.RunInfo{ Service: "checkout-api", Repository: "https://github.com/acme/checkout.git", Commit: "9f3a1c7d2b4e", Branch: "feature/split-report", }, - PlanInfo: PlanInfo{ + PlanMetadata: PlanMetadata{ Platform: "ruby", Framework: "rspec", OSTags: map[string]string{ @@ -79,7 +80,6 @@ func TestPrintPlanReport_AllData(t *testing.T) { FetchDuration: 110 * time.Millisecond, TestSkippingLevel: settings.TestSkippingLevelSuite, TIASuites: 312, - DisabledTests: 3, }, ManagedFlakyTests: managedFlakyTestsReport{ Available: true, @@ -99,7 +99,7 @@ func TestPrintPlanReport_AllData(t *testing.T) { Discovery: discoveryReport{ Available: true, Mode: discoveryModeFull, - Cache: discoveryCacheReport{ + Cache: discoveryCacheResult{ Configured: true, Used: true, }, @@ -252,11 +252,11 @@ Slow suites on dedicated runners func TestPrintPlanReport_FastDiscovery(t *testing.T) { var output strings.Builder - printPlanReport(&output, planReport{ + printPlanReportData(&output, PlanReportData{ RunInfo: runmetadata.RunInfo{ Service: "checkout-api", }, - PlanInfo: PlanInfo{ + PlanMetadata: PlanMetadata{ Platform: "ruby", }, DatadogSettings: datadogSettingsReport{ @@ -290,10 +290,6 @@ func TestPrintPlanReport_FastDiscovery(t *testing.T) { Discovery: discoveryReport{ Available: true, Mode: discoveryModeFast, - Cache: discoveryCacheReport{ - Configured: true, - Reason: "full discovery not required for suite-level skipping", - }, Duration: 120 * time.Millisecond, TestFiles: 24, Suites: 0, @@ -372,7 +368,6 @@ Planning Discovery Method: fast Test files: 24 - Cache: not used (full discovery not required for suite-level skipping) Duration: 120ms Duration estimates Backend durations used: 12 suites @@ -380,7 +375,7 @@ Planning Backend-only suites added: 2 Skipping TIA skippables applied: 8 suites - Disabled tests applied: 0 tests + Disabled tests applied: disabled Suites marked unskippable: 0 Files fully skipped: 6 Run set @@ -408,7 +403,7 @@ Slow suites on dedicated runners func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { var output strings.Builder - printPlanReport(&output, planReport{}) + printPlanReportData(&output, PlanReportData{}) report := output.String() if !strings.Contains(report, "DDTest settings\n Settings: not available") { @@ -448,9 +443,9 @@ func TestPrintPlanReport_MissingSettingsAndData(t *testing.T) { func TestPrintPlanReport_DefaultSettings(t *testing.T) { var output strings.Builder - defaults := settings.DefaultConfig() + defaults := defaultDDTestSettings() - printPlanReport(&output, planReport{ + printPlanReportData(&output, PlanReportData{ DDTestSettings: &defaults, }) @@ -461,7 +456,7 @@ func TestPrintPlanReport_DefaultSettings(t *testing.T) { } func TestPrintDDTestSettingsReport_AllSupportedSettings(t *testing.T) { - config := settings.DefaultConfig() + config := defaultDDTestSettings() config.Platform = "python" config.Framework = "pytest" config.Command = "pytest -q" @@ -524,7 +519,7 @@ func TestPrintDDTestSettingsReport_AllSupportedSettings(t *testing.T) { func TestPrintPlanReport_DisabledFeatures(t *testing.T) { var output strings.Builder - printPlanReport(&output, planReport{ + printPlanReportData(&output, PlanReportData{ DatadogSettings: datadogSettingsReport{ Available: true, }, @@ -543,42 +538,39 @@ func TestPrintPlanReport_DisabledFeatures(t *testing.T) { } func TestReportSummaries(t *testing.T) { - known := newKnownTestsReport(&api.KnownTestsResponseData{ - Tests: api.KnownTestsResponseDataModules{ - "module-a": api.KnownTestsResponseDataSuites{ - "suite-a": []string{"test-a", "test-b"}, - }, - "module-b": api.KnownTestsResponseDataSuites{ - "suite-b": []string{"test-c"}, - "suite-c": []string{"test-d", "test-e"}, + client := &MockTestOptimizationClient{ + KnownTests: &api.KnownTestsResponseData{ + Tests: api.KnownTestsResponseDataModules{ + "module-a": api.KnownTestsResponseDataSuites{ + "suite-a": []string{"test-a", "test-b"}, + }, + "module-b": api.KnownTestsResponseDataSuites{ + "suite-b": []string{"test-c"}, + "suite-c": []string{"test-d", "test-e"}, + }, }, }, - }, time.Millisecond) - if known.Modules != 2 || known.Suites != 3 || known.Tests != 5 { - t.Errorf("unexpected known test summary: %+v", known) - } - - managed := newManagedFlakyTestsReport(&api.TestManagementTestsResponseDataModules{ - Modules: map[string]api.TestManagementTestsResponseDataSuites{ - "module-a": { - Suites: map[string]api.TestManagementTestsResponseDataTests{ - "suite-a": { - Tests: map[string]api.TestManagementTestsResponseDataTestProperties{ - "test-a": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{Quarantined: true}}, - "test-b": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{Disabled: true}}, - "test-c": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{AttemptToFix: true}}, + TestManagementTests: &api.TestManagementTestsResponseDataModules{ + Modules: map[string]api.TestManagementTestsResponseDataSuites{ + "module-a": { + Suites: map[string]api.TestManagementTestsResponseDataTests{ + "suite-a": { + Tests: map[string]api.TestManagementTestsResponseDataTestProperties{ + "test-a": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{Quarantined: true}}, + "test-b": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{Disabled: true}}, + "test-c": {Properties: api.TestManagementTestsResponseDataTestPropertiesAttributes{AttemptToFix: true}}, + }, }, }, }, }, }, - }, time.Millisecond) - if managed.Total != 3 || managed.Quarantined != 1 || managed.Disabled != 1 || managed.AttemptToFix != 1 { - t.Errorf("unexpected managed flaky test summary: %+v", managed) - } - - durations := newTestSuiteDurationsReport(&api.TestSuiteDurationsResponseData{ - TestSuites: map[string]map[string]api.TestSuiteDurationInfo{ + BackendRequestTimingValues: testoptimization.BackendRequestTimings{ + KnownTests: time.Millisecond, + TestManagementTests: time.Millisecond, + TestSuiteDurations: time.Millisecond, + }, + Durations: map[string]map[string]api.TestSuiteDurationInfo{ "module-a": { "suite-a": {}, "suite-b": {}, @@ -587,9 +579,24 @@ func TestReportSummaries(t *testing.T) { "suite-c": {}, }, }, - }, time.Millisecond) - if durations.Modules != 2 || durations.Suites != 3 { - t.Errorf("unexpected test suite durations summary: %+v", durations) + } + + client.GetTestSuiteDurations() + timings := client.BackendRequestTimings() + + known := reportKnownTests(client.GetKnownTests(), timings.KnownTests) + if known.Modules != 2 || known.Suites != 3 || known.Tests != 5 { + t.Errorf("unexpected known test summary: %+v", known) + } + + managed := reportManagedFlakyTests(client.GetTestManagementTestsData(), timings.TestManagementTests) + if managed.Total != 3 || managed.Quarantined != 1 || managed.Disabled != 1 || managed.AttemptToFix != 1 { + t.Errorf("unexpected managed flaky test summary: %+v", managed) + } + + durationReport := reportTestSuiteDurations(client.GetTestSuiteDurations(), timings.TestSuiteDurations) + if durationReport.Modules != 2 || durationReport.Suites != 3 { + t.Errorf("unexpected test suite durations summary: %+v", durationReport) } } @@ -653,10 +660,10 @@ func TestReportFormattingVariants(t *testing.T) { {name: "three digit group count", got: formatCount(123456), want: "123,456"}, {name: "singular count unit", got: formatCountWithUnit(1, "suite", "suites"), want: "1 suite"}, {name: "sub-millisecond duration", got: formatDuration(500 * time.Microsecond), want: "500µs"}, - {name: "cache not configured", got: formatDiscoveryCache(discoveryCacheReport{}), want: "not configured"}, - {name: "cache used", got: formatDiscoveryCache(discoveryCacheReport{Configured: true, Used: true}), want: "used"}, - {name: "cache configured but not used", got: formatDiscoveryCache(discoveryCacheReport{Configured: true}), want: "not used"}, - {name: "cache configured but skipped with reason", got: formatDiscoveryCache(discoveryCacheReport{Configured: true, Reason: "full discovery not required"}), want: "not used (full discovery not required)"}, + {name: "cache not configured", got: formatDiscoveryCache(discoveryCacheResult{}), want: "not configured"}, + {name: "cache used", got: formatDiscoveryCache(discoveryCacheResult{Configured: true, Used: true}), want: "used"}, + {name: "cache configured but not used", got: formatDiscoveryCache(discoveryCacheResult{Configured: true}), want: "not used"}, + {name: "cache configured but skipped with reason", got: formatDiscoveryCache(discoveryCacheResult{Configured: true, NotUsedReason: "full discovery not required"}), want: "not used (full discovery not required)"}, {name: "backend data without duration", got: formatBackendDataValue("not available", 0), want: "not available"}, {name: "optional duration missing", got: formatOptionalDuration(0), want: "not available"}, } @@ -752,9 +759,12 @@ func TestSkippableReportFormattingVariants(t *testing.T) { }) t.Run("disabled tests", func(t *testing.T) { - if got := formatAppliedDisabledTests(skippablesReport{}, skippingApplicationReport{}); got != "not available" { + if got := formatAppliedDisabledTests(datadogSettingsReport{}, managedFlakyTestsReport{}, skippingApplicationReport{}); got != "not available" { t.Fatalf("formatAppliedDisabledTests() = %q, want not available", got) } + if got := formatAppliedDisabledTests(datadogSettingsReport{Available: true}, managedFlakyTestsReport{}, skippingApplicationReport{}); got != "disabled" { + t.Fatalf("formatAppliedDisabledTests() = %q, want disabled", got) + } }) } diff --git a/internal/planner/test_optimization_plan_cache.go b/internal/planner/test_optimization_plan_cache.go index 32ebdc2..42560e0 100644 --- a/internal/planner/test_optimization_plan_cache.go +++ b/internal/planner/test_optimization_plan_cache.go @@ -1,7 +1,6 @@ package planner import ( - "encoding/json" "log/slog" "github.com/DataDog/ddtest/internal/runmetadata" @@ -16,7 +15,7 @@ type testOptimizationPlanCache struct { TestFileWeights map[string]int `json:"testFileWeights"` TestFileDurationSources map[string]testFileDurationSource `json:"testFileDurationSources"` RunInfo runmetadata.RunInfo `json:"runInfo"` - PlanInfo PlanInfo `json:"planInfo"` + PlanMetadata PlanMetadata `json:"planMetadata"` } func (tp *TestPlanner) storeTestOptimizationPlanCache() error { @@ -27,24 +26,24 @@ func (tp *TestPlanner) storeTestOptimizationPlanCache() error { TestFileWeights: tp.testFileWeights, TestFileDurationSources: tp.testFileDurationSources, RunInfo: tp.runInfo, - PlanInfo: tp.planInfo, + PlanMetadata: tp.planMetadata, } return testoptimization.NewCacheManager().StoreTestOptimizationPlanCache(cache) } -func LoadPlan() (PlanInfo, error) { +func LoadPlan() (PlanMetadata, error) { return newTestPlannerWithDefaults().LoadPlan() } -func (tp *TestPlanner) LoadPlan() (PlanInfo, error) { +func (tp *TestPlanner) LoadPlan() (PlanMetadata, error) { if !tp.planLoaded { if err := tp.restoreTestOptimizationPlanCache(); err != nil { - return PlanInfo{}, err + return PlanMetadata{}, err } } - return tp.planInfo, nil + return tp.planMetadata, nil } func (tp *TestPlanner) restoreTestOptimizationPlanCache() error { @@ -59,7 +58,7 @@ func (tp *TestPlanner) restoreTestOptimizationPlanCache() error { tp.testFileWeights = cache.TestFileWeights tp.testFileDurationSources = cache.TestFileDurationSources tp.runInfo = cache.RunInfo - tp.planInfo = cache.PlanInfo + tp.planMetadata = cache.PlanMetadata tp.planLoaded = true testSuitesCount := countTestSuites(tp.testSuiteDurations) @@ -77,63 +76,6 @@ func (tp *TestPlanner) restoreTestOptimizationPlanCache() error { return nil } -type legacyRunInfo struct { - Service string `json:"service"` - Repository string `json:"repository"` - Commit string `json:"commit"` - Branch string `json:"branch"` - Platform string `json:"platform"` - Framework string `json:"framework"` - OSTags map[string]string `json:"osTags"` - RuntimeTags map[string]string `json:"runtimeTags"` -} - -func (c *testOptimizationPlanCache) UnmarshalJSON(data []byte) error { - var decoded struct { - TestSuiteDurations map[string]map[string]api.TestSuiteDurationInfo `json:"testSuiteDurations"` - SuiteAggregates map[testSuiteKey]testSuiteAggregate `json:"suiteAggregates"` - SuitesBySourceFile map[string][]testSuiteKey `json:"suitesBySourceFile"` - TestFileWeights map[string]int `json:"testFileWeights"` - TestFileDurationSources map[string]testFileDurationSource `json:"testFileDurationSources"` - RunInfo legacyRunInfo `json:"runInfo"` - PlanInfo PlanInfo `json:"planInfo"` - } - if err := json.Unmarshal(data, &decoded); err != nil { - return err - } - - c.TestSuiteDurations = decoded.TestSuiteDurations - c.SuiteAggregates = decoded.SuiteAggregates - c.SuitesBySourceFile = decoded.SuitesBySourceFile - c.TestFileWeights = decoded.TestFileWeights - c.TestFileDurationSources = decoded.TestFileDurationSources - c.RunInfo = decoded.RunInfo.runInfo() - c.PlanInfo = decoded.PlanInfo - if c.PlanInfo.IsZero() { - c.PlanInfo = decoded.RunInfo.planInfo() - } - - return nil -} - -func (r legacyRunInfo) runInfo() runmetadata.RunInfo { - return runmetadata.RunInfo{ - Service: r.Service, - Repository: r.Repository, - Commit: r.Commit, - Branch: r.Branch, - } -} - -func (r legacyRunInfo) planInfo() PlanInfo { - return PlanInfo{ - Platform: r.Platform, - Framework: r.Framework, - OSTags: r.OSTags, - RuntimeTags: r.RuntimeTags, - } -} - func readAndNormalizeTestOptimizationPlanCache(cache *testOptimizationPlanCache) error { if err := testoptimization.NewCacheManager().ReadTestOptimizationPlanCache(cache); err != nil { return err diff --git a/internal/planner/test_optimization_plan_cache_test.go b/internal/planner/test_optimization_plan_cache_test.go index a83cee7..27f8de5 100644 --- a/internal/planner/test_optimization_plan_cache_test.go +++ b/internal/planner/test_optimization_plan_cache_test.go @@ -55,6 +55,13 @@ func TestTestPlanner_Plan_StoresTestOptimizationPlanCache(t *testing.T) { if _, err := os.Stat(cachePath); err != nil { t.Fatalf("Expected test optimization plan cache file to be written: %v", err) } + cacheData, err := os.ReadFile(cachePath) + if err != nil { + t.Fatalf("Expected test optimization plan cache file to be readable: %v", err) + } + if !strings.Contains(string(cacheData), `"planMetadata"`) { + t.Fatalf("Expected test optimization plan cache to store planMetadata, got: %s", string(cacheData)) + } restored := NewWithDependencies( &MockPlatformDetector{}, @@ -74,8 +81,8 @@ func TestTestPlanner_Plan_StoresTestOptimizationPlanCache(t *testing.T) { if !reflect.DeepEqual(restored.testFileWeights, runner.testFileWeights) { t.Errorf("Expected restored test file weights to match stored weights.\nexpected: %v\nactual: %v", runner.testFileWeights, restored.testFileWeights) } - if !reflect.DeepEqual(restored.planInfo, runner.planInfo) { - t.Errorf("Expected restored plan info to match stored plan info.\nexpected: %v\nactual: %v", runner.planInfo, restored.planInfo) + if !reflect.DeepEqual(restored.planMetadata, runner.planMetadata) { + t.Errorf("Expected restored plan info to match stored plan info.\nexpected: %v\nactual: %v", runner.planMetadata, restored.planMetadata) } } @@ -214,61 +221,6 @@ func TestTestPlanner_RestoreTestOptimizationPlanCache_ComputesMissingWeights(t * } } -func TestLoadPlan_MigratesLegacyRunInfoPlanFields(t *testing.T) { - tempDir := t.TempDir() - oldWd, _ := os.Getwd() - defer func() { _ = os.Chdir(oldWd) }() - _ = os.Chdir(tempDir) - - type legacyTestOptimizationPlanCache struct { - RunInfo legacyRunInfo `json:"runInfo"` - } - cache := legacyTestOptimizationPlanCache{ - RunInfo: legacyRunInfo{ - Service: "checkout-api", - Repository: "https://github.com/acme/checkout.git", - Commit: "9f3a1c7d2b4e", - Branch: "feature/split-report", - Platform: "ruby", - Framework: "rspec", - OSTags: map[string]string{ - "os.platform": "linux", - "os.architecture": "amd64", - "os.version": "6.8.0", - }, - RuntimeTags: map[string]string{ - "runtime.name": "ruby", - "runtime.version": "3.3.4", - }, - }, - } - if err := testoptimization.NewCacheManager().StoreTestOptimizationPlanCache(cache); err != nil { - t.Fatalf("StoreTestOptimizationPlanCache() should not return error, got: %v", err) - } - - plan, err := LoadPlan() - if err != nil { - t.Fatalf("LoadPlan() should not return error, got: %v", err) - } - - expected := PlanInfo{ - Platform: "ruby", - Framework: "rspec", - OSTags: map[string]string{ - "os.platform": "linux", - "os.architecture": "amd64", - "os.version": "6.8.0", - }, - RuntimeTags: map[string]string{ - "runtime.name": "ruby", - "runtime.version": "3.3.4", - }, - } - if !reflect.DeepEqual(plan, expected) { - t.Errorf("Expected LoadPlan() to migrate legacy plan fields.\nexpected: %v\nactual: %v", expected, plan) - } -} - func TestTestPlanner_LoadPlan_UsesExistingPlannerState(t *testing.T) { tempDir := t.TempDir() oldWd, _ := os.Getwd() @@ -284,7 +236,7 @@ func TestTestPlanner_LoadPlan_UsesExistingPlannerState(t *testing.T) { } planner := newTestPlannerWithDefaults() - planner.planInfo = PlanInfo{ + planner.planMetadata = PlanMetadata{ Platform: "ruby", Framework: "rspec", } diff --git a/internal/runner/report.go b/internal/runner/report.go index 181cb0d..3061b6d 100644 --- a/internal/runner/report.go +++ b/internal/runner/report.go @@ -20,30 +20,30 @@ type runExecutionReport struct { } type runReport struct { - RunInfo runmetadata.RunInfo - PlanInfo planner.PlanInfo - Execution runExecutionReport - Duration time.Duration - Err error + RunInfo runmetadata.RunInfo + PlanMetadata planner.PlanMetadata + Execution runExecutionReport + Duration time.Duration + Err error } func printRunReport(w io.Writer, report runReport) { reportFprintln(w, "+++ DDTest: run report") reportFprintln(w) - printRunInfoReport(w, report.RunInfo, report.PlanInfo) + printRunInfoReport(w, report.RunInfo, report.PlanMetadata) reportFprintln(w) printExecutionReport(w, report) } -func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planInfo planner.PlanInfo) { +func printRunInfoReport(w io.Writer, runInfo runmetadata.RunInfo, planMetadata planner.PlanMetadata) { reportFprintln(w, "Run") reportFprintf(w, " Service: %s\n", valueOrNotAvailable(runInfo.Service)) reportFprintf(w, " Repository: %s\n", valueOrNotAvailable(runInfo.Repository)) reportFprintf(w, " Commit: %s\n", valueOrNotAvailable(runInfo.Commit)) reportFprintf(w, " Branch: %s\n", valueOrNotAvailable(runInfo.Branch)) - reportFprintf(w, " Platform: %s\n", formatPlatform(planInfo.Platform, planInfo.Framework)) - reportFprintf(w, " OS tags: %s\n", formatTagList(planInfo.OSTags, constants.OSPlatform, constants.OSArchitecture, constants.OSVersion)) - reportFprintf(w, " Runtime tags: %s\n", formatTagList(planInfo.RuntimeTags, constants.RuntimeName, constants.RuntimeVersion)) + reportFprintf(w, " Platform: %s\n", formatPlatform(planMetadata.Platform, planMetadata.Framework)) + reportFprintf(w, " OS tags: %s\n", formatTagList(planMetadata.OSTags, constants.OSPlatform, constants.OSArchitecture, constants.OSVersion)) + reportFprintf(w, " Runtime tags: %s\n", formatTagList(planMetadata.RuntimeTags, constants.RuntimeName, constants.RuntimeVersion)) } func printExecutionReport(w io.Writer, report runReport) { diff --git a/internal/runner/report_test.go b/internal/runner/report_test.go index ba6761f..dff74d0 100644 --- a/internal/runner/report_test.go +++ b/internal/runner/report_test.go @@ -21,7 +21,7 @@ func TestPrintRunReport_Passed(t *testing.T) { Commit: "9f3a1c7d2b4e", Branch: "feature/split-report", }, - PlanInfo: planner.PlanInfo{ + PlanMetadata: planner.PlanMetadata{ Platform: "ruby", Framework: "rspec", OSTags: map[string]string{ diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 60500cf..4b8704a 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -23,7 +23,7 @@ type Runner interface { type Planner interface { Plan(ctx context.Context) error - LoadPlan() (planner.PlanInfo, error) + LoadPlan() (planner.PlanMetadata, error) DistributeTestFiles(testFiles []string, parallelRunners int) [][]string } @@ -67,7 +67,7 @@ func (tr *TestRunner) Run(ctx context.Context) error { return fmt.Errorf("failed to check parallel runners count at %s: %w", constants.ParallelRunnersOutputPath, err) } - planInfo, err := tr.planner.LoadPlan() + planMetadata, err := tr.planner.LoadPlan() if err != nil { slog.Error("Test optimization plan is not available", "error", err) return fmt.Errorf("test optimization plan is not available: %w", err) @@ -96,8 +96,8 @@ func (tr *TestRunner) Run(ctx context.Context) error { } slog.Info("Framework detected", "framework", framework.Name()) runInfo := runmetadata.New(ciUtils.GetCITags()) - if planInfo.IsZero() { - planInfo = planner.NewPlanInfo(nil, detectedPlatform.Name(), framework.Name(), detectedPlatform.TestSkippingLevel()) + if planMetadata.IsZero() { + planMetadata = planner.NewPlanMetadata(nil, detectedPlatform.Name(), framework.Name(), detectedPlatform.TestSkippingLevel()) } ciNode := settings.GetCiNode() @@ -114,11 +114,11 @@ func (tr *TestRunner) Run(ctx context.Context) error { if settings.GetReportEnabled() { printRunReport(tr.reportWriter, runReport{ - RunInfo: runInfo, - PlanInfo: planInfo, - Execution: executionResult.report, - Duration: time.Since(startTime), - Err: executionResult.err, + RunInfo: runInfo, + PlanMetadata: planMetadata, + Execution: executionResult.report, + Duration: time.Since(startTime), + Err: executionResult.err, }) } return executionResult.err diff --git a/internal/runner/runner_test.go b/internal/runner/runner_test.go index a6798d5..ca49f52 100644 --- a/internal/runner/runner_test.go +++ b/internal/runner/runner_test.go @@ -21,7 +21,7 @@ type fakePlanner struct { distributeCalls int planFunc func(context.Context) error distributeFunc func([]string, int) [][]string - plan planner.PlanInfo + plan planner.PlanMetadata loadErr error distributedTestFiles [][]string distributedWorkerNums []int @@ -35,7 +35,7 @@ func (f *fakePlanner) Plan(ctx context.Context) error { return nil } -func (f *fakePlanner) LoadPlan() (planner.PlanInfo, error) { +func (f *fakePlanner) LoadPlan() (planner.PlanMetadata, error) { f.loadCalls++ return f.plan, f.loadErr } @@ -68,7 +68,7 @@ func TestTestRunner_Run_PlansThroughPublicClientWhenArtifactsMissing(t *testing. writeRunnerTestFile(t, constants.TestFilesOutputPath, "spec/a_spec.rb\n") return nil }, - plan: planner.PlanInfo{ + plan: planner.PlanMetadata{ Platform: "ruby", Framework: "rspec", }, @@ -258,7 +258,7 @@ func TestTestRunner_Run_WritesReportWhenEnabled(t *testing.T) { framework := &MockFramework{FrameworkName: "rspec"} platform := &MockPlatform{PlatformName: "ruby", Framework: framework} testPlanner := &fakePlanner{ - plan: planner.PlanInfo{ + plan: planner.PlanMetadata{ Platform: "ruby", Framework: "rspec", }, diff --git a/internal/settings/settings.go b/internal/settings/settings.go index adaf371..b159cf4 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -99,7 +99,7 @@ type Config struct { Framework string `mapstructure:"framework"` MinParallelism int `mapstructure:"min_parallelism"` MaxParallelism int `mapstructure:"max_parallelism"` - ParallelRunnerOverhead time.Duration `mapstructure:"parallel_runner_overhead" report:"CI job overhead"` + ParallelRunnerOverhead time.Duration `mapstructure:"parallel_runner_overhead"` WorkerEnv string `mapstructure:"worker_env"` CiNode int `mapstructure:"ci_node"` CiNodeWorkers int `mapstructure:"ci_node_workers"` @@ -114,28 +114,6 @@ type Config struct { ReportEnabled bool `mapstructure:"report_enabled"` } -func DefaultConfig() Config { - return Config{ - Platform: "ruby", - Framework: "rspec", - MinParallelism: DefaultParallelism(), - MaxParallelism: DefaultParallelism(), - ParallelRunnerOverhead: defaultParallelRunnerOverhead, - WorkerEnv: "", - CiNode: -1, - CiNodeWorkers: defaultCiNodeWorkers, - Command: "", - TestsLocation: "", - TestsExcludePattern: "", - TestDiscoveryCache: "", - TestSkippingLevel: TestSkippingLevelTest, - ForceFullTestDiscovery: false, - StrictDiscovery: false, - RuntimeTags: "", - ReportEnabled: true, - } -} - var ( config *Config ) @@ -197,24 +175,23 @@ func Init() { } func setDefaults() { - defaults := DefaultConfig() - viper.SetDefault("platform", defaults.Platform) - viper.SetDefault("framework", defaults.Framework) - viper.SetDefault("min_parallelism", defaults.MinParallelism) - viper.SetDefault("max_parallelism", defaults.MaxParallelism) - viper.SetDefault("parallel_runner_overhead", defaults.ParallelRunnerOverhead.String()) - viper.SetDefault("worker_env", defaults.WorkerEnv) - viper.SetDefault("ci_node", defaults.CiNode) - viper.SetDefault("ci_node_workers", strconv.Itoa(defaults.CiNodeWorkers)) - viper.SetDefault("command", defaults.Command) - viper.SetDefault("tests_location", defaults.TestsLocation) - viper.SetDefault("tests_exclude_pattern", defaults.TestsExcludePattern) - viper.SetDefault("test_discovery_cache", defaults.TestDiscoveryCache) - viper.SetDefault("test_skipping_mode", defaults.TestSkippingLevel.String()) - viper.SetDefault("force_full_test_discovery", defaults.ForceFullTestDiscovery) - viper.SetDefault("strict_discovery", defaults.StrictDiscovery) - viper.SetDefault("runtime_tags", defaults.RuntimeTags) - viper.SetDefault("report_enabled", defaults.ReportEnabled) + viper.SetDefault("platform", "ruby") + viper.SetDefault("framework", "rspec") + viper.SetDefault("min_parallelism", DefaultParallelism()) + viper.SetDefault("max_parallelism", DefaultParallelism()) + viper.SetDefault("parallel_runner_overhead", defaultParallelRunnerOverhead.String()) + viper.SetDefault("worker_env", "") + viper.SetDefault("ci_node", -1) + viper.SetDefault("ci_node_workers", strconv.Itoa(defaultCiNodeWorkers)) + viper.SetDefault("command", "") + viper.SetDefault("tests_location", "") + viper.SetDefault("tests_exclude_pattern", "") + viper.SetDefault("test_discovery_cache", "") + viper.SetDefault("test_skipping_mode", TestSkippingLevelTest) + viper.SetDefault("force_full_test_discovery", false) + viper.SetDefault("strict_discovery", false) + viper.SetDefault("runtime_tags", "") + viper.SetDefault("report_enabled", true) } // NormalizeTestSkippingLevel accepts only the backend-supported TIA skipping modes. diff --git a/internal/testoptimization/api/durations_api.go b/internal/testoptimization/api/durations_api.go index 8ad493f..eb434d6 100644 --- a/internal/testoptimization/api/durations_api.go +++ b/internal/testoptimization/api/durations_api.go @@ -76,6 +76,10 @@ type ( func (c *transport) GetTestSuiteDurations() *TestSuiteDurationsResponseData { startTime := time.Now() + defer func() { + c.backendRequestTimings.TestSuiteDurations = time.Since(startTime) + }() + if c.repositoryURL == "" { slog.Error("Test durations API errored", "duration", time.Since(startTime), "error", "repository URL is required") return emptyTestSuiteDurationsResponseData() diff --git a/internal/testoptimization/api/durations_api_test.go b/internal/testoptimization/api/durations_api_test.go index 9084c4a..5c3f1a9 100644 --- a/internal/testoptimization/api/durations_api_test.go +++ b/internal/testoptimization/api/durations_api_test.go @@ -103,6 +103,9 @@ func TestClientGetTestSuiteDurationsLogsSuccess(t *testing.T) { if len(result.TestSuites) != 1 { t.Errorf("expected 1 module, got %d", len(result.TestSuites)) } + if duration := client.BackendRequestTimings().TestSuiteDurations; duration <= 0 { + t.Errorf("expected test suite durations fetch duration to be recorded, got %s", duration) + } if !strings.Contains(logs.String(), "level=INFO") || !strings.Contains(logs.String(), "Fetched test suite durations") || !strings.Contains(logs.String(), "modulesCount=1") || diff --git a/internal/testoptimization/api/known_tests_api.go b/internal/testoptimization/api/known_tests_api.go index 820cb03..0bab7d2 100644 --- a/internal/testoptimization/api/known_tests_api.go +++ b/internal/testoptimization/api/known_tests_api.go @@ -8,6 +8,7 @@ package api import ( "encoding/json" "fmt" + "time" ) const ( @@ -62,6 +63,11 @@ type ( ) func (c *transport) GetKnownTests() (*KnownTestsResponseData, error) { + startTime := time.Now() + defer func() { + c.backendRequestTimings.KnownTests = time.Since(startTime) + }() + if c.repositoryURL == "" || c.commitSha == "" { return nil, fmt.Errorf("testoptimization.GetKnownTests: repository URL and commit SHA are required") } diff --git a/internal/testoptimization/api/raw_response_test.go b/internal/testoptimization/api/raw_response_test.go index fbeca55..1945c7b 100644 --- a/internal/testoptimization/api/raw_response_test.go +++ b/internal/testoptimization/api/raw_response_test.go @@ -117,6 +117,20 @@ func TestClientStoresRawBackendResponses(t *testing.T) { if string(client.GetTestManagementTestsRawResponse()) != testManagementResponse { t.Fatalf("test management raw response mismatch:\nexpected: %s\nactual: %s", testManagementResponse, string(client.GetTestManagementTestsRawResponse())) } + + timings := client.BackendRequestTimings() + if timings.Settings <= 0 { + t.Errorf("expected settings duration to be recorded, got %s", timings.Settings) + } + if timings.KnownTests <= 0 { + t.Errorf("expected known tests duration to be recorded, got %s", timings.KnownTests) + } + if timings.Skippables <= 0 { + t.Errorf("expected skippables duration to be recorded, got %s", timings.Skippables) + } + if timings.TestManagementTests <= 0 { + t.Errorf("expected test management duration to be recorded, got %s", timings.TestManagementTests) + } } func TestClientBuildsSkippableKeyFromTestBundle(t *testing.T) { diff --git a/internal/testoptimization/api/settings_api.go b/internal/testoptimization/api/settings_api.go index c478882..717a0a7 100644 --- a/internal/testoptimization/api/settings_api.go +++ b/internal/testoptimization/api/settings_api.go @@ -8,6 +8,7 @@ package api import ( "fmt" "log/slog" + "time" "github.com/DataDog/ddtest/internal/settings" ) @@ -71,6 +72,11 @@ type ( ) func (c *transport) GetSettings() (*SettingsResponseData, error) { + startTime := time.Now() + defer func() { + c.backendRequestTimings.Settings = time.Since(startTime) + }() + if c.repositoryURL == "" || c.commitSha == "" { return nil, fmt.Errorf("testoptimization.GetSettings: repository URL and commit SHA are required") } diff --git a/internal/testoptimization/api/skippable.go b/internal/testoptimization/api/skippable.go index 72804f5..d24118e 100644 --- a/internal/testoptimization/api/skippable.go +++ b/internal/testoptimization/api/skippable.go @@ -8,6 +8,7 @@ package api import ( "fmt" "log/slog" + "time" "github.com/DataDog/ddtest/internal/settings" ) @@ -85,6 +86,18 @@ func (s Skippables) Count() int { } func (c *transport) GetSkippableTests() (correlationID string, skippables Skippables, err error) { + startTime := time.Now() + defer func() { + duration := time.Since(startTime) + c.backendRequestTimings.Skippables = duration + if err == nil { + slog.Debug("Finished fetching skippable tests and suites", + "testsCount", len(skippables.Tests), + "suitesCount", len(skippables.Suites), + "duration", duration) + } + }() + if c.repositoryURL == "" || c.commitSha == "" { err = fmt.Errorf("testoptimization.GetSkippableTests: repository URL and commit SHA are required") return diff --git a/internal/testoptimization/api/skippable_test.go b/internal/testoptimization/api/skippable_test.go index 0f36eec..107dbde 100644 --- a/internal/testoptimization/api/skippable_test.go +++ b/internal/testoptimization/api/skippable_test.go @@ -12,6 +12,7 @@ import ( ) func TestTransportGetSkippableTestsRequestAndResponse(t *testing.T) { + logs := captureRawResponseTestLogs(t) var captured skippableRequest response := skippableResponse{ Meta: skippableResponseMeta{CorrelationID: "correlation-id"}, @@ -89,6 +90,12 @@ func TestTransportGetSkippableTestsRequestAndResponse(t *testing.T) { if len(client.GetSkippableTestsRawResponse()) == 0 { t.Fatal("expected raw skippable response to be stored") } + if !strings.Contains(logs.String(), "Finished fetching skippable tests and suites") || + !strings.Contains(logs.String(), "testsCount=1") || + !strings.Contains(logs.String(), "suitesCount=0") || + !strings.Contains(logs.String(), "duration=") { + t.Fatalf("expected skippable fetch duration log, got logs: %s", logs.String()) + } } func TestTransportGetSkippableTestsRequestUsesConfiguredTestLevel(t *testing.T) { diff --git a/internal/testoptimization/api/test_management_tests_api.go b/internal/testoptimization/api/test_management_tests_api.go index 41d4be7..6162687 100644 --- a/internal/testoptimization/api/test_management_tests_api.go +++ b/internal/testoptimization/api/test_management_tests_api.go @@ -7,6 +7,7 @@ package api import ( "fmt" + "time" ) const ( @@ -65,6 +66,11 @@ type ( ) func (c *transport) GetTestManagementTests() (*TestManagementTestsResponseDataModules, error) { + startTime := time.Now() + defer func() { + c.backendRequestTimings.TestManagementTests = time.Since(startTime) + }() + if c.repositoryURL == "" { return nil, fmt.Errorf("testoptimization.GetTestManagementTests: repository URL is required") } diff --git a/internal/testoptimization/api/transport.go b/internal/testoptimization/api/transport.go index bce1b94..ada8390 100644 --- a/internal/testoptimization/api/transport.go +++ b/internal/testoptimization/api/transport.go @@ -46,6 +46,15 @@ type ( GetSkippableTestsRawResponse() json.RawMessage GetTestManagementTests() (*TestManagementTestsResponseDataModules, error) GetTestManagementTestsRawResponse() json.RawMessage + BackendRequestTimings() BackendRequestTimings + } + + BackendRequestTimings struct { + Settings time.Duration + KnownTests time.Duration + Skippables time.Duration + TestManagementTests time.Duration + TestSuiteDurations time.Duration } // transport sends requests to the Datadog backend. @@ -71,6 +80,7 @@ type ( knownTestsRawResponse json.RawMessage skippableTestsRawResponse json.RawMessage testManagementTestsRawResponse json.RawMessage + backendRequestTimings BackendRequestTimings } // testConfigurations represents the test configurations. @@ -362,6 +372,10 @@ func (c *transport) GetTestManagementTestsRawResponse() json.RawMessage { return cloneRawMessage(c.testManagementTestsRawResponse) } +func (c *transport) BackendRequestTimings() BackendRequestTimings { + return c.backendRequestTimings +} + // getURLPath returns the full URL path for the given URL path. func (c *transport) getURLPath(urlPath string) string { if c.agentless { diff --git a/internal/testoptimization/testoptimization.go b/internal/testoptimization/testoptimization.go index 36e194e..4db9be0 100644 --- a/internal/testoptimization/testoptimization.go +++ b/internal/testoptimization/testoptimization.go @@ -21,24 +21,9 @@ import ( const autoDetectServiceName = "" -const ( - libraryCapabilitiesTestImpactAnalysis = "_dd.library_capabilities.test_impact_analysis" - libraryCapabilitiesEarlyFlakeDetection = "_dd.library_capabilities.early_flake_detection" - libraryCapabilitiesAutoTestRetries = "_dd.library_capabilities.auto_test_retries" - libraryCapabilitiesTestManagementQuarantine = "_dd.library_capabilities.test_management.quarantine" - libraryCapabilitiesTestManagementDisable = "_dd.library_capabilities.test_management.disable" - libraryCapabilitiesTestManagementAttemptToFix = "_dd.library_capabilities.test_management.attempt_to_fix" -) - type testOptimizationCloseAction func() -type OperationDurations struct { - Settings time.Duration - KnownTests time.Duration - Skippables time.Duration - TestManagementTests time.Duration - TestSuiteDurations time.Duration -} +type BackendRequestTimings = api.BackendRequestTimings type searchCommitsResponse struct { LocalCommits []string @@ -59,13 +44,11 @@ type TestOptimizationClient struct { closeActionsMutex sync.Mutex closeActions []testOptimizationCloseAction settings *api.SettingsResponseData - knownTests api.KnownTestsResponseData + knownTests *api.KnownTestsResponseData skippables api.Skippables + testManagementTests *api.TestManagementTestsResponseDataModules + testSuiteDurations *api.TestSuiteDurationsResponseData testSkippingLevel settings.TestSkippingLevel - testManagementTests api.TestManagementTestsResponseDataModules - - operationDurationsMutex sync.Mutex - operationDurations OperationDurations } func NewTestOptimizationClient() *TestOptimizationClient { @@ -136,9 +119,6 @@ func (c *TestOptimizationClient) GetSettings() *api.SettingsResponseData { } func (c *TestOptimizationClient) GetSkippables() api.Skippables { - startTime := time.Now() - - slog.Debug("Fetching skippable tests and suites...") c.ensureTestOptimizationInitialized() if c.skippables.Tests == nil { c.skippables.Tests = api.SkippableTests{} @@ -153,12 +133,6 @@ func (c *TestOptimizationClient) GetSkippables() api.Skippables { } } - duration := time.Since(startTime) - slog.Debug("Finished fetching skippable tests and suites", - "testsCount", len(c.skippables.Tests), - "suitesCount", len(c.skippables.Suites), - "duration", duration) - return c.skippables } @@ -167,7 +141,7 @@ func (c *TestOptimizationClient) GetKnownTests() *api.KnownTestsResponseData { return nil } c.ensureTestOptimizationInitialized() - return &c.knownTests + return c.knownTests } func (c *TestOptimizationClient) GetTestManagementTestsData() *api.TestManagementTestsResponseDataModules { @@ -175,7 +149,7 @@ func (c *TestOptimizationClient) GetTestManagementTestsData() *api.TestManagemen return nil } c.ensureTestOptimizationInitialized() - return &c.testManagementTests + return c.testManagementTests } func (c *TestOptimizationClient) GetDisabledTests() map[string]bool { @@ -183,32 +157,26 @@ func (c *TestOptimizationClient) GetDisabledTests() map[string]bool { } func (c *TestOptimizationClient) GetTestSuiteDurations() *api.TestSuiteDurationsResponseData { - startTime := time.Now() - defer func() { - c.recordOperationDuration(func(durations *OperationDurations) { - durations.TestSuiteDurations = time.Since(startTime) - }) - }() + if c.testSuiteDurations != nil { + return c.testSuiteDurations + } testOptimizationTransport := c.ensureAPITransport(autoDetectServiceName) if testOptimizationTransport == nil { - return &api.TestSuiteDurationsResponseData{ + c.testSuiteDurations = &api.TestSuiteDurationsResponseData{ TestSuites: map[string]map[string]api.TestSuiteDurationInfo{}, } + return c.testSuiteDurations } - return testOptimizationTransport.GetTestSuiteDurations() + c.testSuiteDurations = testOptimizationTransport.GetTestSuiteDurations() + return c.testSuiteDurations } -func (c *TestOptimizationClient) OperationDurations() OperationDurations { - c.operationDurationsMutex.Lock() - defer c.operationDurationsMutex.Unlock() - return c.operationDurations -} - -func (c *TestOptimizationClient) recordOperationDuration(update func(*OperationDurations)) { - c.operationDurationsMutex.Lock() - defer c.operationDurationsMutex.Unlock() - update(&c.operationDurations) +func (c *TestOptimizationClient) BackendRequestTimings() BackendRequestTimings { + if c.apiTransport == nil { + return BackendRequestTimings{} + } + return c.apiTransport.BackendRequestTimings() } func (c *TestOptimizationClient) StoreCacheAndExit() { @@ -263,13 +231,6 @@ func traceDebugEnabled() bool { func (c *TestOptimizationClient) ensureSettingsInitialization(serviceName string) *api.SettingsResponseData { c.settingsOnce.Do(func() { - startTime := time.Now() - defer func() { - c.recordOperationDuration(func(durations *OperationDurations) { - durations.Settings = time.Since(startTime) - }) - }() - slog.Debug("testoptimization: initializing settings") defer slog.Debug("testoptimization: settings initialization complete") @@ -374,27 +335,7 @@ func (c *TestOptimizationClient) ensureTestOptimizationInitialized() { return } - additionalTags := map[string]string{ - libraryCapabilitiesEarlyFlakeDetection: "1", - libraryCapabilitiesAutoTestRetries: "1", - libraryCapabilitiesTestImpactAnalysis: "1", - libraryCapabilitiesTestManagementQuarantine: "1", - libraryCapabilitiesTestManagementDisable: "1", - libraryCapabilitiesTestManagementAttemptToFix: "5", - } - defer func() { - if len(additionalTags) > 0 { - slog.Debug("testoptimization: adding additional tags", "tags", additionalTags) //nolint:gocritic // Map structure logging for debugging - environment.AddCITagsMap(additionalTags) - } - }() - - var additionalTagsMutex sync.Mutex - setAdditionalTags := func(key string, value string) { - additionalTagsMutex.Lock() - defer additionalTagsMutex.Unlock() - additionalTags[key] = value - } + var skippablesCorrelationID string var wg sync.WaitGroup @@ -402,17 +343,11 @@ func (c *TestOptimizationClient) ensureTestOptimizationInitialized() { wg.Add(1) go func() { defer wg.Done() - startTime := time.Now() - defer func() { - c.recordOperationDuration(func(durations *OperationDurations) { - durations.KnownTests = time.Since(startTime) - }) - }() - knownTests, err := c.apiTransport.GetKnownTests() + result, err := c.apiTransport.GetKnownTests() if err != nil { slog.Error("testoptimization: error getting test optimization known tests data", "err", err.Error()) - } else if knownTests != nil { - c.knownTests = *knownTests + } else if result != nil { + c.knownTests = result slog.Debug("testoptimization: known tests data loaded.") } }() @@ -422,21 +357,12 @@ func (c *TestOptimizationClient) ensureTestOptimizationInitialized() { wg.Add(1) go func() { defer wg.Done() - startTime := time.Now() - defer func() { - c.recordOperationDuration(func(durations *OperationDurations) { - durations.Skippables = time.Since(startTime) - }) - }() - correlationID, skippables, err := c.apiTransport.GetSkippableTests() + correlationID, result, err := c.apiTransport.GetSkippableTests() if err != nil { slog.Error("testoptimization: error getting test optimization skippable tests", "err", err.Error()) } else { - slog.Debug("testoptimization: skippable tests loaded", - "testsCount", len(skippables.Tests), - "suitesCount", len(skippables.Suites)) - setAdditionalTags(constants.ItrCorrelationIDTag, correlationID) - c.skippables = skippables + c.skippables = result + skippablesCorrelationID = correlationID } }() } @@ -445,23 +371,23 @@ func (c *TestOptimizationClient) ensureTestOptimizationInitialized() { wg.Add(1) go func() { defer wg.Done() - startTime := time.Now() - defer func() { - c.recordOperationDuration(func(durations *OperationDurations) { - durations.TestManagementTests = time.Since(startTime) - }) - }() - testManagementTests, err := c.apiTransport.GetTestManagementTests() + result, err := c.apiTransport.GetTestManagementTests() if err != nil { slog.Error("testoptimization: error getting test optimization test management tests", "err", err.Error()) - } else if testManagementTests != nil { - c.testManagementTests = *testManagementTests + } else if result != nil { + c.testManagementTests = result slog.Debug("testoptimization: test management loaded", "attemptToFixRetries", currentSettings.TestManagement.AttemptToFixRetries) } }() } wg.Wait() + if skippablesCorrelationID != "" { + slog.Debug("testoptimization: adding skippables correlation ID tag", "correlationID", skippablesCorrelationID) + environment.AddCITagsMap(map[string]string{ + constants.ItrCorrelationIDTag: skippablesCorrelationID, + }) + } }) } diff --git a/internal/testoptimization/testoptimization_lifecycle_test.go b/internal/testoptimization/testoptimization_lifecycle_test.go index 75afb98..f27568d 100644 --- a/internal/testoptimization/testoptimization_lifecycle_test.go +++ b/internal/testoptimization/testoptimization_lifecycle_test.go @@ -91,10 +91,6 @@ func TestTestOptimizationClientFeatureGetters(t *testing.T) { t.Fatalf("expected each feature endpoint once, got known=%d skippable=%d testManagement=%d", mockTransport.KnownTestsCalls, mockTransport.SkippableTestsCalls, mockTransport.TestManagementTestsCalls) } - operationDurations := client.OperationDurations() - if operationDurations.KnownTests <= 0 || operationDurations.Skippables <= 0 || operationDurations.TestManagementTests <= 0 { - t.Fatalf("expected feature endpoint durations, got %+v", operationDurations) - } ciTags := environment.GetCITags() if ciTags[constants.ItrCorrelationIDTag] != "correlation-id" { diff --git a/internal/testoptimization/testoptimization_test.go b/internal/testoptimization/testoptimization_test.go index 98b344e..3360501 100644 --- a/internal/testoptimization/testoptimization_test.go +++ b/internal/testoptimization/testoptimization_test.go @@ -55,6 +55,7 @@ type MockAPIClient struct { SendPackFilesBytes int64 SendPackFilesErr error SendPackFilesCalls int + BackendRequestTimingValues api.BackendRequestTimings } func (m *MockAPIClient) GetSettings() (*api.SettingsResponseData, error) { @@ -127,6 +128,10 @@ func (m *MockAPIClient) SendPackFiles(commitSha string, packFiles []string) (byt return m.SendPackFilesBytes, m.SendPackFilesErr } +func (m *MockAPIClient) BackendRequestTimings() api.BackendRequestTimings { + return m.BackendRequestTimingValues +} + func cleanPlanDirectory(t *testing.T) { t.Helper() if err := os.RemoveAll(constants.PlanDirectory); err != nil { @@ -251,6 +256,7 @@ func TestTestOptimizationClient_GetTestSuiteDurations(t *testing.T) { client := newTestOptimizationClientForTest(t, mockAPIClient) result := client.GetTestSuiteDurations() + cachedResult := client.GetTestSuiteDurations() if mockAPIClient.TestSuiteDurationsCalls != 1 { t.Fatalf("GetTestSuiteDurations() should fetch durations once, got %d calls", mockAPIClient.TestSuiteDurationsCalls) @@ -258,8 +264,67 @@ func TestTestOptimizationClient_GetTestSuiteDurations(t *testing.T) { if result.TestSuites["rspec"]["Suite"].SourceFile != "spec/suite_spec.rb" { t.Fatalf("GetTestSuiteDurations() returned %#v, want %#v", result, durations) } - if client.OperationDurations().TestSuiteDurations <= 0 { - t.Fatal("GetTestSuiteDurations() should record operation duration") + if cachedResult != result { + t.Fatal("GetTestSuiteDurations() should return cached durations on subsequent calls") + } +} + +func TestTestOptimizationClient_BackendRequestTimings(t *testing.T) { + repositorySettings := &api.SettingsResponseData{ + KnownTestsEnabled: true, + TestsSkipping: true, + } + repositorySettings.TestManagement.Enabled = true + + mockAPIClient := &MockAPIClient{ + Settings: repositorySettings, + KnownTests: &api.KnownTestsResponseData{Tests: api.KnownTestsResponseDataModules{"module": {"suite": {"test"}}}}, + Skippables: api.NewSkippables(), + TestManagementTestsData: &api.TestManagementTestsResponseDataModules{Modules: map[string]api.TestManagementTestsResponseDataSuites{}}, + TestSuiteDurations: map[string]map[string]api.TestSuiteDurationInfo{"module": {"suite": {}}}, + BackendRequestTimingValues: api.BackendRequestTimings{ + Settings: time.Millisecond, + KnownTests: 2 * time.Millisecond, + Skippables: 3 * time.Millisecond, + TestManagementTests: 4 * time.Millisecond, + TestSuiteDurations: 5 * time.Millisecond, + }, + } + client := newTestOptimizationClientForTest(t, mockAPIClient) + + if err := client.Initialize(map[string]string{}); err != nil { + t.Fatalf("Initialize() failed: %v", err) + } + client.GetSkippables() + client.GetTestSuiteDurations() + + timings := client.BackendRequestTimings() + for name, duration := range map[string]time.Duration{ + "settings": timings.Settings, + "known tests": timings.KnownTests, + "skippables": timings.Skippables, + "test management": timings.TestManagementTests, + "test suite durations": timings.TestSuiteDurations, + } { + if duration <= 0 { + t.Errorf("expected %s duration to be recorded, got %s", name, duration) + } + } + + if client.GetSettings() != repositorySettings { + t.Errorf("expected settings operation to store fetched settings, got %+v", client.GetSettings()) + } + if knownTests := client.GetKnownTests(); knownTests == nil || len(knownTests.Tests) != 1 { + t.Errorf("expected known tests operation to store fetched data, got %+v", knownTests) + } + if skippables := client.GetSkippables(); skippables.Tests == nil || skippables.Suites == nil { + t.Errorf("expected skippables operation to be recorded, got %+v", skippables) + } + if testManagementTests := client.GetTestManagementTestsData(); testManagementTests == nil { + t.Errorf("expected test management operation to store fetched data, got %+v", testManagementTests) + } + if testSuiteDurations := client.GetTestSuiteDurations(); testSuiteDurations == nil || len(testSuiteDurations.TestSuites) != 1 { + t.Errorf("expected duration operation to store fetched durations, got %+v", testSuiteDurations) } } @@ -301,9 +366,6 @@ func TestTestOptimizationClient_Initialize(t *testing.T) { if duration < 0 { t.Error("Initialize() duration should be non-negative") } - if client.OperationDurations().Settings <= 0 { - t.Error("Initialize() should record settings fetch duration") - } } func TestTestOptimizationClient_GetSkippables_NilResponse(t *testing.T) { @@ -376,9 +438,6 @@ func TestTestOptimizationClient_GetSkippables(t *testing.T) { if len(result) != len(expectedTests) { t.Errorf("Expected %d skippable tests, got %d", len(expectedTests), len(result)) } - if client.OperationDurations().Skippables <= 0 { - t.Error("GetSkippables() should record skippables fetch duration") - } } func TestTestOptimizationClient_StoreCacheAndExit(t *testing.T) { @@ -544,9 +603,6 @@ func TestTestOptimizationClient_GetDisabledTests(t *testing.T) { if mockAPIClient.TestManagementTestsCalls != 1 { t.Fatalf("expected test management tests to be fetched once, got %d", mockAPIClient.TestManagementTestsCalls) } - if client.OperationDurations().TestManagementTests <= 0 { - t.Fatal("GetDisabledTests() should record test management fetch duration") - } } func TestTestOptimizationClient_GetDisabledTestsDisabled(t *testing.T) {