From 63b0f145f1356969c5730a7da858ec57cd272dd6 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 20 Apr 2026 15:47:12 +0200 Subject: [PATCH 01/66] Add a first version of the reporter job --- Arcade.slnx | 1 + .../core-templates/job/helix-reporter-job.yml | 94 ++++ .../Reporter/HelixReporterJobUtilities.cs | 109 +++++ .../Microsoft.DotNet.Helix.Reporter.csproj | 28 ++ .../Reporter/Program.cs | 31 ++ .../Reporter/ReporterOptions.cs | 132 ++++++ .../Reporter/ReporterRunner.cs | 418 ++++++++++++++++++ .../HelixReporterJobUtilitiesTests.cs | 53 +++ .../Microsoft.DotNet.Helix.Sdk.Tests.csproj | 1 + src/Microsoft.DotNet.Helix/Sdk/Readme.md | 43 ++ 10 files changed, 910 insertions(+) create mode 100644 eng/common/core-templates/job/helix-reporter-job.yml create mode 100644 src/Microsoft.DotNet.Helix/Reporter/HelixReporterJobUtilities.cs create mode 100644 src/Microsoft.DotNet.Helix/Reporter/Microsoft.DotNet.Helix.Reporter.csproj create mode 100644 src/Microsoft.DotNet.Helix/Reporter/Program.cs create mode 100644 src/Microsoft.DotNet.Helix/Reporter/ReporterOptions.cs create mode 100644 src/Microsoft.DotNet.Helix/Reporter/ReporterRunner.cs create mode 100644 src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixReporterJobUtilitiesTests.cs diff --git a/Arcade.slnx b/Arcade.slnx index 1e9aa9aae67..7ef1f3c8f9d 100644 --- a/Arcade.slnx +++ b/Arcade.slnx @@ -7,6 +7,7 @@ + diff --git a/eng/common/core-templates/job/helix-reporter-job.yml b/eng/common/core-templates/job/helix-reporter-job.yml new file mode 100644 index 00000000000..efa2ad9ab32 --- /dev/null +++ b/eng/common/core-templates/job/helix-reporter-job.yml @@ -0,0 +1,94 @@ +parameters: + jobName: HelixReporter + displayName: Helix Reporter Job + pool: {} + dotnetVersion: 11.0.x + includePreviewVersions: true + toolPackageId: Microsoft.DotNet.Helix.Reporter + toolCommand: dotnet-helix-reporter + toolVersion: '' + toolSource: '' + helixBaseUri: https://helix.dot.net/ + helixAccessToken: '' + pollingIntervalSeconds: 30 + timeoutInMinutes: 360 + stepTimeoutInMinutes: 360 + reporterJobName: Helix Reporter + +jobs: +- job: ${{ parameters.jobName }} + displayName: ${{ parameters.displayName }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + pool: + ${{ if eq(variables['System.TeamProject'], 'public') }}: + name: $(DncEngPublicBuildPool) + image: build.azurelinux.3.amd64.open + ${{ if eq(variables['System.TeamProject'], 'internal') }}: + name: $(DncEngInternalBuildPool) + image: build.azurelinux.3.amd64.open + steps: + - checkout: none + + - task: UseDotNet@2 + displayName: Install .NET SDK + timeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} + inputs: + packageType: sdk + version: ${{ parameters.dotnetVersion }} + includePreviewVersions: ${{ parameters.includePreviewVersions }} + + - pwsh: | + $toolPath = Join-Path $env:AGENT_TEMPDIRECTORY 'helix-reporter-tool' + New-Item -ItemType Directory -Force -Path $toolPath | Out-Null + + Push-Location $env:AGENT_TEMPDIRECTORY + try { + $packageId = '${{ parameters.toolPackageId }}' + $toolVersion = '${{ parameters.toolVersion }}' + $toolSource = '${{ parameters.toolSource }}' + + $updateArgs = @('tool', 'update', '--tool-path', $toolPath, $packageId) + if (-not [string]::IsNullOrWhiteSpace($toolVersion)) { + $updateArgs += @('--version', $toolVersion) + } + if (-not [string]::IsNullOrWhiteSpace($toolSource)) { + $updateArgs += @('--add-source', $toolSource) + } + + & dotnet @updateArgs + if ($LASTEXITCODE -ne 0) { + $installArgs = @('tool', 'install', '--tool-path', $toolPath, $packageId) + if (-not [string]::IsNullOrWhiteSpace($toolVersion)) { + $installArgs += @('--version', $toolVersion) + } + if (-not [string]::IsNullOrWhiteSpace($toolSource)) { + $installArgs += @('--add-source', $toolSource) + } + + & dotnet @installArgs + } + + if ($LASTEXITCODE -ne 0) { + throw "Failed to install the Helix reporter tool package '$packageId'." + } + + Write-Host "##vso[task.prependpath]$toolPath" + Write-Host "##vso[task.setvariable variable=HelixReporterToolPath]$toolPath" + } + finally { + Pop-Location + } + displayName: Install Helix reporter + timeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} + + - pwsh: | + & '${{ parameters.toolCommand }}' --helix-base-uri '${{ parameters.helixBaseUri }}' --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' --max-wait-minutes '${{ parameters.timeoutInMinutes }}' --reporter-job-name '${{ parameters.reporterJobName }}' + + if ($LASTEXITCODE -ne 0) { + throw "The Helix reporter tool exited with code $LASTEXITCODE." + } + displayName: Run Helix reporter + timeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }} diff --git a/src/Microsoft.DotNet.Helix/Reporter/HelixReporterJobUtilities.cs b/src/Microsoft.DotNet.Helix/Reporter/HelixReporterJobUtilities.cs new file mode 100644 index 00000000000..f95f3cb2060 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Reporter/HelixReporterJobUtilities.cs @@ -0,0 +1,109 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.Helix.Reporter +{ + public sealed class AzureDevOpsTimelineRecord + { + [JsonProperty("id")] + public string Id { get; set; } + + [JsonProperty("parentId")] + public string ParentId { get; set; } + + [JsonProperty("type")] + public string Type { get; set; } + + [JsonProperty("name")] + public string Name { get; set; } + + [JsonProperty("state")] + public string State { get; set; } + + [JsonProperty("result")] + public string Result { get; set; } + } + + public static class HelixReporterJobUtilities + { + public static string NormalizeRepository(string repository) + { + if (string.IsNullOrWhiteSpace(repository)) + { + return string.Empty; + } + + repository = repository.Trim().TrimEnd('/'); + if (!Uri.TryCreate(repository, UriKind.Absolute, out Uri uri)) + { + return repository.Trim('/'); + } + + string[] segments = uri.AbsolutePath.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + if (uri.Host.Contains("github.com", StringComparison.OrdinalIgnoreCase) && segments.Length >= 2) + { + return $"{segments[0]}/{segments[1]}"; + } + + int gitIndex = Array.FindIndex(segments, s => string.Equals(s, "_git", StringComparison.OrdinalIgnoreCase)); + if (gitIndex > 0 && segments.Length > gitIndex + 1) + { + string project = segments[gitIndex - 1]; + string repoName = segments[gitIndex + 1]; + if ((string.Equals(project, "internal", StringComparison.OrdinalIgnoreCase) + || string.Equals(project, "public", StringComparison.OrdinalIgnoreCase)) + && repoName.Contains('-', StringComparison.Ordinal)) + { + int separatorIndex = repoName.IndexOf('-', StringComparison.Ordinal); + return $"{repoName.Substring(0, separatorIndex)}/{repoName.Substring(separatorIndex + 1)}"; + } + + return $"{project}/{repoName}"; + } + + return repository; + } + + public static bool AreNonReporterJobsComplete(IEnumerable records, string reporterJobName) + => GetRelevantJobRecords(records, reporterJobName).All(IsTerminal); + + public static bool HasFailedNonReporterJobs(IEnumerable records, string reporterJobName) + => GetRelevantJobRecords(records, reporterJobName).Any(r => + string.Equals(r.Result, "failed", StringComparison.OrdinalIgnoreCase) + || string.Equals(r.Result, "canceled", StringComparison.OrdinalIgnoreCase)); + + public static string GetTestRunName(string helixJobName) + => $"Helix Reporter - {helixJobName}"; + + public static string CleanWorkItemName(string name) + { + if (string.IsNullOrEmpty(name)) + { + return string.Empty; + } + + if (!name.Contains('%')) + { + name = WebUtility.UrlDecode(name); + } + + return name.Replace('/', '-').Replace('\\', '-'); + } + + private static IEnumerable GetRelevantJobRecords(IEnumerable records, string reporterJobName) + { + return (records ?? Enumerable.Empty()) + .Where(r => string.Equals(r.Type, "Job", StringComparison.OrdinalIgnoreCase)) + .Where(r => !string.Equals(r.Name, reporterJobName, StringComparison.OrdinalIgnoreCase)); + } + + private static bool IsTerminal(AzureDevOpsTimelineRecord record) + => string.Equals(record?.State, "completed", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/Microsoft.DotNet.Helix/Reporter/Microsoft.DotNet.Helix.Reporter.csproj b/src/Microsoft.DotNet.Helix/Reporter/Microsoft.DotNet.Helix.Reporter.csproj new file mode 100644 index 00000000000..99cf8e09b83 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Reporter/Microsoft.DotNet.Helix.Reporter.csproj @@ -0,0 +1,28 @@ + + + + $(BundledNETCoreAppTargetFramework) + Exe + true + true + dotnet-helix-reporter + Standalone Helix reporter job tool for Azure DevOps pipelines + + + + + + + + + + + + + + + + + diff --git a/src/Microsoft.DotNet.Helix/Reporter/Program.cs b/src/Microsoft.DotNet.Helix/Reporter/Program.cs new file mode 100644 index 00000000000..e4a2bdf9fae --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Reporter/Program.cs @@ -0,0 +1,31 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Helix.Reporter +{ + internal static class Program + { + public static async Task Main(string[] args) + { + try + { + ReporterOptions options = ReporterOptions.Parse(args); + if (options.ShowHelp) + { + return 0; + } + + ReporterRunner runner = new ReporterRunner(options); + return await runner.RunAsync().ConfigureAwait(false); + } + catch (Exception ex) + { + Console.Error.WriteLine(ex.ToString()); + return 1; + } + } + } +} diff --git a/src/Microsoft.DotNet.Helix/Reporter/ReporterOptions.cs b/src/Microsoft.DotNet.Helix/Reporter/ReporterOptions.cs new file mode 100644 index 00000000000..a9f85efed16 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Reporter/ReporterOptions.cs @@ -0,0 +1,132 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Globalization; +using CommandLine; + +namespace Microsoft.DotNet.Helix.Reporter +{ + public sealed class ReporterOptions + { + public bool ShowHelp { get; private set; } + + [Option("helix-base-uri", HelpText = "Base URI for the Helix service.")] + public string HelixBaseUri { get; set; } = "https://helix.dot.net/"; + + [Option("helix-access-token", HelpText = "Access token for authenticated Helix APIs.")] + public string HelixAccessToken { get; set; } + + [Option("collection-uri", HelpText = "Azure DevOps collection URI.")] + public string CollectionUri { get; set; } + + [Option("team-project", HelpText = "Azure DevOps team project name.")] + public string TeamProject { get; set; } + + [Option("build-id", HelpText = "Azure DevOps build ID.")] + public string BuildId { get; set; } + + [Option("access-token", HelpText = "Azure DevOps system access token.")] + public string AccessToken { get; set; } + + [Option("repository", HelpText = "Repository identifier in owner/repo form.")] + public string Repository { get; set; } + + [Option("polling-interval-seconds", HelpText = "Polling interval in seconds.", Default = 30)] + public int PollingIntervalSeconds { get; set; } = 30; + + [Option("max-wait-minutes", HelpText = "Maximum run time in minutes.", Default = 360)] + public int MaximumWaitMinutes { get; set; } = 360; + + [Option("reporter-job-name", HelpText = "Display name of the reporter job in Azure DevOps.")] + public string ReporterJobName { get; set; } = "Helix Reporter"; + + [Option("working-directory", HelpText = "Directory used to stage downloaded test results.")] + public string WorkingDirectory { get; set; } + + [Option("pr-number", HelpText = "Pull request number for the build, if applicable.")] + public int? PrNumber { get; set; } + + [Option("attempt", HelpText = "Azure DevOps attempt number for the current job.")] + public int? Attempt { get; set; } + + public static ReporterOptions Parse(string[] args) + { + ReporterOptions parsed = null; + var parser = new Parser(settings => + { + settings.CaseInsensitiveEnumValues = true; + settings.HelpWriter = Console.Out; + }); + + parser.ParseArguments(args) + .WithParsed(options => parsed = options) + .WithNotParsed(errors => + { + parsed = new ReporterOptions { ShowHelp = true }; + }); + + if (parsed == null || parsed.ShowHelp) + { + return parsed ?? new ReporterOptions { ShowHelp = true }; + } + + parsed.ApplyEnvironmentDefaults(); + parsed.Validate(); + return parsed; + } + + private void ApplyEnvironmentDefaults() + { + HelixAccessToken ??= Environment.GetEnvironmentVariable("HELIX_ACCESSTOKEN"); + CollectionUri ??= Environment.GetEnvironmentVariable("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI"); + TeamProject ??= Environment.GetEnvironmentVariable("SYSTEM_TEAMPROJECT"); + BuildId ??= Environment.GetEnvironmentVariable("BUILD_BUILDID"); + AccessToken ??= Environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN"); + Repository = HelixReporterJobUtilities.NormalizeRepository( + Repository + ?? Environment.GetEnvironmentVariable("BUILD_REPOSITORY_URI") + ?? Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")); + WorkingDirectory ??= System.IO.Path.Combine(System.IO.Path.GetTempPath(), "helix-reporter", BuildId ?? "unknown"); + PrNumber ??= GetEnvironmentInt("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"); + Attempt ??= GetEnvironmentInt("SYSTEM_JOBATTEMPT"); + } + + private void Validate() + { + CollectionUri = EnsureTrailingSlash(RequireValue(CollectionUri, "collection-uri", "SYSTEM_TEAMFOUNDATIONCOLLECTIONURI")); + TeamProject = RequireValue(TeamProject, "team-project", "SYSTEM_TEAMPROJECT"); + BuildId = RequireValue(BuildId, "build-id", "BUILD_BUILDID"); + AccessToken = RequireValue(AccessToken, "access-token", "SYSTEM_ACCESSTOKEN"); + + if (string.IsNullOrWhiteSpace(Repository)) + { + throw new InvalidOperationException("A repository identifier must be provided either by argument or pipeline environment."); + } + } + + private static string RequireValue(string value, string argumentName, string environmentName) + { + if (string.IsNullOrWhiteSpace(value)) + { + throw new InvalidOperationException($"Missing required option --{argumentName} or environment variable {environmentName}."); + } + + return value; + } + + private static int? GetEnvironmentInt(string environmentName) + { + string value = Environment.GetEnvironmentVariable(environmentName); + if (int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int parsed)) + { + return parsed; + } + + return null; + } + + private static string EnsureTrailingSlash(string uri) + => uri.EndsWith("/", StringComparison.Ordinal) ? uri : uri + "/"; + } +} diff --git a/src/Microsoft.DotNet.Helix/Reporter/ReporterRunner.cs b/src/Microsoft.DotNet.Helix/Reporter/ReporterRunner.cs new file mode 100644 index 00000000000..69c8738e082 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Reporter/ReporterRunner.cs @@ -0,0 +1,418 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using Microsoft.DotNet.Helix.Client; +using Microsoft.DotNet.Helix.Client.Models; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Microsoft.DotNet.Helix.Reporter +{ + internal sealed class ReporterRunner : IDisposable + { + private readonly ReporterOptions _options; + private readonly HttpClient _azdoClient; + private readonly IHelixApi _helixApi; + + public ReporterRunner(ReporterOptions options) + { + _options = options ?? throw new ArgumentNullException(nameof(options)); + Directory.CreateDirectory(_options.WorkingDirectory); + + _helixApi = string.IsNullOrEmpty(_options.HelixAccessToken) + ? ApiFactory.GetAnonymous(_options.HelixBaseUri) + : ApiFactory.GetAuthenticated(_options.HelixBaseUri, _options.HelixAccessToken); + + _azdoClient = new HttpClient(); + string encodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + _options.AccessToken)); + _azdoClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", encodedToken); + _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-reporter"); + } + + public async Task RunAsync() + { + HashSet processedRuns = await GetProcessedRunNamesAsync().ConfigureAwait(false); + DateTimeOffset deadline = DateTimeOffset.UtcNow.AddMinutes(Math.Max(1, _options.MaximumWaitMinutes)); + bool anyNonReporterJobFailures = false; + int failedHelixJobCount = 0; + int processedHelixJobCount = 0; + + while (DateTimeOffset.UtcNow < deadline) + { + AzureDevOpsTimelineRecord[] timelineRecords = await GetTimelineRecordsAsync().ConfigureAwait(false); + JobsByBuildResponse helixResponse = await RetryAsync(() => _helixApi.Job.ByBuildAsync(_options.Repository, _options.PrNumber, int.Parse(_options.BuildId, CultureInfo.InvariantCulture), _options.Attempt)).ConfigureAwait(false); + IEnumerable jobs = helixResponse?.Jobs ?? Enumerable.Empty(); + + int totalHelixJobs = jobs.Count(); + int completedHelixJobs = jobs.Count(j => j.IsTerminal); + int currentFailedJobs = jobs.Count(j => j.WorkItemsFailed > 0 || string.Equals(j.Status, "failed", StringComparison.OrdinalIgnoreCase)); + Console.WriteLine($"[{DateTime.UtcNow:HH:mm:ss}] {completedHelixJobs}/{totalHelixJobs} Helix jobs complete ({currentFailedJobs} failed). Waiting..."); + + foreach (JobBuildSummary job in jobs.Where(j => j.IsTerminal).OrderBy(j => j.JobName, StringComparer.OrdinalIgnoreCase)) + { + string testRunName = HelixReporterJobUtilities.GetTestRunName(job.JobName); + if (processedRuns.Contains(testRunName)) + { + continue; + } + + JobPassFail passFail = await RetryAsync(() => _helixApi.Job.PassFailAsync(job.JobName)).ConfigureAwait(false); + bool passed = await ProcessCompletedJobAsync(job, passFail, testRunName).ConfigureAwait(false); + processedRuns.Add(testRunName); + processedHelixJobCount++; + if (!passed) + { + failedHelixJobCount++; + } + } + + anyNonReporterJobFailures = HelixReporterJobUtilities.HasFailedNonReporterJobs(timelineRecords, _options.ReporterJobName); + bool allPipelineJobsComplete = HelixReporterJobUtilities.AreNonReporterJobsComplete(timelineRecords, _options.ReporterJobName); + bool allHelixJobsComplete = !jobs.Any() || helixResponse.AllJobsComplete || jobs.All(j => j.IsTerminal); + + if (allPipelineJobsComplete && allHelixJobsComplete) + { + Console.WriteLine($"Final summary: processed {processedHelixJobCount} Helix job(s); {failedHelixJobCount} failed."); + if (anyNonReporterJobFailures || failedHelixJobCount > 0) + { + if (anyNonReporterJobFailures) + { + Console.Error.WriteLine("One or more non-reporter pipeline jobs failed."); + } + + if (failedHelixJobCount > 0) + { + Console.Error.WriteLine($"The reporter detected failures in {failedHelixJobCount} Helix job(s)."); + } + + return 1; + } + + return 0; + } + + await Task.Delay(TimeSpan.FromSeconds(Math.Max(5, _options.PollingIntervalSeconds))).ConfigureAwait(false); + } + + Console.Error.WriteLine($"The reporter timed out after {_options.MaximumWaitMinutes} minute(s)."); + return 1; + } + + public void Dispose() + { + _azdoClient.Dispose(); + } + + private async Task ProcessCompletedJobAsync(JobBuildSummary helixJob, JobPassFail passFail, string testRunName) + { + int testRunId = await StartTestRunAsync(testRunName).ConfigureAwait(false); + string resultsDirectory = Path.Combine(_options.WorkingDirectory, MakeSafeDirectoryName(helixJob.JobName)); + Directory.CreateDirectory(resultsDirectory); + + int downloadedFiles = await DownloadTestResultsAsync(helixJob.JobName, passFail, resultsDirectory).ConfigureAwait(false); + bool reporterRan = downloadedFiles > 0 && await TryRunPythonReporterAsync(resultsDirectory, testRunId).ConfigureAwait(false); + if (!reporterRan) + { + await CreateFallbackResultsAsync(testRunId, helixJob.JobName, passFail).ConfigureAwait(false); + } + + await StopTestRunAsync(testRunId, testRunName).ConfigureAwait(false); + + int passedCount = passFail.Passed?.Count ?? 0; + int failedCount = passFail.Failed?.Count ?? 0; + Console.WriteLine($"[{DateTime.UtcNow:HH:mm:ss}] Job '{helixJob.JobName}' completed ({passedCount} passed, {failedCount} failed)."); + return failedCount == 0 && !string.Equals(helixJob.Status, "failed", StringComparison.OrdinalIgnoreCase); + } + + private async Task> GetProcessedRunNamesAsync() + { + JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?buildIds={_options.BuildId}&api-version=7.1-preview.1").ConfigureAwait(false); + var processed = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (JObject run in data?["value"] as JArray ?? new JArray()) + { + string name = run.Value("name"); + string state = run.Value("state"); + if (!string.IsNullOrEmpty(name) + && name.StartsWith("Helix Reporter - ", StringComparison.OrdinalIgnoreCase) + && string.Equals(state, "Completed", StringComparison.OrdinalIgnoreCase)) + { + processed.Add(name); + } + } + + return processed; + } + + private async Task GetTimelineRecordsAsync() + { + JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/build/builds/{_options.BuildId}/timeline?api-version=7.1-preview.2").ConfigureAwait(false); + return data?["records"]?.ToObject() ?? Array.Empty(); + } + + private async Task StartTestRunAsync(string testRunName) + { + JObject result = await SendAsync( + HttpMethod.Post, + $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?api-version=5.0", + new JObject + { + ["automated"] = true, + ["build"] = new JObject { ["id"] = _options.BuildId }, + ["name"] = testRunName, + ["state"] = "InProgress", + }).ConfigureAwait(false); + + return result?["id"]?.ToObject() ?? 0; + } + + private async Task StopTestRunAsync(int testRunId, string testRunName) + { + await SendAsync( + new HttpMethod("PATCH"), + $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=5.0", + new JObject { ["state"] = "Completed" }).ConfigureAwait(false); + + Console.WriteLine($"Stopped test run '{testRunName}'."); + } + + private async Task CreateFallbackResultsAsync(int testRunId, string jobName, JobPassFail passFail) + { + foreach (string workItemName in passFail.Passed ?? ImmutableList.Empty) + { + await CreateFallbackResultAsync(testRunId, jobName, workItemName, failed: false).ConfigureAwait(false); + } + + foreach (string workItemName in passFail.Failed ?? ImmutableList.Empty) + { + await CreateFallbackResultAsync(testRunId, jobName, workItemName, failed: true).ConfigureAwait(false); + } + } + + private async Task CreateFallbackResultAsync(int testRunId, string jobName, string workItemName, bool failed) + { + string cleanName = HelixReporterJobUtilities.CleanWorkItemName(workItemName); + await SendAsync( + HttpMethod.Post, + $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/Runs/{testRunId}/results?api-version=5.1-preview.6", + new JArray + { + new JObject + { + ["automatedTestName"] = $"{cleanName}.WorkItemExecution", + ["automatedTestStorage"] = cleanName, + ["testCaseTitle"] = $"{cleanName} Work Item", + ["outcome"] = failed ? "Failed" : "Passed", + ["state"] = "Completed", + ["errorMessage"] = failed ? "The Helix work item failed. See the Helix logs for more details." : null, + ["durationInMs"] = 60 * 1000, + ["comment"] = new JObject + { + ["HelixJobId"] = jobName, + ["HelixWorkItemName"] = cleanName, + }.ToString(), + } + }).ConfigureAwait(false); + } + + private async Task DownloadTestResultsAsync(string jobName, JobPassFail passFail, string outputDirectory) + { + int count = 0; + JobResultsUri resultsUri = await RetryAsync(() => _helixApi.Job.ResultsAsync(jobName)).ConfigureAwait(false); + IEnumerable workItemNames = (passFail.Passed ?? ImmutableList.Empty).Concat(passFail.Failed ?? ImmutableList.Empty); + + foreach (string workItemName in workItemNames.Distinct(StringComparer.OrdinalIgnoreCase)) + { + var availableFiles = await RetryAsync(() => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false)).ConfigureAwait(false); + string workItemDirectory = Path.Combine(outputDirectory, MakeSafeDirectoryName(workItemName)); + Directory.CreateDirectory(workItemDirectory); + + foreach (var file in availableFiles.Where(f => LooksLikeTestResultFile(f.Name))) + { + string relativePath = file.Name.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar); + string destinationFile = Path.Combine(workItemDirectory, relativePath); + string directory = Path.GetDirectoryName(destinationFile); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + try + { + BlobClient blobClient = CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS); + await blobClient.DownloadToAsync(destinationFile).ConfigureAwait(false); + count++; + } + catch (Exception ex) + { + Console.WriteLine($"Warning: failed to download '{file.Name}' for '{jobName}/{workItemName}': {ex.Message}"); + } + } + } + + return count; + } + + private async Task TryRunPythonReporterAsync(string workingDirectory, int testRunId) + { + string scriptPath = Path.Combine(AppContext.BaseDirectory, "reporter", "run.py"); + if (!File.Exists(scriptPath)) + { + Console.WriteLine($"Warning: reporter script was not found at '{scriptPath}'. Falling back to synthetic work-item results."); + return false; + } + + foreach ((string fileName, string prefixArguments) in GetPythonCandidates()) + { + try + { + var psi = new ProcessStartInfo + { + FileName = fileName, + Arguments = $"{prefixArguments}\"{scriptPath}\" \"{_options.CollectionUri}\" \"{_options.TeamProject}\" \"{testRunId.ToString(CultureInfo.InvariantCulture)}\" \"{_options.AccessToken}\"", + WorkingDirectory = workingDirectory, + UseShellExecute = false, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + + using Process process = Process.Start(psi); + if (process == null) + { + continue; + } + + string stdout = await process.StandardOutput.ReadToEndAsync().ConfigureAwait(false); + string stderr = await process.StandardError.ReadToEndAsync().ConfigureAwait(false); + await process.WaitForExitAsync().ConfigureAwait(false); + + if (!string.IsNullOrWhiteSpace(stdout)) + { + Console.WriteLine(stdout); + } + + if (!string.IsNullOrWhiteSpace(stderr)) + { + Console.WriteLine(stderr); + } + + if (process.ExitCode == 0) + { + return true; + } + } + catch (Exception ex) + { + Console.WriteLine($"Warning: failed to invoke Python reporter via '{fileName}': {ex.Message}"); + } + } + + return false; + } + + private async Task SendAsync(HttpMethod method, string requestUri, JToken body = null) + { + return await RetryAsync(async () => + { + using var request = new HttpRequestMessage(method, requestUri); + if (body != null) + { + request.Content = new StringContent(body.ToString(Formatting.None), Encoding.UTF8, "application/json"); + } + + using HttpResponseMessage response = await _azdoClient.SendAsync(request).ConfigureAwait(false); + string content = response.Content != null ? await response.Content.ReadAsStringAsync().ConfigureAwait(false) : null; + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException($"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}"); + } + + if (string.IsNullOrWhiteSpace(content)) + { + return new JObject(); + } + + return JObject.Parse(content); + }).ConfigureAwait(false); + } + + private static BlobClient CreateBlobClient(string fileLink, string resultsSas) + { + var options = new BlobClientOptions(); + options.Retry.NetworkTimeout = TimeSpan.FromMinutes(5); + + if (string.IsNullOrEmpty(resultsSas)) + { + return new BlobClient(new Uri(fileLink), options); + } + + string strippedUri = fileLink.Contains('?') ? fileLink.Substring(0, fileLink.LastIndexOf('?', StringComparison.Ordinal)) : fileLink; + return new BlobClient(new Uri(strippedUri), new AzureSasCredential(resultsSas), options); + } + + private static bool LooksLikeTestResultFile(string path) + { + string fileName = Path.GetFileName(path); + return fileName.EndsWith(".trx", StringComparison.OrdinalIgnoreCase) + || (fileName.EndsWith(".xml", StringComparison.OrdinalIgnoreCase) + && (fileName.Contains("testresults", StringComparison.OrdinalIgnoreCase) + || fileName.Contains("test-results", StringComparison.OrdinalIgnoreCase) + || fileName.Contains("junit", StringComparison.OrdinalIgnoreCase) + || fileName.Contains("xunit", StringComparison.OrdinalIgnoreCase))); + } + + private static string MakeSafeDirectoryName(string value) + { + foreach (char invalidChar in Path.GetInvalidFileNameChars()) + { + value = value.Replace(invalidChar, '-'); + } + + return value; + } + + private static IEnumerable<(string fileName, string prefixArguments)> GetPythonCandidates() + { + if (OperatingSystem.IsWindows()) + { + yield return ("py", "-3 "); + } + + yield return ("python3", string.Empty); + yield return ("python", string.Empty); + } + + private static async Task RetryAsync(Func> action) + { + Exception last = null; + for (int attempt = 0; attempt < 5; attempt++) + { + try + { + return await action().ConfigureAwait(false); + } + catch (Exception ex) when (attempt < 4) + { + last = ex; + await Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, attempt + 1))).ConfigureAwait(false); + } + } + + throw last ?? new InvalidOperationException("Retry failed without capturing an exception."); + } + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixReporterJobUtilitiesTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixReporterJobUtilitiesTests.cs new file mode 100644 index 00000000000..388d18f0585 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixReporterJobUtilitiesTests.cs @@ -0,0 +1,53 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.Helix.Reporter; +using System; +using Xunit; + +namespace Microsoft.DotNet.Helix.Sdk.Tests +{ + public class HelixReporterJobUtilitiesTests + { + [Theory] + [InlineData("https://github.com/dotnet/arcade", "dotnet/arcade")] + [InlineData("https://dev.azure.com/dnceng/internal/_git/dotnet-arcade", "dotnet/arcade")] + [InlineData("dotnet/arcade", "dotnet/arcade")] + public void NormalizeRepository_ReturnsStableIdentifier(string input, string expected) + { + Assert.Equal(expected, HelixReporterJobUtilities.NormalizeRepository(input)); + } + + [Fact] + public void AreNonReporterJobsComplete_IgnoresReporterRecord() + { + var records = new[] + { + new AzureDevOpsTimelineRecord { Type = "Job", Name = "Build Linux", State = "completed", Result = "succeeded" }, + new AzureDevOpsTimelineRecord { Type = "Job", Name = "Helix Reporter", State = "inProgress", Result = null }, + }; + + Assert.True(HelixReporterJobUtilities.AreNonReporterJobsComplete(records, "Helix Reporter")); + } + + [Fact] + public void HasFailedNonReporterJobs_DetectsFailures() + { + var records = new[] + { + new AzureDevOpsTimelineRecord { Type = "Job", Name = "Build Linux", State = "completed", Result = "failed" }, + new AzureDevOpsTimelineRecord { Type = "Job", Name = "Helix Reporter", State = "inProgress", Result = null }, + }; + + Assert.True(HelixReporterJobUtilities.HasFailedNonReporterJobs(records, "Helix Reporter")); + } + + [Fact] + public void GetTestRunName_ProducesStableName() + { + Assert.Equal( + "Helix Reporter - coreclr-tests-linux-x64", + HelixReporterJobUtilities.GetTestRunName("coreclr-tests-linux-x64")); + } + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj index 7fd8f7231cf..97bb7663e0b 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj @@ -22,6 +22,7 @@ + diff --git a/src/Microsoft.DotNet.Helix/Sdk/Readme.md b/src/Microsoft.DotNet.Helix/Sdk/Readme.md index 6472e55cc28..f497ff20992 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/Readme.md +++ b/src/Microsoft.DotNet.Helix/Sdk/Readme.md @@ -44,6 +44,49 @@ env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure DevOps ``` +### Helix Reporter Job for Azure DevOps + +If you want to decouple Helix test execution from the build agents that submit the work, use the Helix Reporter Job. + +The reporter job is a lightweight dedicated pipeline job that: + +- polls Azure DevOps for pipeline state, +- polls Helix for jobs associated with the current build, +- downloads test result artifacts from completed Helix jobs, +- publishes results to Azure DevOps incrementally, +- returns a final green or red status once all non-reporter jobs and Helix jobs have completed. + +This allows the original build jobs to stop waiting on Helix execution while still preserving test visibility and pass/fail behavior in the pipeline. + +The job is added with the template at [/eng/common/core-templates/job/helix-reporter-job.yml](/eng/common/core-templates/job/helix-reporter-job.yml). + +Example: + +```yaml +jobs: +- template: /eng/common/core-templates/job/helix-reporter-job.yml@self + parameters: + jobName: HelixReporter + displayName: Helix Reporter Job + pollingIntervalSeconds: 30 + timeoutInMinutes: 360 +``` + +Useful parameters: + +- `helixBaseUri`: base URI for the Helix service. Defaults to `https://helix.dot.net/`. +- `helixAccessToken`: optional token for authenticated Helix access on internal builds. +- `pollingIntervalSeconds`: how often the reporter checks for new completed jobs. +- `timeoutInMinutes`: overall timeout for the reporter job. +- `reporterJobName`: name used to identify and exclude the reporter job in the Azure DevOps timeline. + +Behavior notes: + +- The reporter uses its own `SYSTEM_ACCESSTOKEN`, so it does not depend on the shorter-lived token from the job that originally submitted the Helix work. +- If parseable xUnit, JUnit, or TRX result files are available, those are uploaded. +- If no result files are found, the reporter creates synthetic work-item pass/fail results so that failures are still visible in Azure DevOps. +- The reporter is safe to rerun because it checks for already-completed test runs and only processes new results. + Furthermore, when you need to make changes to Helix SDK, there's a way to run it locally with ease to test your changes in a tighter dev loop than having to have to wait for the full PR build. The repository contains E2E tests that utilize the Helix SDK to send test Helix jobs. From c8ebc263819fc02c7b418d0f2cc7848954abe1f2 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 20 Apr 2026 17:23:19 +0200 Subject: [PATCH 02/66] Rename Helix Test Reporter to Helix Job Monitor --- Arcade.slnx | 2 +- ...reporter-job.yml => helix-job-monitor.yml} | 24 +++---- .../Client/CSharp/Job.cs | 1 + .../Client/CSharp/JobStatus.cs | 13 ++++ .../CSharp/generated-code/Models/JobStatus.cs | 16 +++++ .../HelixJobMonitorUtilities.cs} | 22 +++--- .../JobMonitorOptions.cs} | 22 +++--- .../JobMonitorRunner.cs} | 71 +++++++++---------- .../Microsoft.DotNet.Helix.JobMonitor.csproj} | 4 +- .../{Reporter => JobMonitor}/Program.cs | 6 +- ...ts.cs => HelixJobMonitorUtilitiesTests.cs} | 22 +++--- .../Microsoft.DotNet.Helix.Sdk.Tests.csproj | 2 +- src/Microsoft.DotNet.Helix/Sdk/Readme.md | 22 +++--- 13 files changed, 127 insertions(+), 100 deletions(-) rename eng/common/core-templates/job/{helix-reporter-job.yml => helix-job-monitor.yml} (80%) create mode 100644 src/Microsoft.DotNet.Helix/Client/CSharp/JobStatus.cs create mode 100644 src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs rename src/Microsoft.DotNet.Helix/{Reporter/HelixReporterJobUtilities.cs => JobMonitor/HelixJobMonitorUtilities.cs} (79%) rename src/Microsoft.DotNet.Helix/{Reporter/ReporterOptions.cs => JobMonitor/JobMonitorOptions.cs} (87%) rename src/Microsoft.DotNet.Helix/{Reporter/ReporterRunner.cs => JobMonitor/JobMonitorRunner.cs} (83%) rename src/Microsoft.DotNet.Helix/{Reporter/Microsoft.DotNet.Helix.Reporter.csproj => JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj} (83%) rename src/Microsoft.DotNet.Helix/{Reporter => JobMonitor}/Program.cs (77%) rename src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/{HelixReporterJobUtilitiesTests.cs => HelixJobMonitorUtilitiesTests.cs} (61%) diff --git a/Arcade.slnx b/Arcade.slnx index 7ef1f3c8f9d..0ed52fd8f54 100644 --- a/Arcade.slnx +++ b/Arcade.slnx @@ -7,7 +7,7 @@ - + diff --git a/eng/common/core-templates/job/helix-reporter-job.yml b/eng/common/core-templates/job/helix-job-monitor.yml similarity index 80% rename from eng/common/core-templates/job/helix-reporter-job.yml rename to eng/common/core-templates/job/helix-job-monitor.yml index efa2ad9ab32..e7ee3815a56 100644 --- a/eng/common/core-templates/job/helix-reporter-job.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -1,11 +1,11 @@ parameters: - jobName: HelixReporter - displayName: Helix Reporter Job + jobName: HelixJobMonitor + displayName: Helix Job Monitor pool: {} dotnetVersion: 11.0.x includePreviewVersions: true - toolPackageId: Microsoft.DotNet.Helix.Reporter - toolCommand: dotnet-helix-reporter + toolPackageId: Microsoft.DotNet.Helix.JobMonitor + toolCommand: dotnet-helix-job-monitor toolVersion: '' toolSource: '' helixBaseUri: https://helix.dot.net/ @@ -13,7 +13,7 @@ parameters: pollingIntervalSeconds: 30 timeoutInMinutes: 360 stepTimeoutInMinutes: 360 - reporterJobName: Helix Reporter + jobMonitorName: Helix Job Monitor jobs: - job: ${{ parameters.jobName }} @@ -38,7 +38,7 @@ jobs: includePreviewVersions: ${{ parameters.includePreviewVersions }} - pwsh: | - $toolPath = Join-Path $env:AGENT_TEMPDIRECTORY 'helix-reporter-tool' + $toolPath = Join-Path $env:AGENT_TEMPDIRECTORY 'helix-job-monitor-tool' New-Item -ItemType Directory -Force -Path $toolPath | Out-Null Push-Location $env:AGENT_TEMPDIRECTORY @@ -69,25 +69,25 @@ jobs: } if ($LASTEXITCODE -ne 0) { - throw "Failed to install the Helix reporter tool package '$packageId'." + throw "Failed to install the Helix job monitor tool package '$packageId'." } Write-Host "##vso[task.prependpath]$toolPath" - Write-Host "##vso[task.setvariable variable=HelixReporterToolPath]$toolPath" + Write-Host "##vso[task.setvariable variable=HelixJobMonitorToolPath]$toolPath" } finally { Pop-Location } - displayName: Install Helix reporter + displayName: Install Helix job monitor timeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} - pwsh: | - & '${{ parameters.toolCommand }}' --helix-base-uri '${{ parameters.helixBaseUri }}' --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' --max-wait-minutes '${{ parameters.timeoutInMinutes }}' --reporter-job-name '${{ parameters.reporterJobName }}' + & '${{ parameters.toolCommand }}' --helix-base-uri '${{ parameters.helixBaseUri }}' --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' --max-wait-minutes '${{ parameters.timeoutInMinutes }}' --job-monitor-name '${{ parameters.jobMonitorName }}' if ($LASTEXITCODE -ne 0) { - throw "The Helix reporter tool exited with code $LASTEXITCODE." + throw "The Helix job monitor tool exited with code $LASTEXITCODE." } - displayName: Run Helix reporter + displayName: Run Helix job monitor timeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/Job.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/Job.cs index 2cc70f5c118..348ce2546d4 100644 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/Job.cs +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/Job.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Net.NetworkInformation; using System.Threading; using System.Threading.Tasks; using Microsoft.DotNet.Helix.Client.Models; diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/JobStatus.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/JobStatus.cs new file mode 100644 index 00000000000..0cdbf823da1 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/JobStatus.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.DotNet.Helix.Client.Models +{ + public partial class JobStatus + { + public bool IsCompleted => string.Equals(Status, "finished", StringComparison.OrdinalIgnoreCase) + || string.Equals(Status, "failed", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs new file mode 100644 index 00000000000..e5794036be5 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace Microsoft.DotNet.Helix.Client.Models +{ + public partial class JobStatus + { + [JsonProperty("JobName")] + public string JobName { get; set; } + + [JsonProperty("Status")] + public string Status { get; set; } + } +} diff --git a/src/Microsoft.DotNet.Helix/Reporter/HelixReporterJobUtilities.cs b/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs similarity index 79% rename from src/Microsoft.DotNet.Helix/Reporter/HelixReporterJobUtilities.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs index f95f3cb2060..4e1afde8e85 100644 --- a/src/Microsoft.DotNet.Helix/Reporter/HelixReporterJobUtilities.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs @@ -7,7 +7,7 @@ using System.Net; using Newtonsoft.Json; -namespace Microsoft.DotNet.Helix.Reporter +namespace Microsoft.DotNet.Helix.JobMonitor { public sealed class AzureDevOpsTimelineRecord { @@ -30,7 +30,7 @@ public sealed class AzureDevOpsTimelineRecord public string Result { get; set; } } - public static class HelixReporterJobUtilities + public static class HelixJobMonitorUtilities { public static string NormalizeRepository(string repository) { @@ -45,7 +45,7 @@ public static string NormalizeRepository(string repository) return repository.Trim('/'); } - string[] segments = uri.AbsolutePath.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + string[] segments = uri.AbsolutePath.Split(['/'], StringSplitOptions.RemoveEmptyEntries); if (uri.Host.Contains("github.com", StringComparison.OrdinalIgnoreCase) && segments.Length >= 2) { return $"{segments[0]}/{segments[1]}"; @@ -70,16 +70,16 @@ public static string NormalizeRepository(string repository) return repository; } - public static bool AreNonReporterJobsComplete(IEnumerable records, string reporterJobName) - => GetRelevantJobRecords(records, reporterJobName).All(IsTerminal); + public static bool AreNonMonitorJobsComplete(IEnumerable records, string jobMonitorName) + => GetRelevantJobRecords(records, jobMonitorName).All(IsTerminal); - public static bool HasFailedNonReporterJobs(IEnumerable records, string reporterJobName) - => GetRelevantJobRecords(records, reporterJobName).Any(r => + public static bool HasFailedNonMonitorJobs(IEnumerable records, string jobMonitorName) + => GetRelevantJobRecords(records, jobMonitorName).Any(r => string.Equals(r.Result, "failed", StringComparison.OrdinalIgnoreCase) || string.Equals(r.Result, "canceled", StringComparison.OrdinalIgnoreCase)); public static string GetTestRunName(string helixJobName) - => $"Helix Reporter - {helixJobName}"; + => $"Helix Job Monitor - {helixJobName}"; public static string CleanWorkItemName(string name) { @@ -96,11 +96,11 @@ public static string CleanWorkItemName(string name) return name.Replace('/', '-').Replace('\\', '-'); } - private static IEnumerable GetRelevantJobRecords(IEnumerable records, string reporterJobName) + private static IEnumerable GetRelevantJobRecords(IEnumerable records, string jobMonitorName) { - return (records ?? Enumerable.Empty()) + return (records ?? []) .Where(r => string.Equals(r.Type, "Job", StringComparison.OrdinalIgnoreCase)) - .Where(r => !string.Equals(r.Name, reporterJobName, StringComparison.OrdinalIgnoreCase)); + .Where(r => !string.Equals(r.Name, jobMonitorName, StringComparison.OrdinalIgnoreCase)); } private static bool IsTerminal(AzureDevOpsTimelineRecord record) diff --git a/src/Microsoft.DotNet.Helix/Reporter/ReporterOptions.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs similarity index 87% rename from src/Microsoft.DotNet.Helix/Reporter/ReporterOptions.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs index a9f85efed16..4d9517c7903 100644 --- a/src/Microsoft.DotNet.Helix/Reporter/ReporterOptions.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs @@ -5,9 +5,9 @@ using System.Globalization; using CommandLine; -namespace Microsoft.DotNet.Helix.Reporter +namespace Microsoft.DotNet.Helix.JobMonitor { - public sealed class ReporterOptions + public sealed class JobMonitorOptions { public bool ShowHelp { get; private set; } @@ -38,8 +38,8 @@ public sealed class ReporterOptions [Option("max-wait-minutes", HelpText = "Maximum run time in minutes.", Default = 360)] public int MaximumWaitMinutes { get; set; } = 360; - [Option("reporter-job-name", HelpText = "Display name of the reporter job in Azure DevOps.")] - public string ReporterJobName { get; set; } = "Helix Reporter"; + [Option("job-monitor-name", HelpText = "Display name of the Helix Job Monitor job in Azure DevOps.")] + public string JobMonitorName { get; set; } = "Helix Job Monitor"; [Option("working-directory", HelpText = "Directory used to stage downloaded test results.")] public string WorkingDirectory { get; set; } @@ -50,25 +50,25 @@ public sealed class ReporterOptions [Option("attempt", HelpText = "Azure DevOps attempt number for the current job.")] public int? Attempt { get; set; } - public static ReporterOptions Parse(string[] args) + public static JobMonitorOptions Parse(string[] args) { - ReporterOptions parsed = null; + JobMonitorOptions parsed = null; var parser = new Parser(settings => { settings.CaseInsensitiveEnumValues = true; settings.HelpWriter = Console.Out; }); - parser.ParseArguments(args) + parser.ParseArguments(args) .WithParsed(options => parsed = options) .WithNotParsed(errors => { - parsed = new ReporterOptions { ShowHelp = true }; + parsed = new JobMonitorOptions { ShowHelp = true }; }); if (parsed == null || parsed.ShowHelp) { - return parsed ?? new ReporterOptions { ShowHelp = true }; + return parsed ?? new JobMonitorOptions { ShowHelp = true }; } parsed.ApplyEnvironmentDefaults(); @@ -83,11 +83,11 @@ private void ApplyEnvironmentDefaults() TeamProject ??= Environment.GetEnvironmentVariable("SYSTEM_TEAMPROJECT"); BuildId ??= Environment.GetEnvironmentVariable("BUILD_BUILDID"); AccessToken ??= Environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN"); - Repository = HelixReporterJobUtilities.NormalizeRepository( + Repository = HelixJobMonitorUtilities.NormalizeRepository( Repository ?? Environment.GetEnvironmentVariable("BUILD_REPOSITORY_URI") ?? Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")); - WorkingDirectory ??= System.IO.Path.Combine(System.IO.Path.GetTempPath(), "helix-reporter", BuildId ?? "unknown"); + WorkingDirectory ??= System.IO.Path.Combine(System.IO.Path.GetTempPath(), "helix-job-monitor", BuildId ?? "unknown"); PrNumber ??= GetEnvironmentInt("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"); Attempt ??= GetEnvironmentInt("SYSTEM_JOBATTEMPT"); } diff --git a/src/Microsoft.DotNet.Helix/Reporter/ReporterRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs similarity index 83% rename from src/Microsoft.DotNet.Helix/Reporter/ReporterRunner.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 69c8738e082..dc4d410c849 100644 --- a/src/Microsoft.DotNet.Helix/Reporter/ReporterRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -11,7 +11,6 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Text; -using System.Threading; using System.Threading.Tasks; using Azure; using Azure.Storage.Blobs; @@ -20,15 +19,15 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; -namespace Microsoft.DotNet.Helix.Reporter +namespace Microsoft.DotNet.Helix.JobMonitor { - internal sealed class ReporterRunner : IDisposable + internal sealed class JobMonitorRunner : IDisposable { - private readonly ReporterOptions _options; + private readonly JobMonitorOptions _options; private readonly HttpClient _azdoClient; private readonly IHelixApi _helixApi; - public ReporterRunner(ReporterOptions options) + public JobMonitorRunner(JobMonitorOptions options) { _options = options ?? throw new ArgumentNullException(nameof(options)); Directory.CreateDirectory(_options.WorkingDirectory); @@ -40,39 +39,39 @@ public ReporterRunner(ReporterOptions options) _azdoClient = new HttpClient(); string encodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + _options.AccessToken)); _azdoClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", encodedToken); - _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-reporter"); + _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-job-monitor"); } public async Task RunAsync() { HashSet processedRuns = await GetProcessedRunNamesAsync().ConfigureAwait(false); DateTimeOffset deadline = DateTimeOffset.UtcNow.AddMinutes(Math.Max(1, _options.MaximumWaitMinutes)); - bool anyNonReporterJobFailures = false; + bool anyNonMonitorJobFailures = false; int failedHelixJobCount = 0; int processedHelixJobCount = 0; while (DateTimeOffset.UtcNow < deadline) { AzureDevOpsTimelineRecord[] timelineRecords = await GetTimelineRecordsAsync().ConfigureAwait(false); - JobsByBuildResponse helixResponse = await RetryAsync(() => _helixApi.Job.ByBuildAsync(_options.Repository, _options.PrNumber, int.Parse(_options.BuildId, CultureInfo.InvariantCulture), _options.Attempt)).ConfigureAwait(false); - IEnumerable jobs = helixResponse?.Jobs ?? Enumerable.Empty(); + JobStatus[] jobs = await RetryAsync( + () => Task.FromResult(Array.Empty()) + /*TODO _helixApi.PullRequests.ByBuildAsync(_options.Repository, _options.PrNumber, int.Parse(_options.BuildId, CultureInfo.InvariantCulture), _options.Attempt)*/) + .ConfigureAwait(false); - int totalHelixJobs = jobs.Count(); - int completedHelixJobs = jobs.Count(j => j.IsTerminal); - int currentFailedJobs = jobs.Count(j => j.WorkItemsFailed > 0 || string.Equals(j.Status, "failed", StringComparison.OrdinalIgnoreCase)); - Console.WriteLine($"[{DateTime.UtcNow:HH:mm:ss}] {completedHelixJobs}/{totalHelixJobs} Helix jobs complete ({currentFailedJobs} failed). Waiting..."); + int completedHelixJobs = jobs.Count(j => j.IsCompleted); + int currentFailedJobs = jobs.Count(j => j.Status.Equals("failed", StringComparison.OrdinalIgnoreCase)); + Console.WriteLine($"[{DateTime.UtcNow:HH:mm:ss}] {completedHelixJobs}/{jobs.Length} Helix jobs complete ({currentFailedJobs} failed). Waiting..."); - foreach (JobBuildSummary job in jobs.Where(j => j.IsTerminal).OrderBy(j => j.JobName, StringComparer.OrdinalIgnoreCase)) + foreach (JobStatus job in jobs.Where(j => j.IsCompleted).OrderBy(j => j.JobName, StringComparer.OrdinalIgnoreCase)) { - string testRunName = HelixReporterJobUtilities.GetTestRunName(job.JobName); - if (processedRuns.Contains(testRunName)) + if (processedRuns.Contains(job.JobName)) { continue; } JobPassFail passFail = await RetryAsync(() => _helixApi.Job.PassFailAsync(job.JobName)).ConfigureAwait(false); - bool passed = await ProcessCompletedJobAsync(job, passFail, testRunName).ConfigureAwait(false); - processedRuns.Add(testRunName); + bool passed = await ProcessCompletedJobAsync(job, passFail).ConfigureAwait(false); + processedRuns.Add(job.JobName); processedHelixJobCount++; if (!passed) { @@ -80,23 +79,23 @@ public async Task RunAsync() } } - anyNonReporterJobFailures = HelixReporterJobUtilities.HasFailedNonReporterJobs(timelineRecords, _options.ReporterJobName); - bool allPipelineJobsComplete = HelixReporterJobUtilities.AreNonReporterJobsComplete(timelineRecords, _options.ReporterJobName); - bool allHelixJobsComplete = !jobs.Any() || helixResponse.AllJobsComplete || jobs.All(j => j.IsTerminal); + anyNonMonitorJobFailures = HelixJobMonitorUtilities.HasFailedNonMonitorJobs(timelineRecords, _options.JobMonitorName); + bool allPipelineJobsComplete = HelixJobMonitorUtilities.AreNonMonitorJobsComplete(timelineRecords, _options.JobMonitorName); + bool allHelixJobsComplete = jobs.Any() && jobs.All(j => j.IsCompleted); if (allPipelineJobsComplete && allHelixJobsComplete) { Console.WriteLine($"Final summary: processed {processedHelixJobCount} Helix job(s); {failedHelixJobCount} failed."); - if (anyNonReporterJobFailures || failedHelixJobCount > 0) + if (anyNonMonitorJobFailures || failedHelixJobCount > 0) { - if (anyNonReporterJobFailures) + if (anyNonMonitorJobFailures) { - Console.Error.WriteLine("One or more non-reporter pipeline jobs failed."); + Console.Error.WriteLine("One or more non-monitor pipeline jobs failed."); } if (failedHelixJobCount > 0) { - Console.Error.WriteLine($"The reporter detected failures in {failedHelixJobCount} Helix job(s)."); + Console.Error.WriteLine($"The Helix Job Monitor detected failures in {failedHelixJobCount} Helix job(s)."); } return 1; @@ -108,7 +107,7 @@ public async Task RunAsync() await Task.Delay(TimeSpan.FromSeconds(Math.Max(5, _options.PollingIntervalSeconds))).ConfigureAwait(false); } - Console.Error.WriteLine($"The reporter timed out after {_options.MaximumWaitMinutes} minute(s)."); + Console.Error.WriteLine($"The Helix Job Monitor timed out after {_options.MaximumWaitMinutes} minute(s)."); return 1; } @@ -117,8 +116,9 @@ public void Dispose() _azdoClient.Dispose(); } - private async Task ProcessCompletedJobAsync(JobBuildSummary helixJob, JobPassFail passFail, string testRunName) + private async Task ProcessCompletedJobAsync(JobStatus helixJob, JobPassFail passFail) { + string testRunName = HelixJobMonitorUtilities.GetTestRunName(helixJob.JobName); int testRunId = await StartTestRunAsync(testRunName).ConfigureAwait(false); string resultsDirectory = Path.Combine(_options.WorkingDirectory, MakeSafeDirectoryName(helixJob.JobName)); Directory.CreateDirectory(resultsDirectory); @@ -143,12 +143,12 @@ private async Task> GetProcessedRunNamesAsync() JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?buildIds={_options.BuildId}&api-version=7.1-preview.1").ConfigureAwait(false); var processed = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (JObject run in data?["value"] as JArray ?? new JArray()) + foreach (JObject run in data?["value"] as JArray ?? []) { string name = run.Value("name"); string state = run.Value("state"); if (!string.IsNullOrEmpty(name) - && name.StartsWith("Helix Reporter - ", StringComparison.OrdinalIgnoreCase) + && name.StartsWith("Helix Job Monitor - ", StringComparison.OrdinalIgnoreCase) && string.Equals(state, "Completed", StringComparison.OrdinalIgnoreCase)) { processed.Add(name); @@ -161,7 +161,7 @@ private async Task> GetProcessedRunNamesAsync() private async Task GetTimelineRecordsAsync() { JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/build/builds/{_options.BuildId}/timeline?api-version=7.1-preview.2").ConfigureAwait(false); - return data?["records"]?.ToObject() ?? Array.Empty(); + return data?["records"]?.ToObject() ?? []; } private async Task StartTestRunAsync(string testRunName) @@ -205,7 +205,7 @@ private async Task CreateFallbackResultsAsync(int testRunId, string jobName, Job private async Task CreateFallbackResultAsync(int testRunId, string jobName, string workItemName, bool failed) { - string cleanName = HelixReporterJobUtilities.CleanWorkItemName(workItemName); + string cleanName = HelixJobMonitorUtilities.CleanWorkItemName(workItemName); await SendAsync( HttpMethod.Post, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/Runs/{testRunId}/results?api-version=5.1-preview.6", @@ -367,12 +367,9 @@ private static BlobClient CreateBlobClient(string fileLink, string resultsSas) private static bool LooksLikeTestResultFile(string path) { string fileName = Path.GetFileName(path); - return fileName.EndsWith(".trx", StringComparison.OrdinalIgnoreCase) - || (fileName.EndsWith(".xml", StringComparison.OrdinalIgnoreCase) - && (fileName.Contains("testresults", StringComparison.OrdinalIgnoreCase) - || fileName.Contains("test-results", StringComparison.OrdinalIgnoreCase) - || fileName.Contains("junit", StringComparison.OrdinalIgnoreCase) - || fileName.Contains("xunit", StringComparison.OrdinalIgnoreCase))); + return fileName.EndsWith(".xml", StringComparison.OrdinalIgnoreCase) + && (fileName.Contains("testresults", StringComparison.OrdinalIgnoreCase) + || fileName.Contains("test-results", StringComparison.OrdinalIgnoreCase)); } private static string MakeSafeDirectoryName(string value) diff --git a/src/Microsoft.DotNet.Helix/Reporter/Microsoft.DotNet.Helix.Reporter.csproj b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj similarity index 83% rename from src/Microsoft.DotNet.Helix/Reporter/Microsoft.DotNet.Helix.Reporter.csproj rename to src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj index 99cf8e09b83..e4b5b06a672 100644 --- a/src/Microsoft.DotNet.Helix/Reporter/Microsoft.DotNet.Helix.Reporter.csproj +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj @@ -5,8 +5,8 @@ Exe true true - dotnet-helix-reporter - Standalone Helix reporter job tool for Azure DevOps pipelines + dotnet-helix-job-monitor + Standalone Helix Job Monitor tool for Azure DevOps pipelines diff --git a/src/Microsoft.DotNet.Helix/Reporter/Program.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs similarity index 77% rename from src/Microsoft.DotNet.Helix/Reporter/Program.cs rename to src/Microsoft.DotNet.Helix/JobMonitor/Program.cs index e4a2bdf9fae..574d89f7af2 100644 --- a/src/Microsoft.DotNet.Helix/Reporter/Program.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs @@ -4,7 +4,7 @@ using System; using System.Threading.Tasks; -namespace Microsoft.DotNet.Helix.Reporter +namespace Microsoft.DotNet.Helix.JobMonitor { internal static class Program { @@ -12,13 +12,13 @@ public static async Task Main(string[] args) { try { - ReporterOptions options = ReporterOptions.Parse(args); + JobMonitorOptions options = JobMonitorOptions.Parse(args); if (options.ShowHelp) { return 0; } - ReporterRunner runner = new ReporterRunner(options); + JobMonitorRunner runner = new JobMonitorRunner(options); return await runner.RunAsync().ConfigureAwait(false); } catch (Exception ex) diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixReporterJobUtilitiesTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixJobMonitorUtilitiesTests.cs similarity index 61% rename from src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixReporterJobUtilitiesTests.cs rename to src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixJobMonitorUtilitiesTests.cs index 388d18f0585..ade2e952d09 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixReporterJobUtilitiesTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixJobMonitorUtilitiesTests.cs @@ -1,13 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.Helix.Reporter; +using Microsoft.DotNet.Helix.JobMonitor; using System; using Xunit; namespace Microsoft.DotNet.Helix.Sdk.Tests { - public class HelixReporterJobUtilitiesTests + public class HelixJobMonitorUtilitiesTests { [Theory] [InlineData("https://github.com/dotnet/arcade", "dotnet/arcade")] @@ -15,39 +15,39 @@ public class HelixReporterJobUtilitiesTests [InlineData("dotnet/arcade", "dotnet/arcade")] public void NormalizeRepository_ReturnsStableIdentifier(string input, string expected) { - Assert.Equal(expected, HelixReporterJobUtilities.NormalizeRepository(input)); + Assert.Equal(expected, HelixJobMonitorUtilities.NormalizeRepository(input)); } [Fact] - public void AreNonReporterJobsComplete_IgnoresReporterRecord() + public void AreNonMonitorJobsComplete_IgnoresMonitorRecord() { var records = new[] { new AzureDevOpsTimelineRecord { Type = "Job", Name = "Build Linux", State = "completed", Result = "succeeded" }, - new AzureDevOpsTimelineRecord { Type = "Job", Name = "Helix Reporter", State = "inProgress", Result = null }, + new AzureDevOpsTimelineRecord { Type = "Job", Name = "Helix Job Monitor", State = "inProgress", Result = null }, }; - Assert.True(HelixReporterJobUtilities.AreNonReporterJobsComplete(records, "Helix Reporter")); + Assert.True(HelixJobMonitorUtilities.AreNonMonitorJobsComplete(records, "Helix Job Monitor")); } [Fact] - public void HasFailedNonReporterJobs_DetectsFailures() + public void HasFailedNonMonitorJobs_DetectsFailures() { var records = new[] { new AzureDevOpsTimelineRecord { Type = "Job", Name = "Build Linux", State = "completed", Result = "failed" }, - new AzureDevOpsTimelineRecord { Type = "Job", Name = "Helix Reporter", State = "inProgress", Result = null }, + new AzureDevOpsTimelineRecord { Type = "Job", Name = "Helix Job Monitor", State = "inProgress", Result = null }, }; - Assert.True(HelixReporterJobUtilities.HasFailedNonReporterJobs(records, "Helix Reporter")); + Assert.True(HelixJobMonitorUtilities.HasFailedNonMonitorJobs(records, "Helix Job Monitor")); } [Fact] public void GetTestRunName_ProducesStableName() { Assert.Equal( - "Helix Reporter - coreclr-tests-linux-x64", - HelixReporterJobUtilities.GetTestRunName("coreclr-tests-linux-x64")); + "Helix Job Monitor - coreclr-tests-linux-x64", + HelixJobMonitorUtilities.GetTestRunName("coreclr-tests-linux-x64")); } } } diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj index 97bb7663e0b..c464b912e7d 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj @@ -22,7 +22,7 @@ - + diff --git a/src/Microsoft.DotNet.Helix/Sdk/Readme.md b/src/Microsoft.DotNet.Helix/Sdk/Readme.md index f497ff20992..a9a0c730e52 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/Readme.md +++ b/src/Microsoft.DotNet.Helix/Sdk/Readme.md @@ -44,30 +44,30 @@ env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure DevOps ``` -### Helix Reporter Job for Azure DevOps +### Helix Job Monitor for Azure DevOps -If you want to decouple Helix test execution from the build agents that submit the work, use the Helix Reporter Job. +If you want to decouple Helix test execution from the build agents that submit the work, use the Helix Job Monitor. -The reporter job is a lightweight dedicated pipeline job that: +The job monitor is a lightweight dedicated pipeline job that: - polls Azure DevOps for pipeline state, - polls Helix for jobs associated with the current build, - downloads test result artifacts from completed Helix jobs, - publishes results to Azure DevOps incrementally, -- returns a final green or red status once all non-reporter jobs and Helix jobs have completed. +- returns a final green or red status once all non-monitor jobs and Helix jobs have completed. This allows the original build jobs to stop waiting on Helix execution while still preserving test visibility and pass/fail behavior in the pipeline. -The job is added with the template at [/eng/common/core-templates/job/helix-reporter-job.yml](/eng/common/core-templates/job/helix-reporter-job.yml). +The job is added with the template at [/eng/common/core-templates/job/helix-job-monitor.yml](/eng/common/core-templates/job/helix-job-monitor.yml). Example: ```yaml jobs: -- template: /eng/common/core-templates/job/helix-reporter-job.yml@self +- template: /eng/common/core-templates/job/helix-job-monitor.yml@self parameters: - jobName: HelixReporter - displayName: Helix Reporter Job + jobName: HelixJobMonitor + displayName: Helix Job Monitor pollingIntervalSeconds: 30 timeoutInMinutes: 360 ``` @@ -76,9 +76,9 @@ Useful parameters: - `helixBaseUri`: base URI for the Helix service. Defaults to `https://helix.dot.net/`. - `helixAccessToken`: optional token for authenticated Helix access on internal builds. -- `pollingIntervalSeconds`: how often the reporter checks for new completed jobs. -- `timeoutInMinutes`: overall timeout for the reporter job. -- `reporterJobName`: name used to identify and exclude the reporter job in the Azure DevOps timeline. +- `pollingIntervalSeconds`: how often the job monitor checks for new completed jobs. +- `timeoutInMinutes`: overall timeout for the job monitor. +- `jobMonitorName`: name used to identify and exclude the Helix Job Monitor job in the Azure DevOps timeline. Behavior notes: From 8f3a9ae9a048fe7ab0a236abe3ec13f20e1be827 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 20 Apr 2026 17:53:50 +0200 Subject: [PATCH 03/66] Add the AzDO reporter project --- Arcade.slnx | 3 +- Directory.Packages.props | 1 + eng/Version.Details.props | 2 + eng/Version.Details.xml | 4 + .../AzureDevOpsResultPublisher.cs | 736 ++++++++++++++++++ .../AzureDevOpsTestReporter.csproj | 13 + .../PackingTestReporter.cs | 92 +++ .../ResultAggregator.cs | 346 ++++++++ .../TestReportingModels.cs | 183 +++++ 9 files changed, 1379 insertions(+), 1 deletion(-) create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsTestReporter.csproj create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/ResultAggregator.cs create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs diff --git a/Arcade.slnx b/Arcade.slnx index 0ed52fd8f54..05849deea49 100644 --- a/Arcade.slnx +++ b/Arcade.slnx @@ -5,9 +5,10 @@ + - + diff --git a/Directory.Packages.props b/Directory.Packages.props index dc3da890ce4..6d8d8a4c60d 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -56,6 +56,7 @@ + diff --git a/eng/Version.Details.props b/eng/Version.Details.props index 6bbc63729bf..a0bf25ed1f5 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -34,6 +34,7 @@ This file should be imported by eng/Versions.props 10.0.3 10.0.3 10.0.3 + 10.0.3 10.0.3 10.0.3 10.0.3 @@ -82,6 +83,7 @@ This file should be imported by eng/Versions.props $(MicrosoftExtensionsFileProvidersAbstractionsPackageVersion) $(MicrosoftExtensionsFileSystemGlobbingPackageVersion) $(MicrosoftExtensionsHttpPackageVersion) + $(MicrosoftExtensionsLoggingAbstractionsPackageVersion) $(MicrosoftExtensionsLoggingConsolePackageVersion) $(SystemCompositionPackageVersion) $(SystemIOPackagingPackageVersion) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6875dba638a..2a608b88458 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -90,6 +90,10 @@ + + https://github.com/dotnet/runtime + dc5fd7a8dce8309e4add8fd4bd5d8718f221b15a + https://github.com/dotnet/runtime dc5fd7a8dce8309e4add8fd4bd5d8718f221b15a diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs new file mode 100644 index 00000000000..5347ea571ba --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs @@ -0,0 +1,736 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.IO.Compression; +using System.Net; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; + +public enum UploadResult +{ + Success = 1, + UnknownError = 2, + TerminalError = 3, +} + +public static class UploadResultExtensions +{ + public static UploadResult Aggregate(this UploadResult value, UploadResult other) + { + return (UploadResult)Math.Max((int)value, (int)other); + } +} + +public sealed class AzureDevOpsReportingError : Exception +{ + public AzureDevOpsReportingError(string message) + : base(message) + { + } +} + +internal sealed class TerminalError : Exception +{ + public TerminalError(string message) + : base(message) + { + } +} + +public sealed class AzureDevOpsResultPublisher +{ + private const int TestListBuckets = 32; + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + { + WriteIndented = false, + }; + + private static string s_lastSendContent = string.Empty; + + private readonly AzureDevOpsReportingParameters _azdoParameters; + private readonly string _workItemId; + private readonly IEventClient _eventClient; + private readonly IUploadClient _uploadClient; + private readonly HttpClient _httpClient; + private readonly ILogger _logger; + + public AzureDevOpsResultPublisher( + AzureDevOpsReportingParameters azdoParameters, + string workItemId, + IEventClient? eventClient = null, + IUploadClient? uploadClient = null, + HttpClient? httpClient = null, + ILogger? logger = null) + { + _azdoParameters = azdoParameters; + _workItemId = workItemId; + _eventClient = eventClient ?? NullEventClient.Instance; + _uploadClient = uploadClient ?? NullUploadClient.Instance; + _httpClient = httpClient ?? CreateHttpClient(azdoParameters.AccessToken); + _logger = logger.OrNull(); + } + + public async Task TryUploadAsync(IEnumerable results, CancellationToken cancellationToken = default) + { + try + { + await ProcessAsync(results.ToList(), cancellationToken); + return UploadResult.Success; + } + catch (TerminalError ex) + { + await LogErrorAsync(ex, cancellationToken); + return UploadResult.TerminalError; + } + catch (Exception ex) + { + await LogErrorAsync(ex, cancellationToken); + return UploadResult.UnknownError; + } + } + + private async Task ProcessAsync(IReadOnlyList testList, CancellationToken cancellationToken) + { + var converted = ConvertResults(testList).ToList(); + var hotPathTests = new List(); + + foreach (var batch in Batch(converted, 1000, static t => Size(t.Converted))) + { + var publishedTests = await PublishResultsAsync(batch, cancellationToken); + hotPathTests.AddRange(publishedTests); + _logger.LogInformation("Uploaded {Count} results", publishedTests.Count); + } + + await SendMetadataAsync(hotPathTests, testList, cancellationToken); + } + + private async Task LogErrorAsync(Exception exception, CancellationToken cancellationToken) + { + _logger.LogError(exception, "Failed to upload test results to Azure DevOps."); + await _eventClient.ErrorAsync( + HelixEnvironmentSettings.FromEnvironment(), + "DevOpsReportFailure", + $"Failed to upload results: {exception.Message}", + cancellationToken: cancellationToken); + } + + private async Task SendMetadataAsync( + IReadOnlyList backChannelCases, + IEnumerable allTestResults, + CancellationToken cancellationToken) + { + var partitionedResults = new Dictionary>(); + var resultCounts = new Dictionary(StringComparer.Ordinal); + + void ProcessResultForMetadata(AggregatedResult result) + { + resultCounts[result.Result] = resultCounts.TryGetValue(result.Result, out var count) ? count + 1 : 1; + if (!string.Equals(result.Result, "Passed", StringComparison.Ordinal)) + { + return; + } + + var name = result.Name; + string? argumentHash = null; + var partitionKey = name; + var parenthesisIndex = name.IndexOf('('); + if (parenthesisIndex >= 0) + { + var argumentList = name[(parenthesisIndex + 1)..].TrimEnd(')'); + name = name[..parenthesisIndex]; + argumentHash = Convert.ToBase64String(SHA1.HashData(Encoding.UTF8.GetBytes(argumentList))); + partitionKey = name + argumentHash; + } + + var bucket = SHA1.HashData(Encoding.UTF8.GetBytes(partitionKey))[0] % TestListBuckets; + if (!partitionedResults.TryGetValue(bucket, out var testNames)) + { + testNames = new List(); + partitionedResults[bucket] = testNames; + } + + testNames.Add(new TestListRow(name, argumentHash)); + } + + void ProcessTestForMetadata(AggregatedResult result) + { + if (result.AggregationType == AggregationType.DataDriven && result.SubResults.Count > 0) + { + foreach (var subResult in result.SubResults) + { + ProcessTestForMetadata(subResult); + } + } + else if (result.AggregationType == AggregationType.Single) + { + ProcessResultForMetadata(result); + } + } + + foreach (var result in allTestResults) + { + ProcessTestForMetadata(result); + } + + var uploadedUrls = new Dictionary(); + foreach (var (key, testNames) in partitionedResults) + { + var csvBytes = CreateCompressedCsv(testNames); + var fileName = $"{Guid.NewGuid():N}.csv.gz"; + uploadedUrls[key] = await _uploadClient.UploadAsync(csvBytes, fileName, "application/gzip", cancellationToken); + } + + var dataModel = new + { + version = 2, + rerun_tests = backChannelCases, + test_lists = uploadedUrls, + partitions = TestListBuckets, + result_counts = resultCounts, + }; + + var rawBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(dataModel, SerializerOptions)); + var compressedBytes = Compress(rawBytes); + var base64Data = Convert.ToBase64String(compressedBytes); + var fileNameBase = $"__helix_metadata_{Guid.NewGuid():N}.json.gz"; + + await SendWithRetryAsync( + HttpMethod.Post, + $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/attachments?api-version=7.1-preview.1", + new TestRunAttachmentRequest(fileNameBase, base64Data), + cancellationToken); + + var metadataUrl = await _uploadClient.UploadAsync(compressedBytes, fileNameBase, "application/gzip", cancellationToken); + await _eventClient.SendAsync( + new + { + Type = "AzureDevOpsTestRunMetadata", + TestRunProject = _azdoParameters.TeamProject, + TestRunId = _azdoParameters.TestRunId, + Url = metadataUrl, + }, + cancellationToken); + } + + private async Task> PublishResultsAsync( + IReadOnlyList converted, + CancellationToken cancellationToken) + { + var testCaseResults = converted.Select(static c => c.Converted).ToList(); + var originalList = converted.Select(static c => c.Aggregated).ToList(); + + using var response = await SendWithRetryAsync( + HttpMethod.Post, + $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results?api-version=7.1-preview.6", + testCaseResults, + cancellationToken); + + var publishedResults = await ReadPublishedResultsAsync(response, cancellationToken); + if (publishedResults.Count == 0) + { + _logger.LogWarning("The test run appears to have been closed, aborting test result uploads."); + return Array.Empty(); + } + + var hotPathTests = new List(); + foreach (var triplet in publishedResults.Zip(originalList, testCaseResults)) + { + var published = triplet.First; + var original = triplet.Second; + var testCase = triplet.Third; + + if (published.Id == -1) + { + _logger.LogWarning("Azure DevOps test ID returned -1, unable to attach files."); + continue; + } + + testCase = testCase with { Id = published.Id }; + var addedTest = false; + + void AddToHotPath() + { + if (addedTest) + { + return; + } + + addedTest = true; + hotPathTests.Add(testCase); + } + + async Task IterateSubResultsAsync( + IReadOnlyList? publishedSubResults, + IReadOnlyList originalSubResults, + long testId) + { + if (publishedSubResults is null || publishedSubResults.Count == 0) + { + if (originalSubResults.Count > 0) + { + _logger.LogError("Published results do not include sub-results, attachments lost."); + } + + return; + } + + if (original.AggregationType == AggregationType.Rerun) + { + AddToHotPath(); + } + + if (publishedSubResults.Count != originalSubResults.Count) + { + _logger.LogError("Published sub-result counts do not match uploaded attachments. Attachments lost."); + return; + } + + foreach (var subTriplet in publishedSubResults.Zip(originalSubResults, (publishedSubResult, originalSubResult) => (publishedSubResult, originalSubResult))) + { + foreach (var attachment in subTriplet.originalSubResult.Attachments) + { + await SendAttachmentAsync(attachment, testId, subTriplet.publishedSubResult.Id, cancellationToken); + } + + await IterateSubResultsAsync(subTriplet.publishedSubResult.SubResults, subTriplet.originalSubResult.SubResults, testId); + } + } + + foreach (var attachment in original.Attachments) + { + await SendAttachmentAsync(attachment, published.Id, null, cancellationToken); + } + + await IterateSubResultsAsync(published.SubResults, original.SubResults, published.Id); + } + + return hotPathTests; + } + + private async Task SendAttachmentAsync( + TestResultAttachment attachment, + long testId, + long? subResultId, + CancellationToken cancellationToken) + { + var request = new TestRunAttachmentRequest( + attachment.Name, + Convert.ToBase64String(Encoding.UTF8.GetBytes(attachment.Text))); + + var path = subResultId is long subId + ? $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results/{testId}/subresults/{subId}/attachments?api-version=7.1-preview.1" + : $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results/{testId}/attachments?api-version=7.1-preview.1"; + + using var response = await SendWithRetryAsync(HttpMethod.Post, path, request, cancellationToken); + _ = response; + } + + private IEnumerable ConvertResults(IEnumerable results) + { + var settings = HelixEnvironmentSettings.FromEnvironment(); + var comment = JsonSerializer.Serialize(new + { + HelixJobId = settings.CorrelationId, + HelixWorkItemName = settings.WorkItemFriendlyName, + }); + + static string GetResultGroupType(AggregationType aggregationType) + { + return aggregationType switch + { + AggregationType.Single => "None", + AggregationType.DataDriven => "dataDriven", + AggregationType.Rerun => "rerun", + _ => "None", + }; + } + + PublishedSubResult ConvertToSubTest(AggregatedResult result) + { + var customFields = new List(); + if (result.IsFlaky) + { + customFields.Add(new CustomField("IsTestResultFlaky", true)); + } + + if ((result.AttemptId ?? 0) > 1) + { + customFields.Add(new CustomField("AttemptId", result.AttemptId!.Value - 1)); + } + + return new PublishedSubResult + { + Comment = comment ?? string.Empty, + CustomFields = customFields, + DisplayName = result.Name, + Outcome = result.Result, + DurationInMs = result.DurationSeconds * 1000.0, + StackTrace = result.StackTrace, + ErrorMessage = result.FailureMessage, + SubResults = result.SubResults.Count == 0 ? null : result.SubResults.Select(ConvertToSubTest).ToList(), + ResultGroupType = GetResultGroupType(result.AggregationType), + }; + } + + ConvertedResult ConvertResult(AggregatedResult result) + { + var customFields = new List(); + if (result.IsFlaky) + { + customFields.Add(new CustomField("IsTestResultFlaky", true)); + } + + if (result.AggregationType == AggregationType.Rerun && result.SubResults.Count > 1) + { + customFields.Add(new CustomField("AttemptId", result.SubResults.Count - 1)); + } + + return new ConvertedResult( + new PublishedTestCase + { + TestCaseTitle = result.Name, + AutomatedTestName = result.Name, + AutomatedTestType = "helix", + AutomatedTestStorage = _workItemId, + Priority = 1, + DurationInMs = result.DurationSeconds * 1000.0, + Outcome = result.Result, + State = "Completed", + Comment = comment ?? string.Empty, + StackTrace = result.StackTrace, + ErrorMessage = result.FailureMessage, + SubResults = result.SubResults.Count == 0 ? null : result.SubResults.Select(ConvertToSubTest).ToList(), + ResultGroupType = GetResultGroupType(result.AggregationType), + CustomFields = customFields, + }, + result); + } + + var converted = results.Select(ConvertResult).ToList(); + foreach (var result in converted) + { + foreach (var chunk in Chunk(result, 950)) + { + yield return chunk; + } + } + } + + private static IEnumerable Chunk(ConvertedResult test, int limit) + { + if (Size(test.Converted) <= limit) + { + yield return test; + yield break; + } + + var zippedSubTests = (test.Converted.SubResults ?? new List()) + .Zip(test.Aggregated.SubResults, (converted, aggregated) => new ChunkPair(converted, aggregated)); + + foreach (var zippedBatch in Batch(zippedSubTests, limit, static pair => Size(pair.Converted))) + { + yield return new ConvertedResult( + test.Converted with { SubResults = zippedBatch.Select(static x => x.Converted).ToList(), Id = null }, + new AggregatedResult( + test.Aggregated.AggregationType, + test.Aggregated.Name, + test.Aggregated.DurationSeconds, + test.Aggregated.Result, + zippedBatch.Select(static x => x.Aggregated).ToList(), + test.Aggregated.Attachments, + test.Aggregated.FailureMessage, + test.Aggregated.StackTrace, + isFlaky: test.Aggregated.IsFlaky, + attemptId: test.Aggregated.AttemptId)); + } + } + + private static int Size(PublishedTestCase test) + { + return 1 + (test.SubResults?.Sum(Size) ?? 0); + } + + private static int Size(PublishedSubResult test) + { + return 1 + (test.SubResults?.Sum(Size) ?? 0); + } + + private static IEnumerable> Batch(IEnumerable items, int limit, Func getSize) + { + var currentBatch = new List(); + var currentSize = 0; + + foreach (var item in items) + { + var size = getSize(item); + if (size > limit) + { + throw new InvalidOperationException("Cannot split a result larger than the batching limit."); + } + + if (currentSize + size > limit && currentBatch.Count > 0) + { + yield return currentBatch; + currentBatch = new List(); + currentSize = 0; + } + + currentBatch.Add(item); + currentSize += size; + } + + if (currentBatch.Count > 0) + { + yield return currentBatch; + } + } + + private static HttpClient CreateHttpClient(string accessToken) + { + var client = new HttpClient(); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); + + if (!string.IsNullOrWhiteSpace(accessToken)) + { + var basicToken = Convert.ToBase64String(Encoding.ASCII.GetBytes($":{accessToken}")); + client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", basicToken); + } + + return client; + } + + private async Task SendWithRetryAsync( + HttpMethod method, + string relativePath, + object? payload, + CancellationToken cancellationToken) + { + var triesLeft = 10; + var body = payload is null ? null : JsonSerializer.Serialize(payload, SerializerOptions); + if (!string.IsNullOrEmpty(body)) + { + s_lastSendContent = body; + } + + while (true) + { + using var request = new HttpRequestMessage(method, new Uri(_azdoParameters.CollectionUri, relativePath)); + if (body is not null) + { + request.Content = new StringContent(body, Encoding.UTF8, "application/json"); + } + + var response = await _httpClient.SendAsync(request, cancellationToken); + if (response.IsSuccessStatusCode) + { + return response; + } + + var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + if (response.StatusCode == HttpStatusCode.ServiceUnavailable && triesLeft > 0) + { + response.Dispose(); + triesLeft--; + _logger.LogWarning("Hit HTTP 503 from Azure DevOps. Waiting three seconds and trying again."); + await Task.Delay(TimeSpan.FromSeconds(3), cancellationToken); + continue; + } + + if (responseBody.Contains("It may have been deleted", StringComparison.OrdinalIgnoreCase) + || responseBody.Contains("not authorized to access this resource", StringComparison.OrdinalIgnoreCase) + || responseBody.Contains("cannot be added or updated for a test run which is in Completed state", StringComparison.OrdinalIgnoreCase) + || response.StatusCode == HttpStatusCode.Forbidden + || response.StatusCode == HttpStatusCode.Unauthorized) + { + response.Dispose(); + throw new TerminalError(responseBody); + } + + try + { + if (!string.IsNullOrWhiteSpace(s_lastSendContent)) + { + await _uploadClient.UploadAsync( + Encoding.UTF8.GetBytes(s_lastSendContent), + "__failed_azdo_request_content.json", + "text/plain; charset=UTF-8", + cancellationToken); + } + } + catch (Exception uploadException) + { + _logger.LogError(uploadException, "Failed to upload failed request payload."); + } + + response.Dispose(); + throw new AzureDevOpsReportingError($"Azure DevOps request failed with status code {(int)response.StatusCode}: {responseBody}"); + } + } + + private static async Task> ReadPublishedResultsAsync( + HttpResponseMessage response, + CancellationToken cancellationToken) + { + var content = await response.Content.ReadAsStringAsync(cancellationToken); + if (string.IsNullOrWhiteSpace(content)) + { + return Array.Empty(); + } + + using var document = JsonDocument.Parse(content); + var root = document.RootElement; + if (root.ValueKind == JsonValueKind.Array) + { + return root.EnumerateArray().Select(ParsePublishedResult).ToList(); + } + + if (root.TryGetProperty("value", out var value) && value.ValueKind == JsonValueKind.Array) + { + return value.EnumerateArray().Select(ParsePublishedResult).ToList(); + } + + return Array.Empty(); + } + + private static PublishedTestCaseResultReference ParsePublishedResult(JsonElement element) + { + var subResults = new List(); + if (element.TryGetProperty("subResults", out var subResultElement) && subResultElement.ValueKind == JsonValueKind.Array) + { + subResults.AddRange(subResultElement.EnumerateArray().Select(ParsePublishedSubResult)); + } + + return new PublishedTestCaseResultReference( + element.TryGetProperty("id", out var idElement) ? idElement.GetInt64() : -1, + subResults); + } + + private static PublishedSubResultReference ParsePublishedSubResult(JsonElement element) + { + var subResults = new List(); + if (element.TryGetProperty("subResults", out var subResultElement) && subResultElement.ValueKind == JsonValueKind.Array) + { + subResults.AddRange(subResultElement.EnumerateArray().Select(ParsePublishedSubResult)); + } + + return new PublishedSubResultReference( + element.TryGetProperty("id", out var idElement) ? idElement.GetInt64() : -1, + subResults); + } + + private static byte[] CreateCompressedCsv(IEnumerable rows) + { + var builder = new StringBuilder(); + foreach (var row in rows) + { + builder.Append(EscapeCsv(row.TestName)); + builder.Append(','); + builder.Append(EscapeCsv(row.ArgumentHash)); + builder.AppendLine(); + } + + return Compress(Encoding.UTF8.GetBytes(builder.ToString())); + } + + private static string EscapeCsv(string? value) + { + if (string.IsNullOrEmpty(value)) + { + return string.Empty; + } + + if (!value.Contains('"') && !value.Contains(',') && !value.Contains('\n') && !value.Contains('\r')) + { + return value; + } + + return $"\"{value.Replace("\"", "\"\"")}\""; + } + + private static byte[] Compress(ReadOnlySpan rawBytes) + { + using var target = new MemoryStream(); + using (var gzip = new GZipStream(target, CompressionLevel.SmallestSize, leaveOpen: true)) + { + gzip.Write(rawBytes); + } + + return target.ToArray(); + } + + private sealed record ConvertedResult(PublishedTestCase Converted, AggregatedResult Aggregated); + + private sealed record ChunkPair(PublishedSubResult Converted, AggregatedResult Aggregated); + + private sealed record TestListRow(string TestName, string? ArgumentHash); + + private sealed record TestRunAttachmentRequest(string FileName, string Stream); + + private sealed record CustomField(string FieldName, object Value); + + private sealed record PublishedTestCase + { + public long? Id { get; init; } + + public string TestCaseTitle { get; init; } = string.Empty; + + public string AutomatedTestName { get; init; } = string.Empty; + + public string AutomatedTestType { get; init; } = string.Empty; + + public string AutomatedTestStorage { get; init; } = string.Empty; + + public int Priority { get; init; } + + public double DurationInMs { get; init; } + + public string Outcome { get; init; } = string.Empty; + + public string State { get; init; } = string.Empty; + + public string Comment { get; init; } = string.Empty; + + public string? StackTrace { get; init; } + + public string? ErrorMessage { get; init; } + + public List? SubResults { get; init; } + + public string ResultGroupType { get; init; } = string.Empty; + + public List? CustomFields { get; init; } + } + + private sealed record PublishedSubResult + { + public long? Id { get; init; } + + public string Comment { get; init; } = string.Empty; + + public List? CustomFields { get; init; } + + public string DisplayName { get; init; } = string.Empty; + + public string Outcome { get; init; } = string.Empty; + + public double DurationInMs { get; init; } + + public string? StackTrace { get; init; } + + public string? ErrorMessage { get; init; } + + public List? SubResults { get; init; } + + public string ResultGroupType { get; init; } = string.Empty; + } + + private sealed record PublishedTestCaseResultReference(long Id, IReadOnlyList SubResults); + + private sealed record PublishedSubResultReference(long Id, IReadOnlyList SubResults); +} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsTestReporter.csproj b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsTestReporter.csproj new file mode 100644 index 00000000000..6db527d0af5 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsTestReporter.csproj @@ -0,0 +1,13 @@ + + + + $(BundledNETCoreAppTargetFramework) + enable + enable + + + + + + + diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs new file mode 100644 index 00000000000..b18b382b061 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs @@ -0,0 +1,92 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Text.Json; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; + +public sealed class PackingTestReporter : ITestReporter +{ + private const string ReportFileName = "__test_report.json"; + private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + + private readonly AzureDevOpsReportingParameters _azdoParameters; + private readonly ILogger _logger; + + public PackingTestReporter(AzureDevOpsReportingParameters azdoParameters, ILogger? logger = null) + { + _azdoParameters = azdoParameters; + _logger = logger.OrNull(); + } + + public async Task ReportResultsAsync(IReadOnlyList results, CancellationToken cancellationToken = default) + { + var filtered = (results ?? Array.Empty()) + .Where(static x => x is not null) + .ToList(); + + var serialized = new PackedTestReport(_azdoParameters, filtered); + var path = GetFileName(); + var directory = Path.GetDirectoryName(path); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + _logger.LogInformation("Packing {Count} test reports to '{Path}'", filtered.Count, path); + + await using (var saveFile = File.Create(path)) + { + await JsonSerializer.SerializeAsync(saveFile, serialized, SerializerOptions, cancellationToken); + await saveFile.FlushAsync(cancellationToken); + } + + _logger.LogInformation("Packed {Length} bytes", new FileInfo(path).Length); + } + + public static string GetFileName(HelixEnvironmentSettings? settings = null) + { + settings ??= HelixEnvironmentSettings.FromEnvironment(); + var root = settings.WorkitemWorkingDir; + if (string.IsNullOrWhiteSpace(root)) + { + root = Environment.CurrentDirectory; + } + + return Path.Combine(root, ReportFileName); + } + + public static async Task<(AzureDevOpsReportingParameters Parameters, IReadOnlyList Results)?> UnpackResultsAsync( + ILogger? logger = null, + CancellationToken cancellationToken = default) + { + var effectiveLogger = logger.OrNull(); + var settings = HelixEnvironmentSettings.FromEnvironment(); + var path = GetFileName(settings); + + if (!File.Exists(path) && !string.IsNullOrWhiteSpace(settings.WorkitemPayloadDir)) + { + path = Path.Combine(settings.WorkitemPayloadDir, ReportFileName); + } + + if (!File.Exists(path)) + { + return null; + } + + effectiveLogger.LogInformation("Unpacking {Length} bytes from '{Path}'", new FileInfo(path).Length, path); + + await using var saveFile = File.OpenRead(path); + var serialized = await JsonSerializer.DeserializeAsync(saveFile, SerializerOptions, cancellationToken); + if (serialized is null) + { + effectiveLogger.LogError("Unpacked tests were null or invalid."); + return null; + } + + effectiveLogger.LogInformation("Unpacked {Count} test reports", serialized.Results.Count); + return (serialized.AzdoParameters, serialized.Results); + } +} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/ResultAggregator.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/ResultAggregator.cs new file mode 100644 index 00000000000..6447caedfff --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/ResultAggregator.cs @@ -0,0 +1,346 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; + +public enum AggregationType +{ + Single = 0, + Rerun = 1, + DataDriven = 2, +} + +public sealed class AggregatedResult +{ + public AggregatedResult( + AggregationType aggregationType, + string name, + double durationSeconds, + string result, + IReadOnlyList? subResults = null, + IReadOnlyList? attachments = null, + string? failureMessage = null, + string? stackTrace = null, + string? skipReason = null, + bool isFlaky = false, + int? attemptId = null) + { + AggregationType = aggregationType; + Name = name ?? string.Empty; + DurationSeconds = durationSeconds; + Result = result ?? string.Empty; + SubResults = subResults ?? Array.Empty(); + Attachments = attachments ?? Array.Empty(); + FailureMessage = failureMessage ?? skipReason; + StackTrace = stackTrace; + IsFlaky = isFlaky; + AttemptId = attemptId; + } + + public AggregationType AggregationType { get; } + + public string Name { get; } + + public double DurationSeconds { get; } + + public string Result { get; } + + public IReadOnlyList Attachments { get; } + + public IReadOnlyList SubResults { get; } + + public string? FailureMessage { get; } + + public string? StackTrace { get; } + + public int? AttemptId { get; } + + public bool IsFlaky { get; } +} + +public sealed class ResultAggregator +{ + public IReadOnlyList Aggregate(IEnumerable>? results) + { + if (results is null) + { + return Array.Empty(); + } + + string GetResult(TestResult test) + { + if (test.Ignored && string.Equals(test.Result, "Fail", StringComparison.Ordinal)) + { + return "NotApplicable"; + } + + return test.Result switch + { + "Pass" => "Passed", + "Fail" => "Failed", + "Skip" => "NotExecuted", + _ => "None", + }; + } + + static string ParseBasicName(string name) + { + var separatorIndex = name.IndexOf('('); + return separatorIndex >= 0 ? name[..separatorIndex] : name; + } + + AggregatedResult CreateResultFromTest(TestResult result, int? attemptId = null) + { + return new AggregatedResult( + AggregationType.Single, + attemptId is null ? result.Name : $"Attempt #{attemptId} - {result.Name}", + result.DurationSeconds, + GetResult(result), + Array.Empty(), + result.Attachments, + result.FailureMessage, + result.StackTrace, + result.SkipReason, + attemptId: attemptId); + } + + string GetDataDrivenResult(IReadOnlyList groupedResults) + { + if (groupedResults.Count == 0) + { + return "None"; + } + + if (groupedResults.Any(static r => !r.Ignored && r.Result == "Fail")) + { + return "Failed"; + } + + if (groupedResults.Any(static r => r.Result == "Pass")) + { + return "Passed"; + } + + return GetResult(groupedResults[0]); + } + + (bool IsFlaky, string Outcome) GetRerunResult(IReadOnlyList groupedResults) + { + if (groupedResults.Count == 0) + { + return (false, "None"); + } + + var anyPass = groupedResults.Any(static r => r.Result == "Pass"); + var anyFail = groupedResults.Any(static r => !r.Ignored && r.Result == "Fail"); + var isFlaky = anyPass && anyFail; + + if (anyPass) + { + return (isFlaky, "Passed"); + } + + if (anyFail) + { + return (isFlaky, "Failed"); + } + + return (false, GetResult(groupedResults[0])); + } + + AggregatedResult ProcessNamedTest(string name, IReadOnlyList> byIterationThenName) + { + if (byIterationThenName.Count == 1) + { + var singleRun = byIterationThenName[0]; + if (singleRun.Count == 1) + { + return CreateResultFromTest(singleRun[0]); + } + + return new AggregatedResult( + AggregationType.DataDriven, + name, + singleRun.Sum(static r => r.DurationSeconds), + GetDataDrivenResult(singleRun), + singleRun.Select(testResult => CreateResultFromTest(testResult)).ToList()); + } + + var hasDataDriven = byIterationThenName.Any(static x => x.Count > 1); + + if (hasDataDriven) + { + var dataDrivenByFullName = new Dictionary>(StringComparer.Ordinal); + foreach (var iteration in byIterationThenName) + { + foreach (var test in iteration) + { + if (!dataDrivenByFullName.TryGetValue(test.Name, out var list)) + { + list = new List(); + dataDrivenByFullName[test.Name] = list; + } + + list.Add(test); + } + } + + var subResults = new List(); + double totalDuration = 0; + + foreach (var pair in dataDrivenByFullName) + { + var dataDrivenTests = pair.Value; + if (dataDrivenTests.Count == 1) + { + subResults.Add(CreateResultFromTest(dataDrivenTests[0])); + totalDuration += dataDrivenTests[0].DurationSeconds; + continue; + } + + var (isFlaky, aggregateResult) = GetRerunResult(dataDrivenTests); + var partialDuration = dataDrivenTests.Sum(static r => r.DurationSeconds); + totalDuration += partialDuration; + subResults.Add(new AggregatedResult( + AggregationType.Rerun, + pair.Key, + partialDuration, + aggregateResult, + dataDrivenTests.Select((r, index) => CreateResultFromTest(r, index + 1)).ToList(), + isFlaky: isFlaky)); + } + + var aggregateOutcome = "Inconclusive"; + if (dataDrivenByFullName.Values.Any(rerunSet => rerunSet.Where(static r => !r.Ignored).All(static r => r.Result == "Fail"))) + { + aggregateOutcome = "Failed"; + } + else if (dataDrivenByFullName.Values.All(rerunSet => rerunSet.All(static r => r.Result == "Skip"))) + { + aggregateOutcome = "NotExecuted"; + } + else if (dataDrivenByFullName.Values.All(rerunSet => rerunSet.Any(static r => r.Result == "Pass"))) + { + aggregateOutcome = "Passed"; + } + + return new AggregatedResult( + AggregationType.DataDriven, + name, + totalDuration, + aggregateOutcome, + subResults); + } + + var reruns = byIterationThenName.Select(static run => run[0]).ToList(); + var (rerunIsFlaky, rerunOutcome) = GetRerunResult(reruns); + return new AggregatedResult( + AggregationType.Rerun, + name, + reruns.Sum(static r => r.DurationSeconds), + rerunOutcome, + reruns.Select((r, index) => CreateResultFromTest(r, index + 1)).ToList(), + failureMessage: reruns[0].FailureMessage, + stackTrace: reruns[0].StackTrace, + isFlaky: rerunIsFlaky); + } + + AggregatedResult ReduceSimpleResult(AggregatedResult result) + { + if (result.SubResults.Count == 0) + { + return result; + } + + if (result.AggregationType == AggregationType.Rerun) + { + var distinctOutcomes = result.SubResults + .Select(static r => r.Result) + .Distinct(StringComparer.Ordinal) + .Count(); + + if (distinctOutcomes == 1) + { + var single = result.SubResults[0]; + return new AggregatedResult( + AggregationType.Single, + result.Name, + single.DurationSeconds, + single.Result, + attachments: single.Attachments, + failureMessage: single.FailureMessage, + stackTrace: single.StackTrace); + } + + return result; + } + + return new AggregatedResult( + result.AggregationType, + result.Name, + result.DurationSeconds, + result.Result, + result.SubResults.Select(ReduceSimpleResult).ToList(), + result.Attachments, + result.FailureMessage, + result.StackTrace, + isFlaky: result.IsFlaky, + attemptId: result.AttemptId); + } + + var partials = new List>>(); + foreach (var resultSet in results) + { + var perAttempt = new Dictionary>(StringComparer.Ordinal); + foreach (var result in resultSet) + { + var basicName = ParseBasicName(result.Name); + if (!perAttempt.TryGetValue(basicName, out var list)) + { + list = new List(); + perAttempt[basicName] = list; + } + + list.Add(result); + } + + partials.Add(perAttempt); + } + + if (partials.Count == 0 || partials[0].Count == 0) + { + return Array.Empty(); + } + + var aggregate = new List(); + foreach (var run in partials) + { + foreach (var pair in run.ToList()) + { + if (!run.Remove(pair.Key, out var currentSet)) + { + continue; + } + + var fullSet = new List> { currentSet }; + foreach (var otherRun in partials) + { + if (ReferenceEquals(otherRun, run)) + { + continue; + } + + if (otherRun.Remove(pair.Key, out var otherSet)) + { + fullSet.Add(otherSet); + } + } + + aggregate.Add(ProcessNamedTest(pair.Key, fullSet)); + } + } + + return aggregate.Select(ReduceSimpleResult).ToList(); + } +} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs new file mode 100644 index 00000000000..4c677fb556e --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs @@ -0,0 +1,183 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; + +public sealed record TestResultAttachment(string Name, string Text); + +public sealed class TestResult +{ + public TestResult( + string name, + string kind, + string typeName, + string method, + double durationSeconds, + string result, + string? exceptionType, + string? failureMessage, + string? stackTrace, + string? skipReason, + IReadOnlyList? attachments = null) + { + Name = name ?? string.Empty; + Kind = kind ?? string.Empty; + TypeName = typeName ?? string.Empty; + Method = method ?? string.Empty; + DurationSeconds = durationSeconds; + Result = result ?? string.Empty; + ExceptionType = exceptionType; + FailureMessage = failureMessage; + StackTrace = stackTrace; + SkipReason = skipReason; + Attachments = attachments ?? Array.Empty(); + } + + public string Name { get; } + + public string Kind { get; } + + public string TypeName { get; } + + public string Method { get; } + + public double DurationSeconds { get; } + + public string Result { get; } + + public string? ExceptionType { get; } + + public string? FailureMessage { get; } + + public string? StackTrace { get; } + + public string? SkipReason { get; } + + public IReadOnlyList Attachments { get; } + + public bool Ignored { get; set; } +} + +public interface ITestReporter +{ + Task ReportResultsAsync(IReadOnlyList results, CancellationToken cancellationToken = default); +} + +public sealed record AzureDevOpsReportingParameters(Uri CollectionUri, string TeamProject, string TestRunId, string AccessToken); + +public sealed record PackedTestReport(AzureDevOpsReportingParameters AzdoParameters, IReadOnlyList Results); + +public sealed class HelixEnvironmentSettings +{ + public string? CorrelationId { get; init; } + + public string? WorkItemId { get; init; } + + public string? WorkItemFriendlyName { get; init; } + + public string? WorkitemWorkingDir { get; init; } + + public string? WorkitemPayloadDir { get; init; } + + public static HelixEnvironmentSettings FromEnvironment() + { + return new HelixEnvironmentSettings + { + CorrelationId = Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID"), + WorkItemId = Environment.GetEnvironmentVariable("HELIX_WORKITEM_ID"), + WorkItemFriendlyName = Environment.GetEnvironmentVariable("HELIX_WORKITEM_FRIENDLYNAME"), + WorkitemWorkingDir = Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT"), + WorkitemPayloadDir = Environment.GetEnvironmentVariable("HELIX_WORKITEM_PAYLOAD"), + }; + } +} + +public interface IEventClient +{ + Task SendAsync(object payload, CancellationToken cancellationToken = default); + + Task ErrorAsync( + HelixEnvironmentSettings settings, + string errorType, + string message, + string? logUri = null, + CancellationToken cancellationToken = default); +} + +public interface IUploadClient +{ + Task UploadAsync( + Stream file, + string name, + string contentType = "application/octet-stream", + CancellationToken cancellationToken = default); + + Task UploadAsync( + ReadOnlyMemory fileBytes, + string name, + string contentType = "application/octet-stream", + CancellationToken cancellationToken = default); +} + +public sealed class NullEventClient : IEventClient +{ + public static readonly NullEventClient Instance = new(); + + private NullEventClient() + { + } + + public Task SendAsync(object payload, CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } + + public Task ErrorAsync( + HelixEnvironmentSettings settings, + string errorType, + string message, + string? logUri = null, + CancellationToken cancellationToken = default) + { + return Task.CompletedTask; + } +} + +public sealed class NullUploadClient : IUploadClient +{ + public static readonly NullUploadClient Instance = new(); + + private NullUploadClient() + { + } + + public Task UploadAsync( + Stream file, + string name, + string contentType = "application/octet-stream", + CancellationToken cancellationToken = default) + { + return Task.FromResult($"memory://{name}"); + } + + public Task UploadAsync( + ReadOnlyMemory fileBytes, + string name, + string contentType = "application/octet-stream", + CancellationToken cancellationToken = default) + { + return Task.FromResult($"memory://{name}"); + } +} + +public static class LoggerFactoryExtensions +{ + public static ILogger OrNull(this ILogger? logger) + { + return logger ?? NullLogger.Instance; + } +} From 095619376fd37bd8d510e58547a2386919363149 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 21 Apr 2026 12:05:58 +0200 Subject: [PATCH 04/66] Tidy up the reporter --- Arcade.slnx | 2 +- .../AzureDevOpsResultPublisher.cs | 163 +++++++----------- ...tNet.Helix.AzureDevOpsTestReporter.csproj} | 0 .../Model/AzureDevOpsReportingError.cs | 8 + .../Model/PackedTestReport.cs | 6 + .../Model/TerminalError.cs | 8 + .../Model/TestResult.cs | 42 +++++ .../Model/TestResultAttachment.cs | 6 + .../Model/UploadResult.cs | 11 ++ .../Model/UploadResultExtensions.cs | 12 ++ .../PackingTestReporter.cs | 37 ++-- .../ResultAggregator.cs | 133 +++++++------- .../TestReportingModels.cs | 86 +-------- 13 files changed, 238 insertions(+), 276 deletions(-) rename src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/{AzureDevOpsTestReporter.csproj => Microsoft.DotNet.Helix.AzureDevOpsTestReporter.csproj} (100%) create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/AzureDevOpsReportingError.cs create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/PackedTestReport.cs create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TerminalError.cs create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResult.cs create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResultAttachment.cs create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResult.cs create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResultExtensions.cs diff --git a/Arcade.slnx b/Arcade.slnx index 05849deea49..975b61a3e5d 100644 --- a/Arcade.slnx +++ b/Arcade.slnx @@ -5,7 +5,7 @@ - + diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs index 5347ea571ba..3c1228f5c50 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs @@ -1,6 +1,5 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. using System.IO.Compression; using System.Net; @@ -8,45 +7,15 @@ using System.Security.Cryptography; using System.Text; using System.Text.Json; +using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; using Microsoft.Extensions.Logging; namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; -public enum UploadResult -{ - Success = 1, - UnknownError = 2, - TerminalError = 3, -} - -public static class UploadResultExtensions -{ - public static UploadResult Aggregate(this UploadResult value, UploadResult other) - { - return (UploadResult)Math.Max((int)value, (int)other); - } -} - -public sealed class AzureDevOpsReportingError : Exception -{ - public AzureDevOpsReportingError(string message) - : base(message) - { - } -} - -internal sealed class TerminalError : Exception -{ - public TerminalError(string message) - : base(message) - { - } -} - public sealed class AzureDevOpsResultPublisher { private const int TestListBuckets = 32; - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web) + private static readonly JsonSerializerOptions s_serializerOptions = new(JsonSerializerDefaults.Web) { WriteIndented = false, }; @@ -80,7 +49,7 @@ public async Task TryUploadAsync(IEnumerable res { try { - await ProcessAsync(results.ToList(), cancellationToken); + await ProcessAsync([.. results], cancellationToken); return UploadResult.Success; } catch (TerminalError ex) @@ -100,9 +69,9 @@ private async Task ProcessAsync(IReadOnlyList testList, Cancel var converted = ConvertResults(testList).ToList(); var hotPathTests = new List(); - foreach (var batch in Batch(converted, 1000, static t => Size(t.Converted))) + foreach (List batch in Batch(converted, 1000, static t => Size(t.Converted))) { - var publishedTests = await PublishResultsAsync(batch, cancellationToken); + IReadOnlyList publishedTests = await PublishResultsAsync(batch, cancellationToken); hotPathTests.AddRange(publishedTests); _logger.LogInformation("Uploaded {Count} results", publishedTests.Count); } @@ -130,28 +99,28 @@ private async Task SendMetadataAsync( void ProcessResultForMetadata(AggregatedResult result) { - resultCounts[result.Result] = resultCounts.TryGetValue(result.Result, out var count) ? count + 1 : 1; + resultCounts[result.Result] = resultCounts.TryGetValue(result.Result, out int count) ? count + 1 : 1; if (!string.Equals(result.Result, "Passed", StringComparison.Ordinal)) { return; } - var name = result.Name; + string name = result.Name; string? argumentHash = null; - var partitionKey = name; - var parenthesisIndex = name.IndexOf('('); + string partitionKey = name; + int parenthesisIndex = name.IndexOf('('); if (parenthesisIndex >= 0) { - var argumentList = name[(parenthesisIndex + 1)..].TrimEnd(')'); + string argumentList = name[(parenthesisIndex + 1)..].TrimEnd(')'); name = name[..parenthesisIndex]; argumentHash = Convert.ToBase64String(SHA1.HashData(Encoding.UTF8.GetBytes(argumentList))); partitionKey = name + argumentHash; } - var bucket = SHA1.HashData(Encoding.UTF8.GetBytes(partitionKey))[0] % TestListBuckets; - if (!partitionedResults.TryGetValue(bucket, out var testNames)) + int bucket = SHA1.HashData(Encoding.UTF8.GetBytes(partitionKey))[0] % TestListBuckets; + if (!partitionedResults.TryGetValue(bucket, out List? testNames)) { - testNames = new List(); + testNames = []; partitionedResults[bucket] = testNames; } @@ -162,7 +131,7 @@ void ProcessTestForMetadata(AggregatedResult result) { if (result.AggregationType == AggregationType.DataDriven && result.SubResults.Count > 0) { - foreach (var subResult in result.SubResults) + foreach (AggregatedResult subResult in result.SubResults) { ProcessTestForMetadata(subResult); } @@ -173,16 +142,16 @@ void ProcessTestForMetadata(AggregatedResult result) } } - foreach (var result in allTestResults) + foreach (AggregatedResult result in allTestResults) { ProcessTestForMetadata(result); } var uploadedUrls = new Dictionary(); - foreach (var (key, testNames) in partitionedResults) + foreach ((int key, List? testNames) in partitionedResults) { - var csvBytes = CreateCompressedCsv(testNames); - var fileName = $"{Guid.NewGuid():N}.csv.gz"; + byte[] csvBytes = CreateCompressedCsv(testNames); + string fileName = $"{Guid.NewGuid():N}.csv.gz"; uploadedUrls[key] = await _uploadClient.UploadAsync(csvBytes, fileName, "application/gzip", cancellationToken); } @@ -195,10 +164,10 @@ void ProcessTestForMetadata(AggregatedResult result) result_counts = resultCounts, }; - var rawBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(dataModel, SerializerOptions)); - var compressedBytes = Compress(rawBytes); - var base64Data = Convert.ToBase64String(compressedBytes); - var fileNameBase = $"__helix_metadata_{Guid.NewGuid():N}.json.gz"; + byte[] rawBytes = Encoding.UTF8.GetBytes(JsonSerializer.Serialize(dataModel, s_serializerOptions)); + byte[] compressedBytes = Compress(rawBytes); + string base64Data = Convert.ToBase64String(compressedBytes); + string fileNameBase = $"__helix_metadata_{Guid.NewGuid():N}.json.gz"; await SendWithRetryAsync( HttpMethod.Post, @@ -206,7 +175,7 @@ await SendWithRetryAsync( new TestRunAttachmentRequest(fileNameBase, base64Data), cancellationToken); - var metadataUrl = await _uploadClient.UploadAsync(compressedBytes, fileNameBase, "application/gzip", cancellationToken); + string metadataUrl = await _uploadClient.UploadAsync(compressedBytes, fileNameBase, "application/gzip", cancellationToken); await _eventClient.SendAsync( new { @@ -225,25 +194,25 @@ private async Task> PublishResultsAsync( var testCaseResults = converted.Select(static c => c.Converted).ToList(); var originalList = converted.Select(static c => c.Aggregated).ToList(); - using var response = await SendWithRetryAsync( + using HttpResponseMessage response = await SendWithRetryAsync( HttpMethod.Post, $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results?api-version=7.1-preview.6", testCaseResults, cancellationToken); - var publishedResults = await ReadPublishedResultsAsync(response, cancellationToken); + IReadOnlyList publishedResults = await ReadPublishedResultsAsync(response, cancellationToken); if (publishedResults.Count == 0) { _logger.LogWarning("The test run appears to have been closed, aborting test result uploads."); - return Array.Empty(); + return []; } var hotPathTests = new List(); - foreach (var triplet in publishedResults.Zip(originalList, testCaseResults)) + foreach ((PublishedTestCaseResultReference First, AggregatedResult Second, PublishedTestCase Third) triplet in publishedResults.Zip(originalList, testCaseResults)) { - var published = triplet.First; - var original = triplet.Second; - var testCase = triplet.Third; + PublishedTestCaseResultReference published = triplet.First; + AggregatedResult original = triplet.Second; + PublishedTestCase testCase = triplet.Third; if (published.Id == -1) { @@ -252,7 +221,7 @@ private async Task> PublishResultsAsync( } testCase = testCase with { Id = published.Id }; - var addedTest = false; + bool addedTest = false; void AddToHotPath() { @@ -291,9 +260,9 @@ async Task IterateSubResultsAsync( return; } - foreach (var subTriplet in publishedSubResults.Zip(originalSubResults, (publishedSubResult, originalSubResult) => (publishedSubResult, originalSubResult))) + foreach ((PublishedSubResultReference publishedSubResult, AggregatedResult originalSubResult) subTriplet in publishedSubResults.Zip(originalSubResults, (publishedSubResult, originalSubResult) => (publishedSubResult, originalSubResult))) { - foreach (var attachment in subTriplet.originalSubResult.Attachments) + foreach (TestResultAttachment attachment in subTriplet.originalSubResult.Attachments) { await SendAttachmentAsync(attachment, testId, subTriplet.publishedSubResult.Id, cancellationToken); } @@ -302,7 +271,7 @@ async Task IterateSubResultsAsync( } } - foreach (var attachment in original.Attachments) + foreach (TestResultAttachment attachment in original.Attachments) { await SendAttachmentAsync(attachment, published.Id, null, cancellationToken); } @@ -323,18 +292,18 @@ private async Task SendAttachmentAsync( attachment.Name, Convert.ToBase64String(Encoding.UTF8.GetBytes(attachment.Text))); - var path = subResultId is long subId + string path = subResultId is long subId ? $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results/{testId}/subresults/{subId}/attachments?api-version=7.1-preview.1" : $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results/{testId}/attachments?api-version=7.1-preview.1"; - using var response = await SendWithRetryAsync(HttpMethod.Post, path, request, cancellationToken); + using HttpResponseMessage response = await SendWithRetryAsync(HttpMethod.Post, path, request, cancellationToken); _ = response; } private IEnumerable ConvertResults(IEnumerable results) { var settings = HelixEnvironmentSettings.FromEnvironment(); - var comment = JsonSerializer.Serialize(new + string? comment = JsonSerializer.Serialize(new { HelixJobId = settings.CorrelationId, HelixWorkItemName = settings.WorkItemFriendlyName, @@ -373,7 +342,7 @@ PublishedSubResult ConvertToSubTest(AggregatedResult result) DurationInMs = result.DurationSeconds * 1000.0, StackTrace = result.StackTrace, ErrorMessage = result.FailureMessage, - SubResults = result.SubResults.Count == 0 ? null : result.SubResults.Select(ConvertToSubTest).ToList(), + SubResults = result.SubResults.Count == 0 ? null : [.. result.SubResults.Select(ConvertToSubTest)], ResultGroupType = GetResultGroupType(result.AggregationType), }; } @@ -405,7 +374,7 @@ ConvertedResult ConvertResult(AggregatedResult result) Comment = comment ?? string.Empty, StackTrace = result.StackTrace, ErrorMessage = result.FailureMessage, - SubResults = result.SubResults.Count == 0 ? null : result.SubResults.Select(ConvertToSubTest).ToList(), + SubResults = result.SubResults.Count == 0 ? null : [.. result.SubResults.Select(ConvertToSubTest)], ResultGroupType = GetResultGroupType(result.AggregationType), CustomFields = customFields, }, @@ -413,9 +382,9 @@ ConvertedResult ConvertResult(AggregatedResult result) } var converted = results.Select(ConvertResult).ToList(); - foreach (var result in converted) + foreach (ConvertedResult? result in converted) { - foreach (var chunk in Chunk(result, 950)) + foreach (ConvertedResult chunk in Chunk(result, 950)) { yield return chunk; } @@ -430,19 +399,19 @@ private static IEnumerable Chunk(ConvertedResult test, int limi yield break; } - var zippedSubTests = (test.Converted.SubResults ?? new List()) + IEnumerable zippedSubTests = (test.Converted.SubResults ?? []) .Zip(test.Aggregated.SubResults, (converted, aggregated) => new ChunkPair(converted, aggregated)); - foreach (var zippedBatch in Batch(zippedSubTests, limit, static pair => Size(pair.Converted))) + foreach (List zippedBatch in Batch(zippedSubTests, limit, static pair => Size(pair.Converted))) { yield return new ConvertedResult( - test.Converted with { SubResults = zippedBatch.Select(static x => x.Converted).ToList(), Id = null }, + test.Converted with { SubResults = [.. zippedBatch.Select(static x => x.Converted)], Id = null }, new AggregatedResult( test.Aggregated.AggregationType, test.Aggregated.Name, test.Aggregated.DurationSeconds, test.Aggregated.Result, - zippedBatch.Select(static x => x.Aggregated).ToList(), + [.. zippedBatch.Select(static x => x.Aggregated)], test.Aggregated.Attachments, test.Aggregated.FailureMessage, test.Aggregated.StackTrace, @@ -464,11 +433,11 @@ private static int Size(PublishedSubResult test) private static IEnumerable> Batch(IEnumerable items, int limit, Func getSize) { var currentBatch = new List(); - var currentSize = 0; + int currentSize = 0; - foreach (var item in items) + foreach (T? item in items) { - var size = getSize(item); + int size = getSize(item); if (size > limit) { throw new InvalidOperationException("Cannot split a result larger than the batching limit."); @@ -477,7 +446,7 @@ private static IEnumerable> Batch(IEnumerable items, int limit, Fu if (currentSize + size > limit && currentBatch.Count > 0) { yield return currentBatch; - currentBatch = new List(); + currentBatch = []; currentSize = 0; } @@ -498,7 +467,7 @@ private static HttpClient CreateHttpClient(string accessToken) if (!string.IsNullOrWhiteSpace(accessToken)) { - var basicToken = Convert.ToBase64String(Encoding.ASCII.GetBytes($":{accessToken}")); + string basicToken = Convert.ToBase64String(Encoding.ASCII.GetBytes($":{accessToken}")); client.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", basicToken); } @@ -511,8 +480,8 @@ private async Task SendWithRetryAsync( object? payload, CancellationToken cancellationToken) { - var triesLeft = 10; - var body = payload is null ? null : JsonSerializer.Serialize(payload, SerializerOptions); + int triesLeft = 10; + string? body = payload is null ? null : JsonSerializer.Serialize(payload, s_serializerOptions); if (!string.IsNullOrEmpty(body)) { s_lastSendContent = body; @@ -526,13 +495,13 @@ private async Task SendWithRetryAsync( request.Content = new StringContent(body, Encoding.UTF8, "application/json"); } - var response = await _httpClient.SendAsync(request, cancellationToken); + HttpResponseMessage response = await _httpClient.SendAsync(request, cancellationToken); if (response.IsSuccessStatusCode) { return response; } - var responseBody = await response.Content.ReadAsStringAsync(cancellationToken); + string responseBody = await response.Content.ReadAsStringAsync(cancellationToken); if (response.StatusCode == HttpStatusCode.ServiceUnavailable && triesLeft > 0) { response.Dispose(); @@ -577,57 +546,57 @@ private static async Task> ReadP HttpResponseMessage response, CancellationToken cancellationToken) { - var content = await response.Content.ReadAsStringAsync(cancellationToken); + string content = await response.Content.ReadAsStringAsync(cancellationToken); if (string.IsNullOrWhiteSpace(content)) { - return Array.Empty(); + return []; } using var document = JsonDocument.Parse(content); - var root = document.RootElement; + JsonElement root = document.RootElement; if (root.ValueKind == JsonValueKind.Array) { - return root.EnumerateArray().Select(ParsePublishedResult).ToList(); + return [.. root.EnumerateArray().Select(ParsePublishedResult)]; } - if (root.TryGetProperty("value", out var value) && value.ValueKind == JsonValueKind.Array) + if (root.TryGetProperty("value", out JsonElement value) && value.ValueKind == JsonValueKind.Array) { - return value.EnumerateArray().Select(ParsePublishedResult).ToList(); + return [.. value.EnumerateArray().Select(ParsePublishedResult)]; } - return Array.Empty(); + return []; } private static PublishedTestCaseResultReference ParsePublishedResult(JsonElement element) { var subResults = new List(); - if (element.TryGetProperty("subResults", out var subResultElement) && subResultElement.ValueKind == JsonValueKind.Array) + if (element.TryGetProperty("subResults", out JsonElement subResultElement) && subResultElement.ValueKind == JsonValueKind.Array) { subResults.AddRange(subResultElement.EnumerateArray().Select(ParsePublishedSubResult)); } return new PublishedTestCaseResultReference( - element.TryGetProperty("id", out var idElement) ? idElement.GetInt64() : -1, + element.TryGetProperty("id", out JsonElement idElement) ? idElement.GetInt64() : -1, subResults); } private static PublishedSubResultReference ParsePublishedSubResult(JsonElement element) { var subResults = new List(); - if (element.TryGetProperty("subResults", out var subResultElement) && subResultElement.ValueKind == JsonValueKind.Array) + if (element.TryGetProperty("subResults", out JsonElement subResultElement) && subResultElement.ValueKind == JsonValueKind.Array) { subResults.AddRange(subResultElement.EnumerateArray().Select(ParsePublishedSubResult)); } return new PublishedSubResultReference( - element.TryGetProperty("id", out var idElement) ? idElement.GetInt64() : -1, + element.TryGetProperty("id", out JsonElement idElement) ? idElement.GetInt64() : -1, subResults); } private static byte[] CreateCompressedCsv(IEnumerable rows) { var builder = new StringBuilder(); - foreach (var row in rows) + foreach (TestListRow row in rows) { builder.Append(EscapeCsv(row.TestName)); builder.Append(','); diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsTestReporter.csproj b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Microsoft.DotNet.Helix.AzureDevOpsTestReporter.csproj similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsTestReporter.csproj rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Microsoft.DotNet.Helix.AzureDevOpsTestReporter.csproj diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/AzureDevOpsReportingError.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/AzureDevOpsReportingError.cs new file mode 100644 index 00000000000..2d541ab043d --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/AzureDevOpsReportingError.cs @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; + +public sealed class AzureDevOpsReportingError(string message) : Exception(message) +{ +} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/PackedTestReport.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/PackedTestReport.cs new file mode 100644 index 00000000000..ac93f7efb25 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/PackedTestReport.cs @@ -0,0 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; + +public sealed record PackedTestReport(AzureDevOpsReportingParameters AzdoParameters, IReadOnlyList Results); diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TerminalError.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TerminalError.cs new file mode 100644 index 00000000000..46fc04f2f6d --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TerminalError.cs @@ -0,0 +1,8 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; + +internal sealed class TerminalError(string message) : Exception(message) +{ +} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResult.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResult.cs new file mode 100644 index 00000000000..17d29681bb2 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResult.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; + +public sealed class TestResult( + string name, + string kind, + string typeName, + string method, + double durationSeconds, + string result, + string? exceptionType, + string? failureMessage, + string? stackTrace, + string? skipReason, + IReadOnlyList? attachments = null) +{ + public string Name { get; } = name ?? string.Empty; + + public string Kind { get; } = kind ?? string.Empty; + + public string TypeName { get; } = typeName ?? string.Empty; + + public string Method { get; } = method ?? string.Empty; + + public double DurationSeconds { get; } = durationSeconds; + + public string Result { get; } = result ?? string.Empty; + + public string? ExceptionType { get; } = exceptionType; + + public string? FailureMessage { get; } = failureMessage; + + public string? StackTrace { get; } = stackTrace; + + public string? SkipReason { get; } = skipReason; + + public IReadOnlyList Attachments { get; } = attachments ?? []; + + public bool Ignored { get; set; } +} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResultAttachment.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResultAttachment.cs new file mode 100644 index 00000000000..afe26a86688 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResultAttachment.cs @@ -0,0 +1,6 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; + +public sealed record TestResultAttachment(string Name, string Text); diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResult.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResult.cs new file mode 100644 index 00000000000..33ea210a70a --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResult.cs @@ -0,0 +1,11 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; + +public enum UploadResult +{ + Success = 1, + UnknownError = 2, + TerminalError = 3, +} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResultExtensions.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResultExtensions.cs new file mode 100644 index 00000000000..cef847051e6 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResultExtensions.cs @@ -0,0 +1,12 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; + +public static class UploadResultExtensions +{ + public static UploadResult Aggregate(this UploadResult value, UploadResult other) + { + return (UploadResult)Math.Max((int)value, (int)other); + } +} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs index b18b382b061..057afec4ddc 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs @@ -1,35 +1,30 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. using System.Text.Json; +using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; using Microsoft.Extensions.Logging; namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; -public sealed class PackingTestReporter : ITestReporter +public sealed class PackingTestReporter(AzureDevOpsReportingParameters azdoParameters, ILogger? logger = null) + : ITestReporter { private const string ReportFileName = "__test_report.json"; - private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web); + private static readonly JsonSerializerOptions s_serializerOptions = new(JsonSerializerDefaults.Web); - private readonly AzureDevOpsReportingParameters _azdoParameters; - private readonly ILogger _logger; - - public PackingTestReporter(AzureDevOpsReportingParameters azdoParameters, ILogger? logger = null) - { - _azdoParameters = azdoParameters; - _logger = logger.OrNull(); - } + private readonly AzureDevOpsReportingParameters _azdoParameters = azdoParameters; + private readonly ILogger _logger = logger.OrNull(); public async Task ReportResultsAsync(IReadOnlyList results, CancellationToken cancellationToken = default) { - var filtered = (results ?? Array.Empty()) + var filtered = (results ?? []) .Where(static x => x is not null) .ToList(); var serialized = new PackedTestReport(_azdoParameters, filtered); - var path = GetFileName(); - var directory = Path.GetDirectoryName(path); + string path = GetFileName(); + string? directory = Path.GetDirectoryName(path); if (!string.IsNullOrEmpty(directory)) { Directory.CreateDirectory(directory); @@ -37,9 +32,9 @@ public async Task ReportResultsAsync(IReadOnlyList results, Cancella _logger.LogInformation("Packing {Count} test reports to '{Path}'", filtered.Count, path); - await using (var saveFile = File.Create(path)) + await using (FileStream saveFile = File.Create(path)) { - await JsonSerializer.SerializeAsync(saveFile, serialized, SerializerOptions, cancellationToken); + await JsonSerializer.SerializeAsync(saveFile, serialized, s_serializerOptions, cancellationToken); await saveFile.FlushAsync(cancellationToken); } @@ -49,7 +44,7 @@ public async Task ReportResultsAsync(IReadOnlyList results, Cancella public static string GetFileName(HelixEnvironmentSettings? settings = null) { settings ??= HelixEnvironmentSettings.FromEnvironment(); - var root = settings.WorkitemWorkingDir; + string? root = settings.WorkitemWorkingDir; if (string.IsNullOrWhiteSpace(root)) { root = Environment.CurrentDirectory; @@ -62,9 +57,9 @@ public static string GetFileName(HelixEnvironmentSettings? settings = null) ILogger? logger = null, CancellationToken cancellationToken = default) { - var effectiveLogger = logger.OrNull(); + ILogger effectiveLogger = logger.OrNull(); var settings = HelixEnvironmentSettings.FromEnvironment(); - var path = GetFileName(settings); + string path = GetFileName(settings); if (!File.Exists(path) && !string.IsNullOrWhiteSpace(settings.WorkitemPayloadDir)) { @@ -78,8 +73,8 @@ public static string GetFileName(HelixEnvironmentSettings? settings = null) effectiveLogger.LogInformation("Unpacking {Length} bytes from '{Path}'", new FileInfo(path).Length, path); - await using var saveFile = File.OpenRead(path); - var serialized = await JsonSerializer.DeserializeAsync(saveFile, SerializerOptions, cancellationToken); + await using FileStream saveFile = File.OpenRead(path); + PackedTestReport? serialized = await JsonSerializer.DeserializeAsync(saveFile, s_serializerOptions, cancellationToken); if (serialized is null) { effectiveLogger.LogError("Unpacked tests were null or invalid."); diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/ResultAggregator.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/ResultAggregator.cs index 6447caedfff..2a2a15b0384 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/ResultAggregator.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/ResultAggregator.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. + +using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; @@ -11,52 +12,38 @@ public enum AggregationType DataDriven = 2, } -public sealed class AggregatedResult +public sealed class AggregatedResult( + AggregationType aggregationType, + string name, + double durationSeconds, + string result, + IReadOnlyList? subResults = null, + IReadOnlyList? attachments = null, + string? failureMessage = null, + string? stackTrace = null, + string? skipReason = null, + bool isFlaky = false, + int? attemptId = null) { - public AggregatedResult( - AggregationType aggregationType, - string name, - double durationSeconds, - string result, - IReadOnlyList? subResults = null, - IReadOnlyList? attachments = null, - string? failureMessage = null, - string? stackTrace = null, - string? skipReason = null, - bool isFlaky = false, - int? attemptId = null) - { - AggregationType = aggregationType; - Name = name ?? string.Empty; - DurationSeconds = durationSeconds; - Result = result ?? string.Empty; - SubResults = subResults ?? Array.Empty(); - Attachments = attachments ?? Array.Empty(); - FailureMessage = failureMessage ?? skipReason; - StackTrace = stackTrace; - IsFlaky = isFlaky; - AttemptId = attemptId; - } - - public AggregationType AggregationType { get; } + public AggregationType AggregationType { get; } = aggregationType; - public string Name { get; } + public string Name { get; } = name ?? string.Empty; - public double DurationSeconds { get; } + public double DurationSeconds { get; } = durationSeconds; - public string Result { get; } + public string Result { get; } = result ?? string.Empty; - public IReadOnlyList Attachments { get; } + public IReadOnlyList Attachments { get; } = attachments ?? []; - public IReadOnlyList SubResults { get; } + public IReadOnlyList SubResults { get; } = subResults ?? []; - public string? FailureMessage { get; } + public string? FailureMessage { get; } = failureMessage ?? skipReason; - public string? StackTrace { get; } + public string? StackTrace { get; } = stackTrace; - public int? AttemptId { get; } + public int? AttemptId { get; } = attemptId; - public bool IsFlaky { get; } + public bool IsFlaky { get; } = isFlaky; } public sealed class ResultAggregator @@ -65,7 +52,7 @@ public IReadOnlyList Aggregate(IEnumerable(); + return []; } string GetResult(TestResult test) @@ -86,7 +73,7 @@ string GetResult(TestResult test) static string ParseBasicName(string name) { - var separatorIndex = name.IndexOf('('); + int separatorIndex = name.IndexOf('('); return separatorIndex >= 0 ? name[..separatorIndex] : name; } @@ -97,7 +84,7 @@ AggregatedResult CreateResultFromTest(TestResult result, int? attemptId = null) attemptId is null ? result.Name : $"Attempt #{attemptId} - {result.Name}", result.DurationSeconds, GetResult(result), - Array.Empty(), + [], result.Attachments, result.FailureMessage, result.StackTrace, @@ -132,9 +119,9 @@ string GetDataDrivenResult(IReadOnlyList groupedResults) return (false, "None"); } - var anyPass = groupedResults.Any(static r => r.Result == "Pass"); - var anyFail = groupedResults.Any(static r => !r.Ignored && r.Result == "Fail"); - var isFlaky = anyPass && anyFail; + bool anyPass = groupedResults.Any(static r => r.Result == "Pass"); + bool anyFail = groupedResults.Any(static r => !r.Ignored && r.Result == "Fail"); + bool isFlaky = anyPass && anyFail; if (anyPass) { @@ -153,7 +140,7 @@ AggregatedResult ProcessNamedTest(string name, IReadOnlyList> b { if (byIterationThenName.Count == 1) { - var singleRun = byIterationThenName[0]; + List singleRun = byIterationThenName[0]; if (singleRun.Count == 1) { return CreateResultFromTest(singleRun[0]); @@ -164,21 +151,21 @@ AggregatedResult ProcessNamedTest(string name, IReadOnlyList> b name, singleRun.Sum(static r => r.DurationSeconds), GetDataDrivenResult(singleRun), - singleRun.Select(testResult => CreateResultFromTest(testResult)).ToList()); + [.. singleRun.Select(testResult => CreateResultFromTest(testResult))]); } - var hasDataDriven = byIterationThenName.Any(static x => x.Count > 1); + bool hasDataDriven = byIterationThenName.Any(static x => x.Count > 1); if (hasDataDriven) { var dataDrivenByFullName = new Dictionary>(StringComparer.Ordinal); - foreach (var iteration in byIterationThenName) + foreach (List iteration in byIterationThenName) { - foreach (var test in iteration) + foreach (TestResult test in iteration) { - if (!dataDrivenByFullName.TryGetValue(test.Name, out var list)) + if (!dataDrivenByFullName.TryGetValue(test.Name, out List? list)) { - list = new List(); + list = []; dataDrivenByFullName[test.Name] = list; } @@ -189,9 +176,9 @@ AggregatedResult ProcessNamedTest(string name, IReadOnlyList> b var subResults = new List(); double totalDuration = 0; - foreach (var pair in dataDrivenByFullName) + foreach (KeyValuePair> pair in dataDrivenByFullName) { - var dataDrivenTests = pair.Value; + List dataDrivenTests = pair.Value; if (dataDrivenTests.Count == 1) { subResults.Add(CreateResultFromTest(dataDrivenTests[0])); @@ -199,19 +186,19 @@ AggregatedResult ProcessNamedTest(string name, IReadOnlyList> b continue; } - var (isFlaky, aggregateResult) = GetRerunResult(dataDrivenTests); - var partialDuration = dataDrivenTests.Sum(static r => r.DurationSeconds); + (bool isFlaky, string? aggregateResult) = GetRerunResult(dataDrivenTests); + double partialDuration = dataDrivenTests.Sum(static r => r.DurationSeconds); totalDuration += partialDuration; subResults.Add(new AggregatedResult( AggregationType.Rerun, pair.Key, partialDuration, aggregateResult, - dataDrivenTests.Select((r, index) => CreateResultFromTest(r, index + 1)).ToList(), + [.. dataDrivenTests.Select((r, index) => CreateResultFromTest(r, index + 1))], isFlaky: isFlaky)); } - var aggregateOutcome = "Inconclusive"; + string aggregateOutcome = "Inconclusive"; if (dataDrivenByFullName.Values.Any(rerunSet => rerunSet.Where(static r => !r.Ignored).All(static r => r.Result == "Fail"))) { aggregateOutcome = "Failed"; @@ -234,13 +221,13 @@ AggregatedResult ProcessNamedTest(string name, IReadOnlyList> b } var reruns = byIterationThenName.Select(static run => run[0]).ToList(); - var (rerunIsFlaky, rerunOutcome) = GetRerunResult(reruns); + (bool rerunIsFlaky, string? rerunOutcome) = GetRerunResult(reruns); return new AggregatedResult( AggregationType.Rerun, name, reruns.Sum(static r => r.DurationSeconds), rerunOutcome, - reruns.Select((r, index) => CreateResultFromTest(r, index + 1)).ToList(), + [.. reruns.Select((r, index) => CreateResultFromTest(r, index + 1))], failureMessage: reruns[0].FailureMessage, stackTrace: reruns[0].StackTrace, isFlaky: rerunIsFlaky); @@ -255,14 +242,14 @@ AggregatedResult ReduceSimpleResult(AggregatedResult result) if (result.AggregationType == AggregationType.Rerun) { - var distinctOutcomes = result.SubResults + int distinctOutcomes = result.SubResults .Select(static r => r.Result) .Distinct(StringComparer.Ordinal) .Count(); if (distinctOutcomes == 1) { - var single = result.SubResults[0]; + AggregatedResult single = result.SubResults[0]; return new AggregatedResult( AggregationType.Single, result.Name, @@ -281,7 +268,7 @@ AggregatedResult ReduceSimpleResult(AggregatedResult result) result.Name, result.DurationSeconds, result.Result, - result.SubResults.Select(ReduceSimpleResult).ToList(), + [.. result.SubResults.Select(ReduceSimpleResult)], result.Attachments, result.FailureMessage, result.StackTrace, @@ -290,15 +277,15 @@ AggregatedResult ReduceSimpleResult(AggregatedResult result) } var partials = new List>>(); - foreach (var resultSet in results) + foreach (IEnumerable resultSet in results) { var perAttempt = new Dictionary>(StringComparer.Ordinal); - foreach (var result in resultSet) + foreach (TestResult result in resultSet) { - var basicName = ParseBasicName(result.Name); - if (!perAttempt.TryGetValue(basicName, out var list)) + string basicName = ParseBasicName(result.Name); + if (!perAttempt.TryGetValue(basicName, out List? list)) { - list = new List(); + list = []; perAttempt[basicName] = list; } @@ -310,28 +297,28 @@ AggregatedResult ReduceSimpleResult(AggregatedResult result) if (partials.Count == 0 || partials[0].Count == 0) { - return Array.Empty(); + return []; } var aggregate = new List(); - foreach (var run in partials) + foreach (Dictionary> run in partials) { - foreach (var pair in run.ToList()) + foreach (KeyValuePair> pair in run.ToList()) { - if (!run.Remove(pair.Key, out var currentSet)) + if (!run.Remove(pair.Key, out List? currentSet)) { continue; } var fullSet = new List> { currentSet }; - foreach (var otherRun in partials) + foreach (Dictionary> otherRun in partials) { if (ReferenceEquals(otherRun, run)) { continue; } - if (otherRun.Remove(pair.Key, out var otherSet)) + if (otherRun.Remove(pair.Key, out List? otherSet)) { fullSet.Add(otherSet); } @@ -341,6 +328,6 @@ AggregatedResult ReduceSimpleResult(AggregatedResult result) } } - return aggregate.Select(ReduceSimpleResult).ToList(); + return [.. aggregate.Select(ReduceSimpleResult)]; } } diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs index 4c677fb556e..d1a6c92bf5b 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs @@ -1,100 +1,18 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. +using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; -public sealed record TestResultAttachment(string Name, string Text); - -public sealed class TestResult -{ - public TestResult( - string name, - string kind, - string typeName, - string method, - double durationSeconds, - string result, - string? exceptionType, - string? failureMessage, - string? stackTrace, - string? skipReason, - IReadOnlyList? attachments = null) - { - Name = name ?? string.Empty; - Kind = kind ?? string.Empty; - TypeName = typeName ?? string.Empty; - Method = method ?? string.Empty; - DurationSeconds = durationSeconds; - Result = result ?? string.Empty; - ExceptionType = exceptionType; - FailureMessage = failureMessage; - StackTrace = stackTrace; - SkipReason = skipReason; - Attachments = attachments ?? Array.Empty(); - } - - public string Name { get; } - - public string Kind { get; } - - public string TypeName { get; } - - public string Method { get; } - - public double DurationSeconds { get; } - - public string Result { get; } - - public string? ExceptionType { get; } - - public string? FailureMessage { get; } - - public string? StackTrace { get; } - - public string? SkipReason { get; } - - public IReadOnlyList Attachments { get; } - - public bool Ignored { get; set; } -} - public interface ITestReporter { Task ReportResultsAsync(IReadOnlyList results, CancellationToken cancellationToken = default); } -public sealed record AzureDevOpsReportingParameters(Uri CollectionUri, string TeamProject, string TestRunId, string AccessToken); - -public sealed record PackedTestReport(AzureDevOpsReportingParameters AzdoParameters, IReadOnlyList Results); - -public sealed class HelixEnvironmentSettings -{ - public string? CorrelationId { get; init; } - - public string? WorkItemId { get; init; } - - public string? WorkItemFriendlyName { get; init; } - - public string? WorkitemWorkingDir { get; init; } - - public string? WorkitemPayloadDir { get; init; } - - public static HelixEnvironmentSettings FromEnvironment() - { - return new HelixEnvironmentSettings - { - CorrelationId = Environment.GetEnvironmentVariable("HELIX_CORRELATION_ID"), - WorkItemId = Environment.GetEnvironmentVariable("HELIX_WORKITEM_ID"), - WorkItemFriendlyName = Environment.GetEnvironmentVariable("HELIX_WORKITEM_FRIENDLYNAME"), - WorkitemWorkingDir = Environment.GetEnvironmentVariable("HELIX_WORKITEM_ROOT"), - WorkitemPayloadDir = Environment.GetEnvironmentVariable("HELIX_WORKITEM_PAYLOAD"), - }; - } -} +public sealed record AzureDevOpsReportingParameters(Uri CollectionUri, string TeamProject, string TestRunId); public interface IEventClient { From 5efeeeca4e2aed18b37a94316050bb771184e8ce Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 21 Apr 2026 12:06:15 +0200 Subject: [PATCH 05/66] Remove reporter --- .../JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj index e4b5b06a672..af8ee2e65e2 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj @@ -19,10 +19,4 @@ - - - - From 094613a9c558d1bc499db8402fc5198cb131431b Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 21 Apr 2026 13:33:47 +0200 Subject: [PATCH 06/66] Integrate Helix Job Monitor with AzDO test reporter better --- .../AzureDevOpsResultPublisher.cs | 49 ++- .../LocalTestResultsReader.cs | 280 ++++++++++++++++++ .../PackingTestReporter.cs | 43 +-- .../TestReportingModels.cs | 85 +----- .../Microsoft.DotNet.Helix.Client.csproj | 4 +- .../JobMonitor/JobMonitorOptions.cs | 20 +- .../JobMonitor/JobMonitorRunner.cs | 145 +++------ .../Microsoft.DotNet.Helix.JobMonitor.csproj | 1 + .../JobMonitor/Program.cs | 4 +- .../LocalTestResultsReaderTests.cs | 140 +++++++++ .../Microsoft.DotNet.Helix.Sdk.Tests.csproj | 1 + 11 files changed, 545 insertions(+), 227 deletions(-) create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/LocalTestResultsReader.cs create mode 100644 src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs index 3c1228f5c50..faeed58a060 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs @@ -24,23 +24,17 @@ public sealed class AzureDevOpsResultPublisher private readonly AzureDevOpsReportingParameters _azdoParameters; private readonly string _workItemId; - private readonly IEventClient _eventClient; - private readonly IUploadClient _uploadClient; private readonly HttpClient _httpClient; private readonly ILogger _logger; public AzureDevOpsResultPublisher( AzureDevOpsReportingParameters azdoParameters, string workItemId, - IEventClient? eventClient = null, - IUploadClient? uploadClient = null, HttpClient? httpClient = null, ILogger? logger = null) { _azdoParameters = azdoParameters; _workItemId = workItemId; - _eventClient = eventClient ?? NullEventClient.Instance; - _uploadClient = uploadClient ?? NullUploadClient.Instance; _httpClient = httpClient ?? CreateHttpClient(azdoParameters.AccessToken); _logger = logger.OrNull(); } @@ -64,6 +58,25 @@ public async Task TryUploadAsync(IEnumerable res } } + public async Task TryUploadDirectoryAsync(string workingDirectory, CancellationToken cancellationToken = default) + { + IReadOnlyList> parsedResults = new LocalTestResultsReader(_logger).ReadResults(workingDirectory); + if (parsedResults.Count == 0) + { + _logger.LogWarning("No test results were discovered under '{WorkingDirectory}'.", workingDirectory); + return UploadResult.UnknownError; + } + + IReadOnlyList aggregatedResults = new ResultAggregator().Aggregate(parsedResults); + if (aggregatedResults.Count == 0) + { + _logger.LogWarning("Test results were discovered under '{WorkingDirectory}', but none could be aggregated.", workingDirectory); + return UploadResult.UnknownError; + } + + return await TryUploadAsync(aggregatedResults, cancellationToken).ConfigureAwait(false); + } + private async Task ProcessAsync(IReadOnlyList testList, CancellationToken cancellationToken) { var converted = ConvertResults(testList).ToList(); @@ -82,11 +95,13 @@ private async Task ProcessAsync(IReadOnlyList testList, Cancel private async Task LogErrorAsync(Exception exception, CancellationToken cancellationToken) { _logger.LogError(exception, "Failed to upload test results to Azure DevOps."); - await _eventClient.ErrorAsync( + /* TODO + await _eventClient.ErrorAsync( HelixEnvironmentSettings.FromEnvironment(), "DevOpsReportFailure", $"Failed to upload results: {exception.Message}", cancellationToken: cancellationToken); + */ } private async Task SendMetadataAsync( @@ -148,12 +163,13 @@ void ProcessTestForMetadata(AggregatedResult result) } var uploadedUrls = new Dictionary(); + /* TODO foreach ((int key, List? testNames) in partitionedResults) { byte[] csvBytes = CreateCompressedCsv(testNames); string fileName = $"{Guid.NewGuid():N}.csv.gz"; uploadedUrls[key] = await _uploadClient.UploadAsync(csvBytes, fileName, "application/gzip", cancellationToken); - } + }*/ var dataModel = new { @@ -175,6 +191,7 @@ await SendWithRetryAsync( new TestRunAttachmentRequest(fileNameBase, base64Data), cancellationToken); + /* TODO string metadataUrl = await _uploadClient.UploadAsync(compressedBytes, fileNameBase, "application/gzip", cancellationToken); await _eventClient.SendAsync( new @@ -185,6 +202,7 @@ await _eventClient.SendAsync( Url = metadataUrl, }, cancellationToken); + */ } private async Task> PublishResultsAsync( @@ -302,11 +320,10 @@ private async Task SendAttachmentAsync( private IEnumerable ConvertResults(IEnumerable results) { - var settings = HelixEnvironmentSettings.FromEnvironment(); string? comment = JsonSerializer.Serialize(new { - HelixJobId = settings.CorrelationId, - HelixWorkItemName = settings.WorkItemFriendlyName, + HelixJobId = string.IsNullOrWhiteSpace(settings.CorrelationId) ? _workItemId : settings.CorrelationId, + HelixWorkItemName = string.IsNullOrWhiteSpace(settings.WorkItemFriendlyName) ? _workItemId : settings.WorkItemFriendlyName, }); static string GetResultGroupType(AggregationType aggregationType) @@ -460,7 +477,7 @@ private static IEnumerable> Batch(IEnumerable items, int limit, Fu } } - private static HttpClient CreateHttpClient(string accessToken) + private static HttpClient CreateHttpClient(string? accessToken) { var client = new HttpClient(); client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); @@ -489,7 +506,11 @@ private async Task SendWithRetryAsync( while (true) { - using var request = new HttpRequestMessage(method, new Uri(_azdoParameters.CollectionUri, relativePath)); + Uri baseUri = _azdoParameters.CollectionUri.AbsoluteUri.EndsWith('/') + ? _azdoParameters.CollectionUri + : new Uri(_azdoParameters.CollectionUri.AbsoluteUri + '/', UriKind.Absolute); + + using var request = new HttpRequestMessage(method, new Uri(baseUri, relativePath)); if (body is not null) { request.Content = new StringContent(body, Encoding.UTF8, "application/json"); @@ -525,11 +546,13 @@ private async Task SendWithRetryAsync( { if (!string.IsNullOrWhiteSpace(s_lastSendContent)) { + /* TODO await _uploadClient.UploadAsync( Encoding.UTF8.GetBytes(s_lastSendContent), "__failed_azdo_request_content.json", "text/plain; charset=UTF-8", cancellationToken); + */ } } catch (Exception uploadException) diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/LocalTestResultsReader.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/LocalTestResultsReader.cs new file mode 100644 index 00000000000..c9f4283c556 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/LocalTestResultsReader.cs @@ -0,0 +1,280 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Globalization; +using System.Text.Json; +using System.Xml.Linq; +using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +using Microsoft.Extensions.Logging; + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; + +public sealed class LocalTestResultsReader(ILogger? logger = null) +{ + private static readonly JsonSerializerOptions s_serializerOptions = new(JsonSerializerDefaults.Web); + private readonly ILogger _logger = logger.OrNull(); + + public IReadOnlyList> ReadResults(string searchDirectory) + { + if (string.IsNullOrWhiteSpace(searchDirectory) || !Directory.Exists(searchDirectory)) + { + return []; + } + + List packedReportFiles = Directory + .EnumerateFiles(searchDirectory, "__test_report.json", SearchOption.AllDirectories) + .ToList(); + + var packedDirectories = new HashSet( + packedReportFiles + .Select(Path.GetDirectoryName) + .Where(static path => !string.IsNullOrWhiteSpace(path))!, + StringComparer.OrdinalIgnoreCase); + + var allResults = new List>(); + foreach (string packedReportFile in packedReportFiles) + { + IReadOnlyList packedResults = ReadPackedResults(packedReportFile); + if (packedResults.Count > 0) + { + allResults.Add(packedResults); + } + } + + foreach (string filePath in Directory.EnumerateFiles(searchDirectory, "*", SearchOption.AllDirectories)) + { + if (!LooksLikeTestResultFile(filePath) + || string.Equals(Path.GetFileName(filePath), "__test_report.json", StringComparison.OrdinalIgnoreCase) + || packedDirectories.Any(directory => IsPathUnderDirectory(filePath, directory))) + { + continue; + } + + IReadOnlyList parsed = ReadResultFile(filePath); + if (parsed.Count > 0) + { + allResults.Add(parsed); + } + } + + return allResults; + } + + public static bool LooksLikeTestResultFile(string path) + { + string fileName = Path.GetFileName(path); + if (string.Equals(fileName, "__test_report.json", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + return fileName.EndsWith(".trx", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith("testResults.xml", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith("test-results.xml", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith("test_results.xml", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith("junit-results.xml", StringComparison.OrdinalIgnoreCase) + || fileName.EndsWith("junitresults.xml", StringComparison.OrdinalIgnoreCase); + } + + private IReadOnlyList ReadPackedResults(string filePath) + { + try + { + using FileStream stream = File.OpenRead(filePath); + PackedTestReport? report = JsonSerializer.Deserialize(stream, s_serializerOptions); + return report?.Results ?? []; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to read packed test report '{Path}'.", filePath); + return []; + } + } + + private IReadOnlyList ReadResultFile(string filePath) + { + try + { + XDocument document = XDocument.Load(filePath, LoadOptions.PreserveWhitespace); + string rootName = document.Root?.Name.LocalName ?? string.Empty; + string workItemName = new DirectoryInfo(Path.GetDirectoryName(filePath) ?? string.Empty).Name; + + return rootName switch + { + "assemblies" or "assembly" => ReadXunitResults(document), + "TestRun" => ReadTrxResults(document, workItemName), + "testsuites" or "testsuite" => ReadJUnitResults(document, workItemName), + _ => [], + }; + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to parse test results file '{Path}'.", filePath); + return []; + } + } + + private static IReadOnlyList ReadXunitResults(XDocument document) + { + return [.. + document.Descendants().Where(static e => e.Name.LocalName == "test").Select(static test => + { + XElement? failure = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "failure"); + string? message = failure?.Elements().FirstOrDefault(static x => x.Name.LocalName == "message")?.Value?.Trim(); + string? stackTrace = failure?.Elements().FirstOrDefault(static x => x.Name.LocalName == "stack-trace")?.Value?.Trim(); + string? output = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "output")?.Value?.Trim(); + string? skipReason = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "reason")?.Value?.Trim(); + + List attachments = []; + AddAttachmentIfNotEmpty(attachments, "output.txt", output); + + string typeName = GetAttribute(test, "type") ?? string.Empty; + string method = GetAttribute(test, "method") ?? string.Empty; + string name = GetAttribute(test, "name") + ?? (!string.IsNullOrEmpty(typeName) && !string.IsNullOrEmpty(method) ? $"{typeName}.{method}" : method); + + return new TestResult( + name, + "xunit", + typeName, + method, + ParseDouble(GetAttribute(test, "time")), + NormalizeOutcome(GetAttribute(test, "result")), + GetAttribute(failure, "exception-type"), + message, + stackTrace, + skipReason, + attachments); + })]; + } + + private static IReadOnlyList ReadJUnitResults(XDocument document, string workItemName) + { + return [.. + document.Descendants().Where(static e => e.Name.LocalName == "testcase").Select(test => + { + XElement? failure = test.Elements().FirstOrDefault(static x => x.Name.LocalName is "failure" or "error"); + XElement? skipped = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "skipped"); + string? stdout = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "system-out")?.Value?.Trim(); + string? stderr = test.Elements().FirstOrDefault(static x => x.Name.LocalName == "system-err")?.Value?.Trim(); + + List attachments = []; + AddAttachmentIfNotEmpty(attachments, "stdout.txt", stdout); + AddAttachmentIfNotEmpty(attachments, "stderr.txt", stderr); + + string className = GetAttribute(test, "classname") ?? workItemName; + string method = GetAttribute(test, "name") ?? string.Empty; + string name = !string.IsNullOrEmpty(className) ? $"{className}.{method}" : method; + string result = skipped is not null ? "Skip" : failure is not null ? "Fail" : "Pass"; + + return new TestResult( + name, + "junit", + className, + method, + ParseDouble(GetAttribute(test, "time")), + result, + null, + failure?.Value?.Trim(), + null, + skipped?.Value?.Trim(), + attachments); + })]; + } + + private static IReadOnlyList ReadTrxResults(XDocument document, string workItemName) + { + Dictionary unitTestsById = document + .Descendants() + .Where(static e => e.Name.LocalName == "UnitTest") + .Select(static unitTest => (Id: GetAttribute(unitTest, "id"), Element: unitTest)) + .Where(static x => !string.IsNullOrEmpty(x.Id)) + .ToDictionary(static x => x.Id!, static x => x.Element, StringComparer.OrdinalIgnoreCase); + + return [.. + document.Descendants().Where(static e => e.Name.LocalName == "UnitTestResult").Select(result => + { + string testId = GetAttribute(result, "testId") ?? string.Empty; + unitTestsById.TryGetValue(testId, out XElement? unitTest); + XElement? testMethod = unitTest?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "TestMethod"); + + string className = GetAttribute(testMethod, "className") ?? workItemName; + string method = GetAttribute(testMethod, "name") ?? GetAttribute(result, "testName") ?? string.Empty; + string displayName = GetAttribute(result, "testName") + ?? (!string.IsNullOrEmpty(className) ? $"{className}.{method}" : method); + + XElement? output = result.Descendants().FirstOrDefault(static x => x.Name.LocalName == "Output"); + string? failureMessage = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "Message")?.Value?.Trim(); + string? stackTrace = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StackTrace")?.Value?.Trim(); + string? stdout = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StdOut")?.Value?.Trim(); + string? stderr = output?.Descendants().FirstOrDefault(static x => x.Name.LocalName == "StdErr")?.Value?.Trim(); + + List attachments = []; + AddAttachmentIfNotEmpty(attachments, "stdout.txt", stdout); + AddAttachmentIfNotEmpty(attachments, "stderr.txt", stderr); + + string rawOutcome = GetAttribute(result, "outcome") ?? string.Empty; + string normalizedOutcome = NormalizeOutcome(rawOutcome); + string? skipReason = string.Equals(normalizedOutcome, "Skip", StringComparison.Ordinal) ? failureMessage : null; + + return new TestResult( + displayName, + "trx", + className, + method, + ParseDuration(GetAttribute(result, "duration")), + normalizedOutcome, + null, + failureMessage, + stackTrace, + skipReason, + attachments); + })]; + } + + private static string? GetAttribute(XElement? element, string name) + => element?.Attribute(name)?.Value; + + private static double ParseDouble(string? value) + { + return double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double result) + ? result + : 0; + } + + private static double ParseDuration(string? value) + { + return TimeSpan.TryParse(value, CultureInfo.InvariantCulture, out TimeSpan result) + ? result.TotalSeconds + : ParseDouble(value); + } + + private static string NormalizeOutcome(string? value) + { + return value?.Trim().ToLowerInvariant() switch + { + "pass" or "passed" or "success" or "succeeded" => "Pass", + "skip" or "skipped" or "notexecuted" or "notrun" => "Skip", + "fail" or "failed" or "error" or "timeout" or "aborted" => "Fail", + _ => "None", + }; + } + + private static bool IsPathUnderDirectory(string filePath, string directory) + { + string normalizedDirectory = Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)); + string normalizedPath = Path.GetFullPath(filePath); + + return normalizedPath.Equals(normalizedDirectory, StringComparison.OrdinalIgnoreCase) + || normalizedPath.StartsWith(normalizedDirectory + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) + || normalizedPath.StartsWith(normalizedDirectory + Path.AltDirectorySeparatorChar, StringComparison.OrdinalIgnoreCase); + } + + private static void AddAttachmentIfNotEmpty(List attachments, string name, string? text) + { + if (!string.IsNullOrWhiteSpace(text)) + { + attachments.Add(new TestResultAttachment(name, text)); + } + } +} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs index 057afec4ddc..1f58470a679 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs @@ -7,6 +7,11 @@ namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; +public interface ITestReporter +{ + Task ReportResultsAsync(IReadOnlyList results, CancellationToken cancellationToken = default); +} + public sealed class PackingTestReporter(AzureDevOpsReportingParameters azdoParameters, ILogger? logger = null) : ITestReporter { @@ -23,7 +28,7 @@ public async Task ReportResultsAsync(IReadOnlyList results, Cancella .ToList(); var serialized = new PackedTestReport(_azdoParameters, filtered); - string path = GetFileName(); + string path = Path.Combine(Environment.CurrentDirectory, ReportFileName); string? directory = Path.GetDirectoryName(path); if (!string.IsNullOrEmpty(directory)) { @@ -41,32 +46,34 @@ public async Task ReportResultsAsync(IReadOnlyList results, Cancella _logger.LogInformation("Packed {Length} bytes", new FileInfo(path).Length); } - public static string GetFileName(HelixEnvironmentSettings? settings = null) - { - settings ??= HelixEnvironmentSettings.FromEnvironment(); - string? root = settings.WorkitemWorkingDir; - if (string.IsNullOrWhiteSpace(root)) - { - root = Environment.CurrentDirectory; - } - - return Path.Combine(root, ReportFileName); - } - public static async Task<(AzureDevOpsReportingParameters Parameters, IReadOnlyList Results)?> UnpackResultsAsync( + string? searchDirectory = null, ILogger? logger = null, CancellationToken cancellationToken = default) { ILogger effectiveLogger = logger.OrNull(); - var settings = HelixEnvironmentSettings.FromEnvironment(); - string path = GetFileName(settings); + string? path = null; + + if (!string.IsNullOrWhiteSpace(searchDirectory)) + { + path = Path.Combine(searchDirectory, ReportFileName); + if (!File.Exists(path) && Directory.Exists(searchDirectory)) + { + path = Directory.EnumerateFiles(searchDirectory, ReportFileName, SearchOption.AllDirectories).FirstOrDefault(); + } + } - if (!File.Exists(path) && !string.IsNullOrWhiteSpace(settings.WorkitemPayloadDir)) + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) { - path = Path.Combine(settings.WorkitemPayloadDir, ReportFileName); + path = Path.Combine(Environment.CurrentDirectory, ReportFileName); + + if (!File.Exists(path) && !string.IsNullOrWhiteSpace(settings.WorkitemPayloadDir)) + { + path = Path.Combine(settings.WorkitemPayloadDir, ReportFileName); + } } - if (!File.Exists(path)) + if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) { return null; } diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs index d1a6c92bf5b..9ec5c84ec5a 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs @@ -7,90 +7,7 @@ namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; -public interface ITestReporter -{ - Task ReportResultsAsync(IReadOnlyList results, CancellationToken cancellationToken = default); -} - -public sealed record AzureDevOpsReportingParameters(Uri CollectionUri, string TeamProject, string TestRunId); - -public interface IEventClient -{ - Task SendAsync(object payload, CancellationToken cancellationToken = default); - - Task ErrorAsync( - HelixEnvironmentSettings settings, - string errorType, - string message, - string? logUri = null, - CancellationToken cancellationToken = default); -} - -public interface IUploadClient -{ - Task UploadAsync( - Stream file, - string name, - string contentType = "application/octet-stream", - CancellationToken cancellationToken = default); - - Task UploadAsync( - ReadOnlyMemory fileBytes, - string name, - string contentType = "application/octet-stream", - CancellationToken cancellationToken = default); -} - -public sealed class NullEventClient : IEventClient -{ - public static readonly NullEventClient Instance = new(); - - private NullEventClient() - { - } - - public Task SendAsync(object payload, CancellationToken cancellationToken = default) - { - return Task.CompletedTask; - } - - public Task ErrorAsync( - HelixEnvironmentSettings settings, - string errorType, - string message, - string? logUri = null, - CancellationToken cancellationToken = default) - { - return Task.CompletedTask; - } -} - -public sealed class NullUploadClient : IUploadClient -{ - public static readonly NullUploadClient Instance = new(); - - private NullUploadClient() - { - } - - public Task UploadAsync( - Stream file, - string name, - string contentType = "application/octet-stream", - CancellationToken cancellationToken = default) - { - return Task.FromResult($"memory://{name}"); - } - - public Task UploadAsync( - ReadOnlyMemory fileBytes, - string name, - string contentType = "application/octet-stream", - CancellationToken cancellationToken = default) - { - return Task.FromResult($"memory://{name}"); - } -} +public sealed record AzureDevOpsReportingParameters(Uri CollectionUri, string TeamProject, string TestRunId, string? AccessToken = null); public static class LoggerFactoryExtensions { diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj b/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj index 061f83a5238..f2ee42f758d 100644 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj @@ -2,10 +2,10 @@ - $(BundledNETCoreAppTargetFramework) + net10.0 true This package provides access to the Helix Api located at https://helix.dot.net/ - https://helix.dot.net/api/openapi.json + https://helix.int-dot.net/api/openapi.json HelixApi README.md diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs index 4d9517c7903..b2b037bde5e 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs @@ -9,14 +9,19 @@ namespace Microsoft.DotNet.Helix.JobMonitor { public sealed class JobMonitorOptions { + // Helix API access token + public string HelixAccessToken { get; set; } + + /// + /// Azure DevOps build token + /// + public string SystemAccessToken { get; set; } + public bool ShowHelp { get; private set; } [Option("helix-base-uri", HelpText = "Base URI for the Helix service.")] public string HelixBaseUri { get; set; } = "https://helix.dot.net/"; - [Option("helix-access-token", HelpText = "Access token for authenticated Helix APIs.")] - public string HelixAccessToken { get; set; } - [Option("collection-uri", HelpText = "Azure DevOps collection URI.")] public string CollectionUri { get; set; } @@ -26,9 +31,6 @@ public sealed class JobMonitorOptions [Option("build-id", HelpText = "Azure DevOps build ID.")] public string BuildId { get; set; } - [Option("access-token", HelpText = "Azure DevOps system access token.")] - public string AccessToken { get; set; } - [Option("repository", HelpText = "Repository identifier in owner/repo form.")] public string Repository { get; set; } @@ -82,7 +84,7 @@ private void ApplyEnvironmentDefaults() CollectionUri ??= Environment.GetEnvironmentVariable("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI"); TeamProject ??= Environment.GetEnvironmentVariable("SYSTEM_TEAMPROJECT"); BuildId ??= Environment.GetEnvironmentVariable("BUILD_BUILDID"); - AccessToken ??= Environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN"); + SystemAccessToken ??= Environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN"); Repository = HelixJobMonitorUtilities.NormalizeRepository( Repository ?? Environment.GetEnvironmentVariable("BUILD_REPOSITORY_URI") @@ -97,7 +99,7 @@ private void Validate() CollectionUri = EnsureTrailingSlash(RequireValue(CollectionUri, "collection-uri", "SYSTEM_TEAMFOUNDATIONCOLLECTIONURI")); TeamProject = RequireValue(TeamProject, "team-project", "SYSTEM_TEAMPROJECT"); BuildId = RequireValue(BuildId, "build-id", "BUILD_BUILDID"); - AccessToken = RequireValue(AccessToken, "access-token", "SYSTEM_ACCESSTOKEN"); + SystemAccessToken = RequireValue(SystemAccessToken, "access-token", "SYSTEM_ACCESSTOKEN"); if (string.IsNullOrWhiteSpace(Repository)) { @@ -127,6 +129,6 @@ private static string RequireValue(string value, string argumentName, string env } private static string EnsureTrailingSlash(string uri) - => uri.EndsWith("/", StringComparison.Ordinal) ? uri : uri + "/"; + => uri.EndsWith('/') ? uri : uri + '/'; } } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index dc4d410c849..295ad23c2bc 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; -using System.Diagnostics; using System.Globalization; using System.IO; using System.Linq; @@ -14,6 +13,8 @@ using System.Threading.Tasks; using Azure; using Azure.Storage.Blobs; +using Microsoft.DotNet.Helix.AzureDevOpsTestReporter; +using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; using Microsoft.DotNet.Helix.Client; using Microsoft.DotNet.Helix.Client.Models; using Newtonsoft.Json; @@ -37,14 +38,14 @@ public JobMonitorRunner(JobMonitorOptions options) : ApiFactory.GetAuthenticated(_options.HelixBaseUri, _options.HelixAccessToken); _azdoClient = new HttpClient(); - string encodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + _options.AccessToken)); + string encodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + _options.SystemAccessToken)); _azdoClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", encodedToken); _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-job-monitor"); } public async Task RunAsync() { - HashSet processedRuns = await GetProcessedRunNamesAsync().ConfigureAwait(false); + HashSet processedRuns = await GetProcessedRunNamesAsync(); DateTimeOffset deadline = DateTimeOffset.UtcNow.AddMinutes(Math.Max(1, _options.MaximumWaitMinutes)); bool anyNonMonitorJobFailures = false; int failedHelixJobCount = 0; @@ -52,11 +53,10 @@ public async Task RunAsync() while (DateTimeOffset.UtcNow < deadline) { - AzureDevOpsTimelineRecord[] timelineRecords = await GetTimelineRecordsAsync().ConfigureAwait(false); + AzureDevOpsTimelineRecord[] timelineRecords = await GetTimelineRecordsAsync(); JobStatus[] jobs = await RetryAsync( () => Task.FromResult(Array.Empty()) - /*TODO _helixApi.PullRequests.ByBuildAsync(_options.Repository, _options.PrNumber, int.Parse(_options.BuildId, CultureInfo.InvariantCulture), _options.Attempt)*/) - .ConfigureAwait(false); + /*TODO _helixApi.PullRequests.ByBuildAsync(_options.Repository, _options.PrNumber, int.Parse(_options.BuildId, CultureInfo.InvariantCulture), _options.Attempt)*/); int completedHelixJobs = jobs.Count(j => j.IsCompleted); int currentFailedJobs = jobs.Count(j => j.Status.Equals("failed", StringComparison.OrdinalIgnoreCase)); @@ -69,8 +69,8 @@ public async Task RunAsync() continue; } - JobPassFail passFail = await RetryAsync(() => _helixApi.Job.PassFailAsync(job.JobName)).ConfigureAwait(false); - bool passed = await ProcessCompletedJobAsync(job, passFail).ConfigureAwait(false); + JobPassFail passFail = await RetryAsync(() => _helixApi.Job.PassFailAsync(job.JobName)); + bool passed = await ProcessCompletedJobAsync(job, passFail); processedRuns.Add(job.JobName); processedHelixJobCount++; if (!passed) @@ -81,7 +81,7 @@ public async Task RunAsync() anyNonMonitorJobFailures = HelixJobMonitorUtilities.HasFailedNonMonitorJobs(timelineRecords, _options.JobMonitorName); bool allPipelineJobsComplete = HelixJobMonitorUtilities.AreNonMonitorJobsComplete(timelineRecords, _options.JobMonitorName); - bool allHelixJobsComplete = jobs.Any() && jobs.All(j => j.IsCompleted); + bool allHelixJobsComplete = jobs.Length != 0 && jobs.All(j => j.IsCompleted); if (allPipelineJobsComplete && allHelixJobsComplete) { @@ -104,7 +104,7 @@ public async Task RunAsync() return 0; } - await Task.Delay(TimeSpan.FromSeconds(Math.Max(5, _options.PollingIntervalSeconds))).ConfigureAwait(false); + await Task.Delay(TimeSpan.FromSeconds(Math.Max(5, _options.PollingIntervalSeconds))); } Console.Error.WriteLine($"The Helix Job Monitor timed out after {_options.MaximumWaitMinutes} minute(s)."); @@ -119,18 +119,19 @@ public void Dispose() private async Task ProcessCompletedJobAsync(JobStatus helixJob, JobPassFail passFail) { string testRunName = HelixJobMonitorUtilities.GetTestRunName(helixJob.JobName); - int testRunId = await StartTestRunAsync(testRunName).ConfigureAwait(false); + int testRunId = await StartTestRunAsync(testRunName); string resultsDirectory = Path.Combine(_options.WorkingDirectory, MakeSafeDirectoryName(helixJob.JobName)); Directory.CreateDirectory(resultsDirectory); - int downloadedFiles = await DownloadTestResultsAsync(helixJob.JobName, passFail, resultsDirectory).ConfigureAwait(false); - bool reporterRan = downloadedFiles > 0 && await TryRunPythonReporterAsync(resultsDirectory, testRunId).ConfigureAwait(false); + int downloadedFiles = await DownloadTestResultsAsync(helixJob.JobName, passFail, resultsDirectory); + bool reporterRan = downloadedFiles > 0 + && await TryUploadDownloadedResultsAsync(resultsDirectory, testRunId, helixJob.JobName); if (!reporterRan) { - await CreateFallbackResultsAsync(testRunId, helixJob.JobName, passFail).ConfigureAwait(false); + await CreateFallbackResultsAsync(testRunId, helixJob.JobName, passFail); } - await StopTestRunAsync(testRunId, testRunName).ConfigureAwait(false); + await StopTestRunAsync(testRunId, testRunName); int passedCount = passFail.Passed?.Count ?? 0; int failedCount = passFail.Failed?.Count ?? 0; @@ -140,7 +141,7 @@ private async Task ProcessCompletedJobAsync(JobStatus helixJob, JobPassFai private async Task> GetProcessedRunNamesAsync() { - JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?buildIds={_options.BuildId}&api-version=7.1-preview.1").ConfigureAwait(false); + JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?buildIds={_options.BuildId}&api-version=7.1-preview.1"); var processed = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (JObject run in data?["value"] as JArray ?? []) @@ -160,7 +161,7 @@ private async Task> GetProcessedRunNamesAsync() private async Task GetTimelineRecordsAsync() { - JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/build/builds/{_options.BuildId}/timeline?api-version=7.1-preview.2").ConfigureAwait(false); + JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/build/builds/{_options.BuildId}/timeline?api-version=7.1-preview.2"); return data?["records"]?.ToObject() ?? []; } @@ -175,7 +176,7 @@ private async Task StartTestRunAsync(string testRunName) ["build"] = new JObject { ["id"] = _options.BuildId }, ["name"] = testRunName, ["state"] = "InProgress", - }).ConfigureAwait(false); + }); return result?["id"]?.ToObject() ?? 0; } @@ -185,7 +186,7 @@ private async Task StopTestRunAsync(int testRunId, string testRunName) await SendAsync( new HttpMethod("PATCH"), $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=5.0", - new JObject { ["state"] = "Completed" }).ConfigureAwait(false); + new JObject { ["state"] = "Completed" }); Console.WriteLine($"Stopped test run '{testRunName}'."); } @@ -194,12 +195,12 @@ private async Task CreateFallbackResultsAsync(int testRunId, string jobName, Job { foreach (string workItemName in passFail.Passed ?? ImmutableList.Empty) { - await CreateFallbackResultAsync(testRunId, jobName, workItemName, failed: false).ConfigureAwait(false); + await CreateFallbackResultAsync(testRunId, jobName, workItemName, failed: false); } foreach (string workItemName in passFail.Failed ?? ImmutableList.Empty) { - await CreateFallbackResultAsync(testRunId, jobName, workItemName, failed: true).ConfigureAwait(false); + await CreateFallbackResultAsync(testRunId, jobName, workItemName, failed: true); } } @@ -226,22 +227,22 @@ await SendAsync( ["HelixWorkItemName"] = cleanName, }.ToString(), } - }).ConfigureAwait(false); + }); } private async Task DownloadTestResultsAsync(string jobName, JobPassFail passFail, string outputDirectory) { int count = 0; - JobResultsUri resultsUri = await RetryAsync(() => _helixApi.Job.ResultsAsync(jobName)).ConfigureAwait(false); + JobResultsUri resultsUri = await RetryAsync(() => _helixApi.Job.ResultsAsync(jobName)); IEnumerable workItemNames = (passFail.Passed ?? ImmutableList.Empty).Concat(passFail.Failed ?? ImmutableList.Empty); foreach (string workItemName in workItemNames.Distinct(StringComparer.OrdinalIgnoreCase)) { - var availableFiles = await RetryAsync(() => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false)).ConfigureAwait(false); + IImmutableList availableFiles = await RetryAsync(() => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false)); string workItemDirectory = Path.Combine(outputDirectory, MakeSafeDirectoryName(workItemName)); Directory.CreateDirectory(workItemDirectory); - foreach (var file in availableFiles.Where(f => LooksLikeTestResultFile(f.Name))) + foreach (UploadedFile file in availableFiles.Where(f => LooksLikeTestResultFile(f.Name))) { string relativePath = file.Name.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar); string destinationFile = Path.Combine(workItemDirectory, relativePath); @@ -254,7 +255,7 @@ private async Task DownloadTestResultsAsync(string jobName, JobPassFail pas try { BlobClient blobClient = CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS); - await blobClient.DownloadToAsync(destinationFile).ConfigureAwait(false); + await blobClient.DownloadToAsync(destinationFile); count++; } catch (Exception ex) @@ -267,61 +268,23 @@ private async Task DownloadTestResultsAsync(string jobName, JobPassFail pas return count; } - private async Task TryRunPythonReporterAsync(string workingDirectory, int testRunId) + private async Task TryUploadDownloadedResultsAsync(string workingDirectory, int testRunId, string jobName) { - string scriptPath = Path.Combine(AppContext.BaseDirectory, "reporter", "run.py"); - if (!File.Exists(scriptPath)) + var publisher = new AzureDevOpsResultPublisher( + new AzureDevOpsReportingParameters( + new Uri(_options.CollectionUri, UriKind.Absolute), + _options.TeamProject, + testRunId.ToString(CultureInfo.InvariantCulture), + _options.SystemAccessToken), + jobName); + + UploadResult uploadResult = await publisher.TryUploadDirectoryAsync(workingDirectory); + if (uploadResult != UploadResult.Success) { - Console.WriteLine($"Warning: reporter script was not found at '{scriptPath}'. Falling back to synthetic work-item results."); - return false; + Console.WriteLine($"Warning: test result upload for '{jobName}' returned '{uploadResult}'. Falling back to synthetic work-item results."); } - foreach ((string fileName, string prefixArguments) in GetPythonCandidates()) - { - try - { - var psi = new ProcessStartInfo - { - FileName = fileName, - Arguments = $"{prefixArguments}\"{scriptPath}\" \"{_options.CollectionUri}\" \"{_options.TeamProject}\" \"{testRunId.ToString(CultureInfo.InvariantCulture)}\" \"{_options.AccessToken}\"", - WorkingDirectory = workingDirectory, - UseShellExecute = false, - RedirectStandardOutput = true, - RedirectStandardError = true, - }; - - using Process process = Process.Start(psi); - if (process == null) - { - continue; - } - - string stdout = await process.StandardOutput.ReadToEndAsync().ConfigureAwait(false); - string stderr = await process.StandardError.ReadToEndAsync().ConfigureAwait(false); - await process.WaitForExitAsync().ConfigureAwait(false); - - if (!string.IsNullOrWhiteSpace(stdout)) - { - Console.WriteLine(stdout); - } - - if (!string.IsNullOrWhiteSpace(stderr)) - { - Console.WriteLine(stderr); - } - - if (process.ExitCode == 0) - { - return true; - } - } - catch (Exception ex) - { - Console.WriteLine($"Warning: failed to invoke Python reporter via '{fileName}': {ex.Message}"); - } - } - - return false; + return uploadResult == UploadResult.Success; } private async Task SendAsync(HttpMethod method, string requestUri, JToken body = null) @@ -334,8 +297,8 @@ private async Task SendAsync(HttpMethod method, string requestUri, JTok request.Content = new StringContent(body.ToString(Formatting.None), Encoding.UTF8, "application/json"); } - using HttpResponseMessage response = await _azdoClient.SendAsync(request).ConfigureAwait(false); - string content = response.Content != null ? await response.Content.ReadAsStringAsync().ConfigureAwait(false) : null; + using HttpResponseMessage response = await _azdoClient.SendAsync(request); + string content = response.Content != null ? await response.Content.ReadAsStringAsync() : null; if (!response.IsSuccessStatusCode) { throw new HttpRequestException($"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}"); @@ -347,7 +310,7 @@ private async Task SendAsync(HttpMethod method, string requestUri, JTok } return JObject.Parse(content); - }).ConfigureAwait(false); + }); } private static BlobClient CreateBlobClient(string fileLink, string resultsSas) @@ -365,12 +328,7 @@ private static BlobClient CreateBlobClient(string fileLink, string resultsSas) } private static bool LooksLikeTestResultFile(string path) - { - string fileName = Path.GetFileName(path); - return fileName.EndsWith(".xml", StringComparison.OrdinalIgnoreCase) - && (fileName.Contains("testresults", StringComparison.OrdinalIgnoreCase) - || fileName.Contains("test-results", StringComparison.OrdinalIgnoreCase)); - } + => LocalTestResultsReader.LooksLikeTestResultFile(path); private static string MakeSafeDirectoryName(string value) { @@ -382,17 +340,6 @@ private static string MakeSafeDirectoryName(string value) return value; } - private static IEnumerable<(string fileName, string prefixArguments)> GetPythonCandidates() - { - if (OperatingSystem.IsWindows()) - { - yield return ("py", "-3 "); - } - - yield return ("python3", string.Empty); - yield return ("python", string.Empty); - } - private static async Task RetryAsync(Func> action) { Exception last = null; @@ -400,12 +347,12 @@ private static async Task RetryAsync(Func> action) { try { - return await action().ConfigureAwait(false); + return await action(); } catch (Exception ex) when (attempt < 4) { last = ex; - await Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, attempt + 1))).ConfigureAwait(false); + await Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, attempt + 1))); } } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj index af8ee2e65e2..cf096b3f232 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj @@ -16,6 +16,7 @@ + diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs index 574d89f7af2..10729b8ea8c 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs @@ -18,8 +18,8 @@ public static async Task Main(string[] args) return 0; } - JobMonitorRunner runner = new JobMonitorRunner(options); - return await runner.RunAsync().ConfigureAwait(false); + JobMonitorRunner runner = new(options); + return await runner.RunAsync(); } catch (Exception ex) { diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs new file mode 100644 index 00000000000..52559f16586 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs @@ -0,0 +1,140 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.DotNet.Helix.AzureDevOpsTestReporter; +using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +using Xunit; + +namespace Microsoft.DotNet.Helix.Sdk.Tests +{ + public class LocalTestResultsReaderTests + { + [Fact] + public async Task PackingTestReporter_CanUnpackFromSpecifiedDirectory() + { + string tempDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(tempDirectory); + string originalDirectory = Environment.CurrentDirectory; + + try + { + Environment.CurrentDirectory = tempDirectory; + + var reporter = new PackingTestReporter( + new AzureDevOpsReportingParameters(new Uri("https://dev.azure.com/dnceng/"), "arcade", "42")); + + await reporter.ReportResultsAsync( + [ + new TestResult( + "Sample.Tests.Passes", + "unit", + "Sample.Tests", + "Passes", + 1.25, + "Pass", + null, + null, + null, + null) + ]); + + var unpacked = await PackingTestReporter.UnpackResultsAsync(tempDirectory); + + Assert.True(unpacked.HasValue); + Assert.Equal("42", unpacked.Value.Parameters.TestRunId); + Assert.Single(unpacked.Value.Results); + Assert.Equal("Sample.Tests.Passes", unpacked.Value.Results[0].Name); + } + finally + { + Environment.CurrentDirectory = originalDirectory; + Directory.Delete(tempDirectory, recursive: true); + } + } + + [Fact] + public void LocalTestResultsReader_ReadsXunitFileFromDownloadedResults() + { + string tempDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); + string workItemDirectory = Path.Combine(tempDirectory, "work-item"); + Directory.CreateDirectory(workItemDirectory); + + try + { + File.WriteAllText( + Path.Combine(workItemDirectory, "testResults.xml"), + """ + + + + + + + + """); + + var reader = new LocalTestResultsReader(); + var resultSets = reader.ReadResults(tempDirectory); + var aggregate = new ResultAggregator().Aggregate(resultSets); + AggregatedResult result = Assert.Single(aggregate); + + Assert.Equal("Sample.Tests.Passes", result.Name); + Assert.Equal("Passed", result.Result); + } + finally + { + Directory.Delete(tempDirectory, recursive: true); + } + } + + [Fact] + public async Task LocalTestResultsReader_CombinesPackedAndXmlResultsAcrossWorkItems() + { + string tempDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); + string packedDirectory = Path.Combine(tempDirectory, "packed-item"); + string xmlDirectory = Path.Combine(tempDirectory, "xml-item"); + Directory.CreateDirectory(packedDirectory); + Directory.CreateDirectory(xmlDirectory); + string originalDirectory = Environment.CurrentDirectory; + + try + { + Environment.CurrentDirectory = packedDirectory; + var reporter = new PackingTestReporter( + new AzureDevOpsReportingParameters(new Uri("https://dev.azure.com/dnceng/"), "arcade", "42")); + await reporter.ReportResultsAsync( + [ + new TestResult("Packed.Tests.Passes", "unit", "Packed.Tests", "Passes", 1, "Pass", null, null, null, null) + ]); + + File.WriteAllText( + Path.Combine(xmlDirectory, "testResults.xml"), + """ + + + + + + + + """); + + var resultSets = new LocalTestResultsReader().ReadResults(tempDirectory); + var aggregate = new ResultAggregator().Aggregate(resultSets); + + Assert.Equal(2, aggregate.Count); + Assert.Contains(aggregate, static x => x.Name == "Packed.Tests.Passes"); + Assert.Contains(aggregate, static x => x.Name == "Xml.Tests.Passes"); + } + finally + { + Environment.CurrentDirectory = originalDirectory; + Directory.Delete(tempDirectory, recursive: true); + } + } + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj index c464b912e7d..2d2dfa4c78a 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj @@ -21,6 +21,7 @@ + From b94c37fb9f0c50e44a46abf129c73c3e93e8738a Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 21 Apr 2026 13:56:24 +0200 Subject: [PATCH 07/66] Add the new endpoint to Helix Client --- .../Client/CSharp/generated-code/Aggregate.cs | 30 +++++ .../Client/CSharp/generated-code/Job.cs | 113 ++++++++++++++++++ .../Models/PullRequestJobSummary.cs | 41 +++++++ 3 files changed, 184 insertions(+) create mode 100644 src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/PullRequestJobSummary.cs diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Aggregate.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Aggregate.cs index 3fe2105fe64..8d5a8562198 100644 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Aggregate.cs +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Aggregate.cs @@ -21,6 +21,7 @@ public partial interface IAggregate IImmutableList groupBy, IImmutableList otherProperties, string workitem, + string branch = default, string build = default, string creator = default, string name = default, @@ -45,6 +46,7 @@ public partial interface IAggregate Task> JobSummaryAsync( IImmutableList groupBy, int maxResultSets, + string branch = default, string build = default, string creator = default, string name = default, @@ -55,6 +57,7 @@ public partial interface IAggregate Task> WorkItemSummaryAsync( IImmutableList groupBy, + string branch = default, string build = default, string creator = default, string name = default, @@ -75,6 +78,7 @@ public partial interface IAggregate ); Task> PropertiesAsync( + string branch = default, string build = default, string creator = default, string name = default, @@ -92,6 +96,7 @@ public partial interface IAggregate IImmutableList groupBy, int maxGroups, int maxResults, + string branch = default, string build = default, string creator = default, string name = default, @@ -134,6 +139,7 @@ public Aggregate(HelixApi client) IImmutableList groupBy, IImmutableList otherProperties, string workitem, + string branch = default, string build = default, string creator = default, string name = default, @@ -183,6 +189,10 @@ public Aggregate(HelixApi client) { _url.AppendQuery("Build", Client.Serialize(build)); } + if (!string.IsNullOrEmpty(branch)) + { + _url.AppendQuery("Branch", Client.Serialize(branch)); + } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); @@ -463,6 +473,7 @@ internal async Task OnBuildFailed(Request req, Response res) public async Task> JobSummaryAsync( IImmutableList groupBy, int maxResultSets, + string branch = default, string build = default, string creator = default, string name = default, @@ -502,6 +513,10 @@ internal async Task OnBuildFailed(Request req, Response res) { _url.AppendQuery("Build", Client.Serialize(build)); } + if (!string.IsNullOrEmpty(branch)) + { + _url.AppendQuery("Branch", Client.Serialize(branch)); + } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); @@ -574,6 +589,7 @@ internal async Task OnJobSummaryFailed(Request req, Response res) public async Task> WorkItemSummaryAsync( IImmutableList groupBy, + string branch = default, string build = default, string creator = default, string name = default, @@ -613,6 +629,10 @@ internal async Task OnJobSummaryFailed(Request req, Response res) { _url.AppendQuery("Build", Client.Serialize(build)); } + if (!string.IsNullOrEmpty(branch)) + { + _url.AppendQuery("Branch", Client.Serialize(branch)); + } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); @@ -822,6 +842,7 @@ internal async Task OnAnalysisDetailFailed(Request req, Response res) partial void HandleFailedPropertiesRequest(RestApiException ex); public async Task> PropertiesAsync( + string branch = default, string build = default, string creator = default, string name = default, @@ -856,6 +877,10 @@ internal async Task OnAnalysisDetailFailed(Request req, Response res) { _url.AppendQuery("Build", Client.Serialize(build)); } + if (!string.IsNullOrEmpty(branch)) + { + _url.AppendQuery("Branch", Client.Serialize(branch)); + } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); @@ -994,6 +1019,7 @@ internal async Task OnInvestigation_ContinueFailed(Request req, Response res) IImmutableList groupBy, int maxGroups, int maxResults, + string branch = default, string build = default, string creator = default, string name = default, @@ -1033,6 +1059,10 @@ internal async Task OnInvestigation_ContinueFailed(Request req, Response res) { _url.AppendQuery("Build", Client.Serialize(build)); } + if (!string.IsNullOrEmpty(branch)) + { + _url.AppendQuery("Branch", Client.Serialize(branch)); + } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs index 13b6133036a..7e19bebbb7f 100644 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs @@ -25,6 +25,7 @@ public partial interface IJob ); Task> ListAsync( + string branch = default, string build = default, int? count = default, string creator = default, @@ -44,6 +45,14 @@ public partial interface IJob CancellationToken cancellationToken = default ); + Task> PullRequestJobsAsync( + string organization, + int pullRequestId, + string repository, + int? count = default, + CancellationToken cancellationToken = default + ); + Task SummaryAsync( string job, CancellationToken cancellationToken = default @@ -184,6 +193,7 @@ internal async Task OnNewFailed(Request req, Response res) partial void HandleFailedListRequest(RestApiException ex); public async Task> ListAsync( + string branch = default, string build = default, int? count = default, string creator = default, @@ -219,6 +229,10 @@ internal async Task OnNewFailed(Request req, Response res) { _url.AppendQuery("Build", Client.Serialize(build)); } + if (!string.IsNullOrEmpty(branch)) + { + _url.AppendQuery("Branch", Client.Serialize(branch)); + } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); @@ -430,6 +444,105 @@ internal async Task OnPassFailFailed(Request req, Response res) throw ex; } + partial void HandleFailedPullRequestJobsRequest(RestApiException ex); + + public async Task> PullRequestJobsAsync( + string organization, + int pullRequestId, + string repository, + int? count = default, + CancellationToken cancellationToken = default + ) + { + + if (string.IsNullOrEmpty(organization)) + { + throw new ArgumentNullException(nameof(organization)); + } + + if (string.IsNullOrEmpty(repository)) + { + throw new ArgumentNullException(nameof(repository)); + } + + const string apiVersion = "2019-06-17"; + + var _baseUri = Client.Options.BaseUri; + var _url = new RequestUriBuilder(); + _url.Reset(_baseUri); + _url.AppendPath( + "/api/jobs/pullrequest-jobs", + false); + + if (!string.IsNullOrEmpty(organization)) + { + _url.AppendQuery("organization", Client.Serialize(organization)); + } + if (!string.IsNullOrEmpty(repository)) + { + _url.AppendQuery("repository", Client.Serialize(repository)); + } + if (pullRequestId != default(int)) + { + _url.AppendQuery("pullRequestId", Client.Serialize(pullRequestId)); + } + if (count != default(int?)) + { + _url.AppendQuery("count", Client.Serialize(count)); + } + _url.AppendQuery("api-version", Client.Serialize(apiVersion)); + + + using (var _req = Client.Pipeline.CreateRequest()) + { + _req.Uri = _url; + _req.Method = RequestMethod.Get; + + using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) + { + if (_res.Status < 200 || _res.Status >= 300) + { + await OnPullRequestJobsFailed(_req, _res).ConfigureAwait(false); + } + + if (_res.ContentStream == null) + { + await OnPullRequestJobsFailed(_req, _res).ConfigureAwait(false); + } + + using (var _reader = new StreamReader(_res.ContentStream)) + { + var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); + var _body = Client.Deserialize>(_content); + return _body; + } + } + } + } + + internal async Task OnPullRequestJobsFailed(Request req, Response res) + { + string content = null; + if (res.ContentStream != null) + { + using (var reader = new StreamReader(res.ContentStream)) + { + content = await reader.ReadToEndAsync().ConfigureAwait(false); + } + } + + var ex = new RestApiException( + req, + res, + content, + Client.Deserialize(content) + ); + HandleFailedPullRequestJobsRequest(ex); + HandleFailedRequest(ex); + Client.OnFailedRequest(ex); + throw ex; + } + partial void HandleFailedSummaryRequest(RestApiException ex); public async Task SummaryAsync( diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/PullRequestJobSummary.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/PullRequestJobSummary.cs new file mode 100644 index 00000000000..5d33d34d9fa --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/PullRequestJobSummary.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.Helix.Client.Models +{ + public partial class PullRequestJobSummary + { + public PullRequestJobSummary(string jobId, string status) + { + JobId = jobId; + Status = status; + } + + [JsonProperty("JobId")] + public string JobId { get; set; } + + [JsonProperty("Status")] + public string Status { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (string.IsNullOrEmpty(JobId)) + { + return false; + } + if (string.IsNullOrEmpty(Status)) + { + return false; + } + return true; + } + } + } +} From 60577b7f8b1281d75872942e9448171f21fcd9b6 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 21 Apr 2026 14:59:06 +0200 Subject: [PATCH 08/66] Download files and pass them to test publisher --- .../AzureDevOpsResultPublisher.cs | 57 +++---- .../LocalTestResultsReader.cs | 54 +------ .../Model/UploadResult.cs | 11 -- .../JobMonitor/JobMonitorRunner.cs | 153 +++++++++--------- 4 files changed, 95 insertions(+), 180 deletions(-) delete mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResult.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs index faeed58a060..9a7cbff1c46 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs @@ -23,63 +23,54 @@ public sealed class AzureDevOpsResultPublisher private static string s_lastSendContent = string.Empty; private readonly AzureDevOpsReportingParameters _azdoParameters; - private readonly string _workItemId; private readonly HttpClient _httpClient; private readonly ILogger _logger; public AzureDevOpsResultPublisher( AzureDevOpsReportingParameters azdoParameters, - string workItemId, HttpClient? httpClient = null, ILogger? logger = null) { _azdoParameters = azdoParameters; - _workItemId = workItemId; _httpClient = httpClient ?? CreateHttpClient(azdoParameters.AccessToken); _logger = logger.OrNull(); } - public async Task TryUploadAsync(IEnumerable results, CancellationToken cancellationToken = default) - { - try - { - await ProcessAsync([.. results], cancellationToken); - return UploadResult.Success; - } - catch (TerminalError ex) - { - await LogErrorAsync(ex, cancellationToken); - return UploadResult.TerminalError; - } - catch (Exception ex) - { - await LogErrorAsync(ex, cancellationToken); - return UploadResult.UnknownError; - } - } - - public async Task TryUploadDirectoryAsync(string workingDirectory, CancellationToken cancellationToken = default) + public async Task UploadDirectoryAsync(string workingDirectory, object resultMetadata, CancellationToken cancellationToken = default) { IReadOnlyList> parsedResults = new LocalTestResultsReader(_logger).ReadResults(workingDirectory); if (parsedResults.Count == 0) { _logger.LogWarning("No test results were discovered under '{WorkingDirectory}'.", workingDirectory); - return UploadResult.UnknownError; + return; } IReadOnlyList aggregatedResults = new ResultAggregator().Aggregate(parsedResults); if (aggregatedResults.Count == 0) { _logger.LogWarning("Test results were discovered under '{WorkingDirectory}', but none could be aggregated.", workingDirectory); - return UploadResult.UnknownError; + return; } - return await TryUploadAsync(aggregatedResults, cancellationToken).ConfigureAwait(false); + await UploadTestResultsAsync(aggregatedResults, resultMetadata, cancellationToken); } - private async Task ProcessAsync(IReadOnlyList testList, CancellationToken cancellationToken) + public async Task UploadTestResultsAsync(IEnumerable results, object resultMetadata, CancellationToken cancellationToken = default) { - var converted = ConvertResults(testList).ToList(); + try + { + await ProcessAsync([.. results], resultMetadata, cancellationToken); + } + catch (TerminalError ex) + { + await LogErrorAsync(ex, cancellationToken); + throw; + } + } + + private async Task ProcessAsync(IReadOnlyList testList, object resultMetadata, CancellationToken cancellationToken) + { + var converted = ConvertResults(testList, resultMetadata).ToList(); var hotPathTests = new List(); foreach (List batch in Batch(converted, 1000, static t => Size(t.Converted))) @@ -318,14 +309,8 @@ private async Task SendAttachmentAsync( _ = response; } - private IEnumerable ConvertResults(IEnumerable results) + private IEnumerable ConvertResults(IEnumerable results, object resultMetadata) { - string? comment = JsonSerializer.Serialize(new - { - HelixJobId = string.IsNullOrWhiteSpace(settings.CorrelationId) ? _workItemId : settings.CorrelationId, - HelixWorkItemName = string.IsNullOrWhiteSpace(settings.WorkItemFriendlyName) ? _workItemId : settings.WorkItemFriendlyName, - }); - static string GetResultGroupType(AggregationType aggregationType) { return aggregationType switch @@ -352,7 +337,7 @@ PublishedSubResult ConvertToSubTest(AggregatedResult result) return new PublishedSubResult { - Comment = comment ?? string.Empty, + Comment = JsonSerializer.Serialize(resultMetadata) ?? string.Empty, CustomFields = customFields, DisplayName = result.Name, Outcome = result.Result, diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/LocalTestResultsReader.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/LocalTestResultsReader.cs index c9f4283c556..807cbe9d89f 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/LocalTestResultsReader.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/LocalTestResultsReader.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Globalization; -using System.Text.Json; using System.Xml.Linq; using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; using Microsoft.Extensions.Logging; @@ -11,7 +10,6 @@ namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; public sealed class LocalTestResultsReader(ILogger? logger = null) { - private static readonly JsonSerializerOptions s_serializerOptions = new(JsonSerializerDefaults.Web); private readonly ILogger _logger = logger.OrNull(); public IReadOnlyList> ReadResults(string searchDirectory) @@ -21,31 +19,11 @@ public IReadOnlyList> ReadResults(string searchDirecto return []; } - List packedReportFiles = Directory - .EnumerateFiles(searchDirectory, "__test_report.json", SearchOption.AllDirectories) - .ToList(); - - var packedDirectories = new HashSet( - packedReportFiles - .Select(Path.GetDirectoryName) - .Where(static path => !string.IsNullOrWhiteSpace(path))!, - StringComparer.OrdinalIgnoreCase); - var allResults = new List>(); - foreach (string packedReportFile in packedReportFiles) - { - IReadOnlyList packedResults = ReadPackedResults(packedReportFile); - if (packedResults.Count > 0) - { - allResults.Add(packedResults); - } - } foreach (string filePath in Directory.EnumerateFiles(searchDirectory, "*", SearchOption.AllDirectories)) { - if (!LooksLikeTestResultFile(filePath) - || string.Equals(Path.GetFileName(filePath), "__test_report.json", StringComparison.OrdinalIgnoreCase) - || packedDirectories.Any(directory => IsPathUnderDirectory(filePath, directory))) + if (!LooksLikeTestResultFile(filePath)) { continue; } @@ -63,11 +41,6 @@ public IReadOnlyList> ReadResults(string searchDirecto public static bool LooksLikeTestResultFile(string path) { string fileName = Path.GetFileName(path); - if (string.Equals(fileName, "__test_report.json", StringComparison.OrdinalIgnoreCase)) - { - return true; - } - return fileName.EndsWith(".trx", StringComparison.OrdinalIgnoreCase) || fileName.EndsWith("testResults.xml", StringComparison.OrdinalIgnoreCase) || fileName.EndsWith("test-results.xml", StringComparison.OrdinalIgnoreCase) @@ -76,21 +49,6 @@ public static bool LooksLikeTestResultFile(string path) || fileName.EndsWith("junitresults.xml", StringComparison.OrdinalIgnoreCase); } - private IReadOnlyList ReadPackedResults(string filePath) - { - try - { - using FileStream stream = File.OpenRead(filePath); - PackedTestReport? report = JsonSerializer.Deserialize(stream, s_serializerOptions); - return report?.Results ?? []; - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to read packed test report '{Path}'.", filePath); - return []; - } - } - private IReadOnlyList ReadResultFile(string filePath) { try @@ -260,16 +218,6 @@ private static string NormalizeOutcome(string? value) }; } - private static bool IsPathUnderDirectory(string filePath, string directory) - { - string normalizedDirectory = Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)); - string normalizedPath = Path.GetFullPath(filePath); - - return normalizedPath.Equals(normalizedDirectory, StringComparison.OrdinalIgnoreCase) - || normalizedPath.StartsWith(normalizedDirectory + Path.DirectorySeparatorChar, StringComparison.OrdinalIgnoreCase) - || normalizedPath.StartsWith(normalizedDirectory + Path.AltDirectorySeparatorChar, StringComparison.OrdinalIgnoreCase); - } - private static void AddAttachmentIfNotEmpty(List attachments, string name, string? text) { if (!string.IsNullOrWhiteSpace(text)) diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResult.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResult.cs deleted file mode 100644 index 33ea210a70a..00000000000 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResult.cs +++ /dev/null @@ -1,11 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; - -public enum UploadResult -{ - Success = 1, - UnknownError = 2, - TerminalError = 3, -} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 295ad23c2bc..1e38e7b767a 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -10,11 +10,11 @@ using System.Net.Http; using System.Net.Http.Headers; using System.Text; +using System.Threading; using System.Threading.Tasks; using Azure; using Azure.Storage.Blobs; using Microsoft.DotNet.Helix.AzureDevOpsTestReporter; -using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; using Microsoft.DotNet.Helix.Client; using Microsoft.DotNet.Helix.Client.Models; using Newtonsoft.Json; @@ -46,17 +46,24 @@ public JobMonitorRunner(JobMonitorOptions options) public async Task RunAsync() { HashSet processedRuns = await GetProcessedRunNamesAsync(); - DateTimeOffset deadline = DateTimeOffset.UtcNow.AddMinutes(Math.Max(1, _options.MaximumWaitMinutes)); + + var cancellationTokenSource = new CancellationTokenSource(); + cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(_options.MaximumWaitMinutes)); + CancellationToken cancellationToken = cancellationTokenSource.Token; + bool anyNonMonitorJobFailures = false; int failedHelixJobCount = 0; int processedHelixJobCount = 0; - while (DateTimeOffset.UtcNow < deadline) + while (true) { + cancellationToken.ThrowIfCancellationRequested(); + AzureDevOpsTimelineRecord[] timelineRecords = await GetTimelineRecordsAsync(); JobStatus[] jobs = await RetryAsync( - () => Task.FromResult(Array.Empty()) - /*TODO _helixApi.PullRequests.ByBuildAsync(_options.Repository, _options.PrNumber, int.Parse(_options.BuildId, CultureInfo.InvariantCulture), _options.Attempt)*/); + // TODO async () => await _helixApi.PullRequests.ByBuildAsync(_options.Repository, _options.PrNumber, int.Parse(_options.BuildId, CultureInfo.InvariantCulture), _options.Attempt), + () => Task.FromResult(Array.Empty()), + cancellationToken); int completedHelixJobs = jobs.Count(j => j.IsCompleted); int currentFailedJobs = jobs.Count(j => j.Status.Equals("failed", StringComparison.OrdinalIgnoreCase)); @@ -69,8 +76,8 @@ public async Task RunAsync() continue; } - JobPassFail passFail = await RetryAsync(() => _helixApi.Job.PassFailAsync(job.JobName)); - bool passed = await ProcessCompletedJobAsync(job, passFail); + JobPassFail passFail = await RetryAsync(() => _helixApi.Job.PassFailAsync(job.JobName, cancellationToken), cancellationToken); + bool passed = await ProcessCompletedJobAsync(job, passFail, cancellationToken); processedRuns.Add(job.JobName); processedHelixJobCount++; if (!passed) @@ -104,11 +111,8 @@ public async Task RunAsync() return 0; } - await Task.Delay(TimeSpan.FromSeconds(Math.Max(5, _options.PollingIntervalSeconds))); + await Task.Delay(TimeSpan.FromSeconds(Math.Max(5, _options.PollingIntervalSeconds)), cancellationToken); } - - Console.Error.WriteLine($"The Helix Job Monitor timed out after {_options.MaximumWaitMinutes} minute(s)."); - return 1; } public void Dispose() @@ -116,19 +120,24 @@ public void Dispose() _azdoClient.Dispose(); } - private async Task ProcessCompletedJobAsync(JobStatus helixJob, JobPassFail passFail) + private async Task ProcessCompletedJobAsync(JobStatus helixJob, JobPassFail passFail, CancellationToken cancellationToken) { string testRunName = HelixJobMonitorUtilities.GetTestRunName(helixJob.JobName); int testRunId = await StartTestRunAsync(testRunName); - string resultsDirectory = Path.Combine(_options.WorkingDirectory, MakeSafeDirectoryName(helixJob.JobName)); + string resultsDirectory = Path.Combine(_options.WorkingDirectory, SanitizeDirName(helixJob.JobName)); Directory.CreateDirectory(resultsDirectory); - int downloadedFiles = await DownloadTestResultsAsync(helixJob.JobName, passFail, resultsDirectory); - bool reporterRan = downloadedFiles > 0 - && await TryUploadDownloadedResultsAsync(resultsDirectory, testRunId, helixJob.JobName); - if (!reporterRan) + List downloadedFiles = await DownloadTestResultsAsync(helixJob.JobName, passFail, resultsDirectory); + + try { - await CreateFallbackResultsAsync(testRunId, helixJob.JobName, passFail); + await UploadDownloadedResultsAsync(downloadedFiles, testRunId, cancellationToken); + } + catch + { + // TODO: Handle here + Console.WriteLine($"🚨 Failed to upload test results for job {helixJob.JobName} to Azure DevOps. Test run ID was {testRunId}."); + return false; } await StopTestRunAsync(testRunId, testRunName); @@ -144,7 +153,7 @@ private async Task> GetProcessedRunNamesAsync() JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?buildIds={_options.BuildId}&api-version=7.1-preview.1"); var processed = new HashSet(StringComparer.OrdinalIgnoreCase); - foreach (JObject run in data?["value"] as JArray ?? []) + foreach (JObject run in (data?["value"] as JArray ?? []).Cast()) { string name = run.Value("name"); string state = run.Value("state"); @@ -191,62 +200,38 @@ await SendAsync( Console.WriteLine($"Stopped test run '{testRunName}'."); } - private async Task CreateFallbackResultsAsync(int testRunId, string jobName, JobPassFail passFail) + private async Task> DownloadTestResultsAsync( + string jobName, + JobPassFail passFail, + string outputDirectory, + CancellationToken cancellationToken) { - foreach (string workItemName in passFail.Passed ?? ImmutableList.Empty) - { - await CreateFallbackResultAsync(testRunId, jobName, workItemName, failed: false); - } + List downloadedFiles = []; - foreach (string workItemName in passFail.Failed ?? ImmutableList.Empty) - { - await CreateFallbackResultAsync(testRunId, jobName, workItemName, failed: true); - } - } + JobResultsUri resultsUri = await RetryAsync( + () => _helixApi.Job.ResultsAsync(jobName), + cancellationToken); - private async Task CreateFallbackResultAsync(int testRunId, string jobName, string workItemName, bool failed) - { - string cleanName = HelixJobMonitorUtilities.CleanWorkItemName(workItemName); - await SendAsync( - HttpMethod.Post, - $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/Runs/{testRunId}/results?api-version=5.1-preview.6", - new JArray - { - new JObject - { - ["automatedTestName"] = $"{cleanName}.WorkItemExecution", - ["automatedTestStorage"] = cleanName, - ["testCaseTitle"] = $"{cleanName} Work Item", - ["outcome"] = failed ? "Failed" : "Passed", - ["state"] = "Completed", - ["errorMessage"] = failed ? "The Helix work item failed. See the Helix logs for more details." : null, - ["durationInMs"] = 60 * 1000, - ["comment"] = new JObject - { - ["HelixJobId"] = jobName, - ["HelixWorkItemName"] = cleanName, - }.ToString(), - } - }); - } + IEnumerable workItemNames = (passFail.Passed ?? []) + .Concat(passFail.Failed ?? []) + .Distinct(StringComparer.OrdinalIgnoreCase); - private async Task DownloadTestResultsAsync(string jobName, JobPassFail passFail, string outputDirectory) - { - int count = 0; - JobResultsUri resultsUri = await RetryAsync(() => _helixApi.Job.ResultsAsync(jobName)); - IEnumerable workItemNames = (passFail.Passed ?? ImmutableList.Empty).Concat(passFail.Failed ?? ImmutableList.Empty); - - foreach (string workItemName in workItemNames.Distinct(StringComparer.OrdinalIgnoreCase)) + foreach (string workItemName in workItemNames) { - IImmutableList availableFiles = await RetryAsync(() => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false)); - string workItemDirectory = Path.Combine(outputDirectory, MakeSafeDirectoryName(workItemName)); + IImmutableList availableFiles = await RetryAsync( + () => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false), + cancellationToken); + + string workItemDirectory = Path.Combine(outputDirectory, SanitizeDirName(workItemName)); Directory.CreateDirectory(workItemDirectory); + List workItemFiles = []; foreach (UploadedFile file in availableFiles.Where(f => LooksLikeTestResultFile(f.Name))) { string relativePath = file.Name.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar); string destinationFile = Path.Combine(workItemDirectory, relativePath); string directory = Path.GetDirectoryName(destinationFile); + if (!string.IsNullOrEmpty(directory)) { Directory.CreateDirectory(directory); @@ -254,37 +239,41 @@ private async Task DownloadTestResultsAsync(string jobName, JobPassFail pas try { + Console.WriteLine($"Downloading {file.Name} for work item {workItemName} in job {jobName}..."); + BlobClient blobClient = CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS); - await blobClient.DownloadToAsync(destinationFile); - count++; + await blobClient.DownloadToAsync(destinationFile, cancellationToken); + workItemFiles.Add(destinationFile); } catch (Exception ex) { Console.WriteLine($"Warning: failed to download '{file.Name}' for '{jobName}/{workItemName}': {ex.Message}"); } } + + downloadedFiles.Add(new WorkItemTestResults(jobName, workItemName, workItemFiles)); } - return count; + return downloadedFiles; } - private async Task TryUploadDownloadedResultsAsync(string workingDirectory, int testRunId, string jobName) + private async Task UploadDownloadedResultsAsync(WorkItemTestResults testResults, int testRunId, CancellationToken cancellationToken) { var publisher = new AzureDevOpsResultPublisher( new AzureDevOpsReportingParameters( new Uri(_options.CollectionUri, UriKind.Absolute), _options.TeamProject, testRunId.ToString(CultureInfo.InvariantCulture), - _options.SystemAccessToken), - jobName); - - UploadResult uploadResult = await publisher.TryUploadDirectoryAsync(workingDirectory); - if (uploadResult != UploadResult.Success) - { - Console.WriteLine($"Warning: test result upload for '{jobName}' returned '{uploadResult}'. Falling back to synthetic work-item results."); - } + _options.SystemAccessToken)); - return uploadResult == UploadResult.Success; + await publisher.UploadTestResultsAsync( + testResults.TestResultFiles, + new + { + HelixJobId = testResults.JobName, + HelixWorkItemName = testResults.WorkItemName, + }, + cancellationToken); } private async Task SendAsync(HttpMethod method, string requestUri, JToken body = null) @@ -306,7 +295,7 @@ private async Task SendAsync(HttpMethod method, string requestUri, JTok if (string.IsNullOrWhiteSpace(content)) { - return new JObject(); + return []; } return JObject.Parse(content); @@ -330,7 +319,7 @@ private static BlobClient CreateBlobClient(string fileLink, string resultsSas) private static bool LooksLikeTestResultFile(string path) => LocalTestResultsReader.LooksLikeTestResultFile(path); - private static string MakeSafeDirectoryName(string value) + private static string SanitizeDirName(string value) { foreach (char invalidChar in Path.GetInvalidFileNameChars()) { @@ -340,11 +329,13 @@ private static string MakeSafeDirectoryName(string value) return value; } - private static async Task RetryAsync(Func> action) + private static async Task RetryAsync(Func> action, CancellationToken cancellationToken) { Exception last = null; for (int attempt = 0; attempt < 5; attempt++) { + cancellationToken.ThrowIfCancellationRequested(); + try { return await action(); @@ -352,11 +343,13 @@ private static async Task RetryAsync(Func> action) catch (Exception ex) when (attempt < 4) { last = ex; - await Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, attempt + 1))); + await Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, attempt + 1)), cancellationToken); } } throw last ?? new InvalidOperationException("Retry failed without capturing an exception."); } } + + record WorkItemTestResults(string JobName, string WorkItemName, List TestResultFiles); } From 777de0466afa58fbdd7ebd57e95caef5ed0cb655 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 21 Apr 2026 15:01:40 +0200 Subject: [PATCH 09/66] Rename to AzureDevOpsTestPublisher --- Arcade.slnx | 2 +- .../AzureDevOpsResultPublisher.cs | 4 +-- .../LocalTestResultsReader.cs | 32 ++----------------- ...Net.Helix.AzureDevOpsTestPublisher.csproj} | 0 .../Model/AzureDevOpsReportingError.cs | 2 +- .../Model/PackedTestReport.cs | 2 +- .../Model/TerminalError.cs | 2 +- .../Model/TestResult.cs | 2 +- .../Model/TestResultAttachment.cs | 2 +- .../Model/UploadResultExtensions.cs | 2 +- .../PackingTestReporter.cs | 4 +-- .../ResultAggregator.cs | 4 +-- .../TestReportingModels.cs | 4 +-- .../JobMonitor/JobMonitorRunner.cs | 2 +- .../Microsoft.DotNet.Helix.JobMonitor.csproj | 2 +- .../LocalTestResultsReaderTests.cs | 4 +-- .../Microsoft.DotNet.Helix.Sdk.Tests.csproj | 2 +- 17 files changed, 23 insertions(+), 49 deletions(-) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter => AzureDevOpsTestPublisher}/AzureDevOpsResultPublisher.cs (99%) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter => AzureDevOpsTestPublisher}/LocalTestResultsReader.cs (91%) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter/Microsoft.DotNet.Helix.AzureDevOpsTestReporter.csproj => AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj} (100%) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter => AzureDevOpsTestPublisher}/Model/AzureDevOpsReportingError.cs (77%) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter => AzureDevOpsTestPublisher}/Model/PackedTestReport.cs (80%) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter => AzureDevOpsTestPublisher}/Model/TerminalError.cs (76%) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter => AzureDevOpsTestPublisher}/Model/TestResult.cs (94%) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter => AzureDevOpsTestPublisher}/Model/TestResultAttachment.cs (76%) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter => AzureDevOpsTestPublisher}/Model/UploadResultExtensions.cs (84%) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter => AzureDevOpsTestPublisher}/PackingTestReporter.cs (96%) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter => AzureDevOpsTestPublisher}/ResultAggregator.cs (98%) rename src/Microsoft.DotNet.Helix/{AzureDevOpsTestReporter => AzureDevOpsTestPublisher}/TestReportingModels.cs (81%) diff --git a/Arcade.slnx b/Arcade.slnx index 975b61a3e5d..923cb6e046d 100644 --- a/Arcade.slnx +++ b/Arcade.slnx @@ -5,7 +5,7 @@ - + diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs similarity index 99% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs index 9a7cbff1c46..4f3dc0a90ac 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs @@ -7,10 +7,10 @@ using System.Security.Cryptography; using System.Text; using System.Text.Json; -using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Microsoft.Extensions.Logging; -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; public sealed class AzureDevOpsResultPublisher { diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/LocalTestResultsReader.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs similarity index 91% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/LocalTestResultsReader.cs rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs index 807cbe9d89f..cd1c4482de0 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/LocalTestResultsReader.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs @@ -3,41 +3,15 @@ using System.Globalization; using System.Xml.Linq; -using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Microsoft.Extensions.Logging; -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; public sealed class LocalTestResultsReader(ILogger? logger = null) { private readonly ILogger _logger = logger.OrNull(); - public IReadOnlyList> ReadResults(string searchDirectory) - { - if (string.IsNullOrWhiteSpace(searchDirectory) || !Directory.Exists(searchDirectory)) - { - return []; - } - - var allResults = new List>(); - - foreach (string filePath in Directory.EnumerateFiles(searchDirectory, "*", SearchOption.AllDirectories)) - { - if (!LooksLikeTestResultFile(filePath)) - { - continue; - } - - IReadOnlyList parsed = ReadResultFile(filePath); - if (parsed.Count > 0) - { - allResults.Add(parsed); - } - } - - return allResults; - } - public static bool LooksLikeTestResultFile(string path) { string fileName = Path.GetFileName(path); @@ -49,7 +23,7 @@ public static bool LooksLikeTestResultFile(string path) || fileName.EndsWith("junitresults.xml", StringComparison.OrdinalIgnoreCase); } - private IReadOnlyList ReadResultFile(string filePath) + public IReadOnlyList ReadResultFile(string filePath) { try { diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Microsoft.DotNet.Helix.AzureDevOpsTestReporter.csproj b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj similarity index 100% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Microsoft.DotNet.Helix.AzureDevOpsTestReporter.csproj rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/AzureDevOpsReportingError.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingError.cs similarity index 77% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/AzureDevOpsReportingError.cs rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingError.cs index 2d541ab043d..90af5a10e36 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/AzureDevOpsReportingError.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingError.cs @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; public sealed class AzureDevOpsReportingError(string message) : Exception(message) { diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/PackedTestReport.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/PackedTestReport.cs similarity index 80% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/PackedTestReport.cs rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/PackedTestReport.cs index ac93f7efb25..1cee30425a4 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/PackedTestReport.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/PackedTestReport.cs @@ -1,6 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; public sealed record PackedTestReport(AzureDevOpsReportingParameters AzdoParameters, IReadOnlyList Results); diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TerminalError.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TerminalError.cs similarity index 76% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TerminalError.cs rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TerminalError.cs index 46fc04f2f6d..174d20afaee 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TerminalError.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TerminalError.cs @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; internal sealed class TerminalError(string message) : Exception(message) { diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResult.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResult.cs similarity index 94% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResult.cs rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResult.cs index 17d29681bb2..61904efb6c6 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResult.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResult.cs @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; public sealed class TestResult( string name, diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResultAttachment.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResultAttachment.cs similarity index 76% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResultAttachment.cs rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResultAttachment.cs index afe26a86688..d5f1522b550 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/TestResultAttachment.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/TestResultAttachment.cs @@ -1,6 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; public sealed record TestResultAttachment(string Name, string Text); diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResultExtensions.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/UploadResultExtensions.cs similarity index 84% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResultExtensions.cs rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/UploadResultExtensions.cs index cef847051e6..f10191f03dc 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/Model/UploadResultExtensions.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/UploadResultExtensions.cs @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; public static class UploadResultExtensions { diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/PackingTestReporter.cs similarity index 96% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/PackingTestReporter.cs index 1f58470a679..658fbf9a41a 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/PackingTestReporter.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/PackingTestReporter.cs @@ -2,10 +2,10 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Text.Json; -using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Microsoft.Extensions.Logging; -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; public interface ITestReporter { diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/ResultAggregator.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/ResultAggregator.cs similarity index 98% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/ResultAggregator.cs rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/ResultAggregator.cs index 2a2a15b0384..2526ca9d0f3 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/ResultAggregator.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/ResultAggregator.cs @@ -1,9 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; public enum AggregationType { diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestReportingModels.cs similarity index 81% rename from src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs rename to src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestReportingModels.cs index 9ec5c84ec5a..e9637f8546b 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestReporter/TestReportingModels.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestReportingModels.cs @@ -1,11 +1,11 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging.Abstractions; -namespace Microsoft.DotNet.Helix.AzureDevOpsTestReporter; +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; public sealed record AzureDevOpsReportingParameters(Uri CollectionUri, string TeamProject, string TestRunId, string? AccessToken = null); diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 1e38e7b767a..4b06b1a12a1 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -14,7 +14,7 @@ using System.Threading.Tasks; using Azure; using Azure.Storage.Blobs; -using Microsoft.DotNet.Helix.AzureDevOpsTestReporter; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; using Microsoft.DotNet.Helix.Client; using Microsoft.DotNet.Helix.Client.Models; using Newtonsoft.Json; diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj index cf096b3f232..a4173c27ef1 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj @@ -16,7 +16,7 @@ - + diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs index 52559f16586..8d576cafacf 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs @@ -5,8 +5,8 @@ using System.IO; using System.Linq; using System.Threading.Tasks; -using Microsoft.DotNet.Helix.AzureDevOpsTestReporter; -using Microsoft.DotNet.Helix.AzureDevOpsTestReporter.Model; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Xunit; namespace Microsoft.DotNet.Helix.Sdk.Tests diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj index 2d2dfa4c78a..d3ae66e3729 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj @@ -21,7 +21,7 @@ - + From 9d564b4c56069f57c2c5def68e2b43106f17937b Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 21 Apr 2026 15:29:47 +0200 Subject: [PATCH 10/66] Revert Helix Client changes --- .../Client/CSharp/Microsoft.DotNet.Helix.Client.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj b/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj index f2ee42f758d..061f83a5238 100644 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj @@ -2,10 +2,10 @@ - net10.0 + $(BundledNETCoreAppTargetFramework) true This package provides access to the Helix Api located at https://helix.dot.net/ - https://helix.int-dot.net/api/openapi.json + https://helix.dot.net/api/openapi.json HelixApi README.md From 76b28ff45783cea2d4017e8e2f26c00fd0d09da6 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Tue, 21 Apr 2026 18:17:54 +0200 Subject: [PATCH 11/66] WIP - read test results from the downloaded files --- .../AzureDevOpsResultPublisher.cs | 21 +++-- .../LocalTestResultsReader.cs | 5 +- .../Model/UploadResultExtensions.cs | 12 --- .../PackingTestReporter.cs | 94 ------------------- .../JobMonitor/JobMonitorRunner.cs | 31 +++--- .../Microsoft.DotNet.Helix.JobMonitor.csproj | 2 +- .../LocalTestResultsReaderTests.cs | 65 ++----------- .../Microsoft.DotNet.Helix.Sdk.Tests.csproj | 2 +- 8 files changed, 44 insertions(+), 188 deletions(-) delete mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/UploadResultExtensions.cs delete mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/PackingTestReporter.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs index 4f3dc0a90ac..4ae4990cf9c 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs @@ -36,19 +36,22 @@ public AzureDevOpsResultPublisher( _logger = logger.OrNull(); } - public async Task UploadDirectoryAsync(string workingDirectory, object resultMetadata, CancellationToken cancellationToken = default) + public async Task UploadTestResultsAsync(List testResultFiles, object resultMetadata, CancellationToken cancellationToken = default) { - IReadOnlyList> parsedResults = new LocalTestResultsReader(_logger).ReadResults(workingDirectory); - if (parsedResults.Count == 0) + var testResultReader = new LocalTestResultsReader(_logger); + + Task>[] parseTasks = [.. testResultFiles.Select(file => testResultReader.ReadResultFileAsync(file, cancellationToken))]; + IReadOnlyList[] parsedResults = await Task.WhenAll(parseTasks); + if (parsedResults.Length == 0) { - _logger.LogWarning("No test results were discovered under '{WorkingDirectory}'.", workingDirectory); + _logger.LogWarning("No test results were discovered under."); return; } IReadOnlyList aggregatedResults = new ResultAggregator().Aggregate(parsedResults); if (aggregatedResults.Count == 0) { - _logger.LogWarning("Test results were discovered under '{WorkingDirectory}', but none could be aggregated.", workingDirectory); + _logger.LogWarning("Test results were discovered but none could be aggregated."); return; } @@ -322,6 +325,8 @@ static string GetResultGroupType(AggregationType aggregationType) }; } + string comment = JsonSerializer.Serialize(resultMetadata) ?? string.Empty; + PublishedSubResult ConvertToSubTest(AggregatedResult result) { var customFields = new List(); @@ -337,7 +342,7 @@ PublishedSubResult ConvertToSubTest(AggregatedResult result) return new PublishedSubResult { - Comment = JsonSerializer.Serialize(resultMetadata) ?? string.Empty, + Comment = comment, CustomFields = customFields, DisplayName = result.Name, Outcome = result.Result, @@ -368,12 +373,12 @@ ConvertedResult ConvertResult(AggregatedResult result) TestCaseTitle = result.Name, AutomatedTestName = result.Name, AutomatedTestType = "helix", - AutomatedTestStorage = _workItemId, + AutomatedTestStorage = comment, // TODO: This was workitem ID Priority = 1, DurationInMs = result.DurationSeconds * 1000.0, Outcome = result.Result, State = "Completed", - Comment = comment ?? string.Empty, + Comment = comment, StackTrace = result.StackTrace, ErrorMessage = result.FailureMessage, SubResults = result.SubResults.Count == 0 ? null : [.. result.SubResults.Select(ConvertToSubTest)], diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs index cd1c4482de0..d7a1d72c294 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs @@ -23,11 +23,12 @@ public static bool LooksLikeTestResultFile(string path) || fileName.EndsWith("junitresults.xml", StringComparison.OrdinalIgnoreCase); } - public IReadOnlyList ReadResultFile(string filePath) + public async Task> ReadResultFileAsync(string filePath, CancellationToken cancellationToken = default) { try { - XDocument document = XDocument.Load(filePath, LoadOptions.PreserveWhitespace); + using FileStream stream = File.OpenRead(filePath); + XDocument document = await XDocument.LoadAsync(stream, LoadOptions.PreserveWhitespace, cancellationToken); string rootName = document.Root?.Name.LocalName ?? string.Empty; string workItemName = new DirectoryInfo(Path.GetDirectoryName(filePath) ?? string.Empty).Name; diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/UploadResultExtensions.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/UploadResultExtensions.cs deleted file mode 100644 index f10191f03dc..00000000000 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/UploadResultExtensions.cs +++ /dev/null @@ -1,12 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; - -public static class UploadResultExtensions -{ - public static UploadResult Aggregate(this UploadResult value, UploadResult other) - { - return (UploadResult)Math.Max((int)value, (int)other); - } -} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/PackingTestReporter.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/PackingTestReporter.cs deleted file mode 100644 index 658fbf9a41a..00000000000 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/PackingTestReporter.cs +++ /dev/null @@ -1,94 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System.Text.Json; -using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; -using Microsoft.Extensions.Logging; - -namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; - -public interface ITestReporter -{ - Task ReportResultsAsync(IReadOnlyList results, CancellationToken cancellationToken = default); -} - -public sealed class PackingTestReporter(AzureDevOpsReportingParameters azdoParameters, ILogger? logger = null) - : ITestReporter -{ - private const string ReportFileName = "__test_report.json"; - private static readonly JsonSerializerOptions s_serializerOptions = new(JsonSerializerDefaults.Web); - - private readonly AzureDevOpsReportingParameters _azdoParameters = azdoParameters; - private readonly ILogger _logger = logger.OrNull(); - - public async Task ReportResultsAsync(IReadOnlyList results, CancellationToken cancellationToken = default) - { - var filtered = (results ?? []) - .Where(static x => x is not null) - .ToList(); - - var serialized = new PackedTestReport(_azdoParameters, filtered); - string path = Path.Combine(Environment.CurrentDirectory, ReportFileName); - string? directory = Path.GetDirectoryName(path); - if (!string.IsNullOrEmpty(directory)) - { - Directory.CreateDirectory(directory); - } - - _logger.LogInformation("Packing {Count} test reports to '{Path}'", filtered.Count, path); - - await using (FileStream saveFile = File.Create(path)) - { - await JsonSerializer.SerializeAsync(saveFile, serialized, s_serializerOptions, cancellationToken); - await saveFile.FlushAsync(cancellationToken); - } - - _logger.LogInformation("Packed {Length} bytes", new FileInfo(path).Length); - } - - public static async Task<(AzureDevOpsReportingParameters Parameters, IReadOnlyList Results)?> UnpackResultsAsync( - string? searchDirectory = null, - ILogger? logger = null, - CancellationToken cancellationToken = default) - { - ILogger effectiveLogger = logger.OrNull(); - string? path = null; - - if (!string.IsNullOrWhiteSpace(searchDirectory)) - { - path = Path.Combine(searchDirectory, ReportFileName); - if (!File.Exists(path) && Directory.Exists(searchDirectory)) - { - path = Directory.EnumerateFiles(searchDirectory, ReportFileName, SearchOption.AllDirectories).FirstOrDefault(); - } - } - - if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) - { - path = Path.Combine(Environment.CurrentDirectory, ReportFileName); - - if (!File.Exists(path) && !string.IsNullOrWhiteSpace(settings.WorkitemPayloadDir)) - { - path = Path.Combine(settings.WorkitemPayloadDir, ReportFileName); - } - } - - if (string.IsNullOrWhiteSpace(path) || !File.Exists(path)) - { - return null; - } - - effectiveLogger.LogInformation("Unpacking {Length} bytes from '{Path}'", new FileInfo(path).Length, path); - - await using FileStream saveFile = File.OpenRead(path); - PackedTestReport? serialized = await JsonSerializer.DeserializeAsync(saveFile, s_serializerOptions, cancellationToken); - if (serialized is null) - { - effectiveLogger.LogError("Unpacked tests were null or invalid."); - return null; - } - - effectiveLogger.LogInformation("Unpacked {Count} test reports", serialized.Results.Count); - return (serialized.AzdoParameters, serialized.Results); - } -} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 4b06b1a12a1..882b0db3e42 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -127,10 +127,9 @@ private async Task ProcessCompletedJobAsync(JobStatus helixJob, JobPassFai string resultsDirectory = Path.Combine(_options.WorkingDirectory, SanitizeDirName(helixJob.JobName)); Directory.CreateDirectory(resultsDirectory); - List downloadedFiles = await DownloadTestResultsAsync(helixJob.JobName, passFail, resultsDirectory); - try { + List downloadedFiles = await DownloadTestResultsAsync(helixJob.JobName, passFail, resultsDirectory, cancellationToken); await UploadDownloadedResultsAsync(downloadedFiles, testRunId, cancellationToken); } catch @@ -257,7 +256,7 @@ private async Task> DownloadTestResultsAsync( return downloadedFiles; } - private async Task UploadDownloadedResultsAsync(WorkItemTestResults testResults, int testRunId, CancellationToken cancellationToken) + private async Task UploadDownloadedResultsAsync(List testResults, int testRunId, CancellationToken cancellationToken) { var publisher = new AzureDevOpsResultPublisher( new AzureDevOpsReportingParameters( @@ -266,17 +265,21 @@ private async Task UploadDownloadedResultsAsync(WorkItemTestResults testResults, testRunId.ToString(CultureInfo.InvariantCulture), _options.SystemAccessToken)); - await publisher.UploadTestResultsAsync( - testResults.TestResultFiles, - new - { - HelixJobId = testResults.JobName, - HelixWorkItemName = testResults.WorkItemName, - }, + foreach (WorkItemTestResults workItemTestResult in testResults) + { + await publisher.UploadTestResultsAsync( + workItemTestResult.TestResultFiles, + // Metadata that will be appended to each test case + new + { + HelixJobId = workItemTestResult.JobName, + HelixWorkItemName = workItemTestResult.WorkItemName, + }, cancellationToken); + } } - private async Task SendAsync(HttpMethod method, string requestUri, JToken body = null) + private async Task SendAsync(HttpMethod method, string requestUri, JToken body = null, CancellationToken cancellationToken = default) { return await RetryAsync(async () => { @@ -286,8 +289,8 @@ private async Task SendAsync(HttpMethod method, string requestUri, JTok request.Content = new StringContent(body.ToString(Formatting.None), Encoding.UTF8, "application/json"); } - using HttpResponseMessage response = await _azdoClient.SendAsync(request); - string content = response.Content != null ? await response.Content.ReadAsStringAsync() : null; + using HttpResponseMessage response = await _azdoClient.SendAsync(request, cancellationToken); + string content = response.Content != null ? await response.Content.ReadAsStringAsync(cancellationToken) : null; if (!response.IsSuccessStatusCode) { throw new HttpRequestException($"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}"); @@ -299,7 +302,7 @@ private async Task SendAsync(HttpMethod method, string requestUri, JTok } return JObject.Parse(content); - }); + }, cancellationToken); } private static BlobClient CreateBlobClient(string fileLink, string resultsSas) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj index a4173c27ef1..9e1acc6d5b6 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj @@ -16,7 +16,7 @@ - + diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs index 8d576cafacf..d2b52176931 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs @@ -2,8 +2,8 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Collections.Generic; using System.IO; -using System.Linq; using System.Threading.Tasks; using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; @@ -14,50 +14,7 @@ namespace Microsoft.DotNet.Helix.Sdk.Tests public class LocalTestResultsReaderTests { [Fact] - public async Task PackingTestReporter_CanUnpackFromSpecifiedDirectory() - { - string tempDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); - Directory.CreateDirectory(tempDirectory); - string originalDirectory = Environment.CurrentDirectory; - - try - { - Environment.CurrentDirectory = tempDirectory; - - var reporter = new PackingTestReporter( - new AzureDevOpsReportingParameters(new Uri("https://dev.azure.com/dnceng/"), "arcade", "42")); - - await reporter.ReportResultsAsync( - [ - new TestResult( - "Sample.Tests.Passes", - "unit", - "Sample.Tests", - "Passes", - 1.25, - "Pass", - null, - null, - null, - null) - ]); - - var unpacked = await PackingTestReporter.UnpackResultsAsync(tempDirectory); - - Assert.True(unpacked.HasValue); - Assert.Equal("42", unpacked.Value.Parameters.TestRunId); - Assert.Single(unpacked.Value.Results); - Assert.Equal("Sample.Tests.Passes", unpacked.Value.Results[0].Name); - } - finally - { - Environment.CurrentDirectory = originalDirectory; - Directory.Delete(tempDirectory, recursive: true); - } - } - - [Fact] - public void LocalTestResultsReader_ReadsXunitFileFromDownloadedResults() + public async Task LocalTestResultsReader_ReadsXunitFileFromDownloadedResults() { string tempDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString("N")); string workItemDirectory = Path.Combine(tempDirectory, "work-item"); @@ -78,8 +35,9 @@ public void LocalTestResultsReader_ReadsXunitFileFromDownloadedResults() """); var reader = new LocalTestResultsReader(); - var resultSets = reader.ReadResults(tempDirectory); - var aggregate = new ResultAggregator().Aggregate(resultSets); + string filePath = Path.Combine(workItemDirectory, "testResults.xml"); + IReadOnlyList resultSets = await reader.ReadResultFileAsync(filePath); + IReadOnlyList aggregate = new ResultAggregator().Aggregate([resultSets]); AggregatedResult result = Assert.Single(aggregate); Assert.Equal("Sample.Tests.Passes", result.Name); @@ -104,15 +62,10 @@ public async Task LocalTestResultsReader_CombinesPackedAndXmlResultsAcrossWorkIt try { Environment.CurrentDirectory = packedDirectory; - var reporter = new PackingTestReporter( - new AzureDevOpsReportingParameters(new Uri("https://dev.azure.com/dnceng/"), "arcade", "42")); - await reporter.ReportResultsAsync( - [ - new TestResult("Packed.Tests.Passes", "unit", "Packed.Tests", "Passes", 1, "Pass", null, null, null, null) - ]); + string filePath = Path.Combine(xmlDirectory, "testResults.xml"); File.WriteAllText( - Path.Combine(xmlDirectory, "testResults.xml"), + filePath, """ @@ -123,8 +76,8 @@ await reporter.ReportResultsAsync( """); - var resultSets = new LocalTestResultsReader().ReadResults(tempDirectory); - var aggregate = new ResultAggregator().Aggregate(resultSets); + IReadOnlyList resultSets = await new LocalTestResultsReader().ReadResultFileAsync(filePath); + IReadOnlyList aggregate = new ResultAggregator().Aggregate([resultSets]); Assert.Equal(2, aggregate.Count); Assert.Contains(aggregate, static x => x.Name == "Packed.Tests.Passes"); diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj index d3ae66e3729..9a9eab6b519 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests.csproj @@ -21,7 +21,7 @@ - + From 3d77bd486c620ce29a3d10dd1a8774aee7821fd3 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 09:57:08 +0200 Subject: [PATCH 12/66] Add org + repo name options separately --- .../JobMonitor/HelixJobMonitorUtilities.cs | 54 ------------------- .../JobMonitor/JobMonitorOptions.cs | 40 ++++++++------ .../HelixJobMonitorUtilitiesTests.cs | 10 ---- 3 files changed, 25 insertions(+), 79 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs b/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs index 4e1afde8e85..86e9b18bbea 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Linq; -using System.Net; using Newtonsoft.Json; namespace Microsoft.DotNet.Helix.JobMonitor @@ -32,44 +31,6 @@ public sealed class AzureDevOpsTimelineRecord public static class HelixJobMonitorUtilities { - public static string NormalizeRepository(string repository) - { - if (string.IsNullOrWhiteSpace(repository)) - { - return string.Empty; - } - - repository = repository.Trim().TrimEnd('/'); - if (!Uri.TryCreate(repository, UriKind.Absolute, out Uri uri)) - { - return repository.Trim('/'); - } - - string[] segments = uri.AbsolutePath.Split(['/'], StringSplitOptions.RemoveEmptyEntries); - if (uri.Host.Contains("github.com", StringComparison.OrdinalIgnoreCase) && segments.Length >= 2) - { - return $"{segments[0]}/{segments[1]}"; - } - - int gitIndex = Array.FindIndex(segments, s => string.Equals(s, "_git", StringComparison.OrdinalIgnoreCase)); - if (gitIndex > 0 && segments.Length > gitIndex + 1) - { - string project = segments[gitIndex - 1]; - string repoName = segments[gitIndex + 1]; - if ((string.Equals(project, "internal", StringComparison.OrdinalIgnoreCase) - || string.Equals(project, "public", StringComparison.OrdinalIgnoreCase)) - && repoName.Contains('-', StringComparison.Ordinal)) - { - int separatorIndex = repoName.IndexOf('-', StringComparison.Ordinal); - return $"{repoName.Substring(0, separatorIndex)}/{repoName.Substring(separatorIndex + 1)}"; - } - - return $"{project}/{repoName}"; - } - - return repository; - } - public static bool AreNonMonitorJobsComplete(IEnumerable records, string jobMonitorName) => GetRelevantJobRecords(records, jobMonitorName).All(IsTerminal); @@ -81,21 +42,6 @@ public static bool HasFailedNonMonitorJobs(IEnumerable $"Helix Job Monitor - {helixJobName}"; - public static string CleanWorkItemName(string name) - { - if (string.IsNullOrEmpty(name)) - { - return string.Empty; - } - - if (!name.Contains('%')) - { - name = WebUtility.UrlDecode(name); - } - - return name.Replace('/', '-').Replace('\\', '-'); - } - private static IEnumerable GetRelevantJobRecords(IEnumerable records, string jobMonitorName) { return (records ?? []) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs index b2b037bde5e..cb3f7eddb9c 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs @@ -19,8 +19,17 @@ public sealed class JobMonitorOptions public bool ShowHelp { get; private set; } - [Option("helix-base-uri", HelpText = "Base URI for the Helix service.")] - public string HelixBaseUri { get; set; } = "https://helix.dot.net/"; + [Option("organization", HelpText = "Organization name (e.g. 'dotnet' for 'dotnet/runtime').")] + public string Organization { get; set; } + + [Option("repository", HelpText = "Repository name (e.g. 'runtime' for 'dotnet/runtime').")] + public string RepositoryName { get; set; } + + [Option("pr-number", HelpText = "Pull request number for the build, if applicable.")] + public int? PrNumber { get; set; } + + [Option("build-id", HelpText = "Azure DevOps build ID.")] + public string BuildId { get; set; } [Option("collection-uri", HelpText = "Azure DevOps collection URI.")] public string CollectionUri { get; set; } @@ -28,11 +37,8 @@ public sealed class JobMonitorOptions [Option("team-project", HelpText = "Azure DevOps team project name.")] public string TeamProject { get; set; } - [Option("build-id", HelpText = "Azure DevOps build ID.")] - public string BuildId { get; set; } - - [Option("repository", HelpText = "Repository identifier in owner/repo form.")] - public string Repository { get; set; } + [Option("helix-base-uri", HelpText = "Base URI for the Helix service.")] + public string HelixBaseUri { get; set; } = "https://helix.dot.net/"; [Option("polling-interval-seconds", HelpText = "Polling interval in seconds.", Default = 30)] public int PollingIntervalSeconds { get; set; } = 30; @@ -46,9 +52,6 @@ public sealed class JobMonitorOptions [Option("working-directory", HelpText = "Directory used to stage downloaded test results.")] public string WorkingDirectory { get; set; } - [Option("pr-number", HelpText = "Pull request number for the build, if applicable.")] - public int? PrNumber { get; set; } - [Option("attempt", HelpText = "Azure DevOps attempt number for the current job.")] public int? Attempt { get; set; } @@ -85,10 +88,7 @@ private void ApplyEnvironmentDefaults() TeamProject ??= Environment.GetEnvironmentVariable("SYSTEM_TEAMPROJECT"); BuildId ??= Environment.GetEnvironmentVariable("BUILD_BUILDID"); SystemAccessToken ??= Environment.GetEnvironmentVariable("SYSTEM_ACCESSTOKEN"); - Repository = HelixJobMonitorUtilities.NormalizeRepository( - Repository - ?? Environment.GetEnvironmentVariable("BUILD_REPOSITORY_URI") - ?? Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME")); + RepositoryName ??= Environment.GetEnvironmentVariable("BUILD_REPOSITORY_NAME"); WorkingDirectory ??= System.IO.Path.Combine(System.IO.Path.GetTempPath(), "helix-job-monitor", BuildId ?? "unknown"); PrNumber ??= GetEnvironmentInt("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"); Attempt ??= GetEnvironmentInt("SYSTEM_JOBATTEMPT"); @@ -101,10 +101,20 @@ private void Validate() BuildId = RequireValue(BuildId, "build-id", "BUILD_BUILDID"); SystemAccessToken = RequireValue(SystemAccessToken, "access-token", "SYSTEM_ACCESSTOKEN"); - if (string.IsNullOrWhiteSpace(Repository)) + if (string.IsNullOrWhiteSpace(RepositoryName)) { throw new InvalidOperationException("A repository identifier must be provided either by argument or pipeline environment."); } + + if (string.IsNullOrWhiteSpace(Organization)) + { + throw new InvalidOperationException("Organization must be provided either by argument or pipeline environment."); + } + + if (!PrNumber.HasValue) + { + throw new InvalidOperationException("Pull request number must be provided either by argument or pipeline environment."); + } } private static string RequireValue(string value, string argumentName, string environmentName) diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixJobMonitorUtilitiesTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixJobMonitorUtilitiesTests.cs index ade2e952d09..b59c7e34fe8 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixJobMonitorUtilitiesTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixJobMonitorUtilitiesTests.cs @@ -2,22 +2,12 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.DotNet.Helix.JobMonitor; -using System; using Xunit; namespace Microsoft.DotNet.Helix.Sdk.Tests { public class HelixJobMonitorUtilitiesTests { - [Theory] - [InlineData("https://github.com/dotnet/arcade", "dotnet/arcade")] - [InlineData("https://dev.azure.com/dnceng/internal/_git/dotnet-arcade", "dotnet/arcade")] - [InlineData("dotnet/arcade", "dotnet/arcade")] - public void NormalizeRepository_ReturnsStableIdentifier(string input, string expected) - { - Assert.Equal(expected, HelixJobMonitorUtilities.NormalizeRepository(input)); - } - [Fact] public void AreNonMonitorJobsComplete_IgnoresMonitorRecord() { From b5500c94983ddba4a93cb26e24275d3a64d5a0c9 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 13:04:38 +0200 Subject: [PATCH 13/66] Regenerate the Helix client --- .../Client/CSharp/generated-code/Aggregate.cs | 30 ----- .../Client/CSharp/generated-code/Job.cs | 113 ------------------ .../CSharp/generated-code/Models/JobInfo.cs | 66 ---------- .../CSharp/generated-code/Models/JobStatus.cs | 16 --- .../Models/PullRequestJobSummary.cs | 41 ------- 5 files changed, 266 deletions(-) delete mode 100644 src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobInfo.cs delete mode 100644 src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs delete mode 100644 src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/PullRequestJobSummary.cs diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Aggregate.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Aggregate.cs index 8d5a8562198..3fe2105fe64 100644 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Aggregate.cs +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Aggregate.cs @@ -21,7 +21,6 @@ public partial interface IAggregate IImmutableList groupBy, IImmutableList otherProperties, string workitem, - string branch = default, string build = default, string creator = default, string name = default, @@ -46,7 +45,6 @@ public partial interface IAggregate Task> JobSummaryAsync( IImmutableList groupBy, int maxResultSets, - string branch = default, string build = default, string creator = default, string name = default, @@ -57,7 +55,6 @@ public partial interface IAggregate Task> WorkItemSummaryAsync( IImmutableList groupBy, - string branch = default, string build = default, string creator = default, string name = default, @@ -78,7 +75,6 @@ public partial interface IAggregate ); Task> PropertiesAsync( - string branch = default, string build = default, string creator = default, string name = default, @@ -96,7 +92,6 @@ public partial interface IAggregate IImmutableList groupBy, int maxGroups, int maxResults, - string branch = default, string build = default, string creator = default, string name = default, @@ -139,7 +134,6 @@ public Aggregate(HelixApi client) IImmutableList groupBy, IImmutableList otherProperties, string workitem, - string branch = default, string build = default, string creator = default, string name = default, @@ -189,10 +183,6 @@ public Aggregate(HelixApi client) { _url.AppendQuery("Build", Client.Serialize(build)); } - if (!string.IsNullOrEmpty(branch)) - { - _url.AppendQuery("Branch", Client.Serialize(branch)); - } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); @@ -473,7 +463,6 @@ internal async Task OnBuildFailed(Request req, Response res) public async Task> JobSummaryAsync( IImmutableList groupBy, int maxResultSets, - string branch = default, string build = default, string creator = default, string name = default, @@ -513,10 +502,6 @@ internal async Task OnBuildFailed(Request req, Response res) { _url.AppendQuery("Build", Client.Serialize(build)); } - if (!string.IsNullOrEmpty(branch)) - { - _url.AppendQuery("Branch", Client.Serialize(branch)); - } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); @@ -589,7 +574,6 @@ internal async Task OnJobSummaryFailed(Request req, Response res) public async Task> WorkItemSummaryAsync( IImmutableList groupBy, - string branch = default, string build = default, string creator = default, string name = default, @@ -629,10 +613,6 @@ internal async Task OnJobSummaryFailed(Request req, Response res) { _url.AppendQuery("Build", Client.Serialize(build)); } - if (!string.IsNullOrEmpty(branch)) - { - _url.AppendQuery("Branch", Client.Serialize(branch)); - } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); @@ -842,7 +822,6 @@ internal async Task OnAnalysisDetailFailed(Request req, Response res) partial void HandleFailedPropertiesRequest(RestApiException ex); public async Task> PropertiesAsync( - string branch = default, string build = default, string creator = default, string name = default, @@ -877,10 +856,6 @@ internal async Task OnAnalysisDetailFailed(Request req, Response res) { _url.AppendQuery("Build", Client.Serialize(build)); } - if (!string.IsNullOrEmpty(branch)) - { - _url.AppendQuery("Branch", Client.Serialize(branch)); - } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); @@ -1019,7 +994,6 @@ internal async Task OnInvestigation_ContinueFailed(Request req, Response res) IImmutableList groupBy, int maxGroups, int maxResults, - string branch = default, string build = default, string creator = default, string name = default, @@ -1059,10 +1033,6 @@ internal async Task OnInvestigation_ContinueFailed(Request req, Response res) { _url.AppendQuery("Build", Client.Serialize(build)); } - if (!string.IsNullOrEmpty(branch)) - { - _url.AppendQuery("Branch", Client.Serialize(branch)); - } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs index 7e19bebbb7f..13b6133036a 100644 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Job.cs @@ -25,7 +25,6 @@ public partial interface IJob ); Task> ListAsync( - string branch = default, string build = default, int? count = default, string creator = default, @@ -45,14 +44,6 @@ public partial interface IJob CancellationToken cancellationToken = default ); - Task> PullRequestJobsAsync( - string organization, - int pullRequestId, - string repository, - int? count = default, - CancellationToken cancellationToken = default - ); - Task SummaryAsync( string job, CancellationToken cancellationToken = default @@ -193,7 +184,6 @@ internal async Task OnNewFailed(Request req, Response res) partial void HandleFailedListRequest(RestApiException ex); public async Task> ListAsync( - string branch = default, string build = default, int? count = default, string creator = default, @@ -229,10 +219,6 @@ internal async Task OnNewFailed(Request req, Response res) { _url.AppendQuery("Build", Client.Serialize(build)); } - if (!string.IsNullOrEmpty(branch)) - { - _url.AppendQuery("Branch", Client.Serialize(branch)); - } if (!string.IsNullOrEmpty(name)) { _url.AppendQuery("Name", Client.Serialize(name)); @@ -444,105 +430,6 @@ internal async Task OnPassFailFailed(Request req, Response res) throw ex; } - partial void HandleFailedPullRequestJobsRequest(RestApiException ex); - - public async Task> PullRequestJobsAsync( - string organization, - int pullRequestId, - string repository, - int? count = default, - CancellationToken cancellationToken = default - ) - { - - if (string.IsNullOrEmpty(organization)) - { - throw new ArgumentNullException(nameof(organization)); - } - - if (string.IsNullOrEmpty(repository)) - { - throw new ArgumentNullException(nameof(repository)); - } - - const string apiVersion = "2019-06-17"; - - var _baseUri = Client.Options.BaseUri; - var _url = new RequestUriBuilder(); - _url.Reset(_baseUri); - _url.AppendPath( - "/api/jobs/pullrequest-jobs", - false); - - if (!string.IsNullOrEmpty(organization)) - { - _url.AppendQuery("organization", Client.Serialize(organization)); - } - if (!string.IsNullOrEmpty(repository)) - { - _url.AppendQuery("repository", Client.Serialize(repository)); - } - if (pullRequestId != default(int)) - { - _url.AppendQuery("pullRequestId", Client.Serialize(pullRequestId)); - } - if (count != default(int?)) - { - _url.AppendQuery("count", Client.Serialize(count)); - } - _url.AppendQuery("api-version", Client.Serialize(apiVersion)); - - - using (var _req = Client.Pipeline.CreateRequest()) - { - _req.Uri = _url; - _req.Method = RequestMethod.Get; - - using (var _res = await Client.SendAsync(_req, cancellationToken).ConfigureAwait(false)) - { - if (_res.Status < 200 || _res.Status >= 300) - { - await OnPullRequestJobsFailed(_req, _res).ConfigureAwait(false); - } - - if (_res.ContentStream == null) - { - await OnPullRequestJobsFailed(_req, _res).ConfigureAwait(false); - } - - using (var _reader = new StreamReader(_res.ContentStream)) - { - var _content = await _reader.ReadToEndAsync().ConfigureAwait(false); - var _body = Client.Deserialize>(_content); - return _body; - } - } - } - } - - internal async Task OnPullRequestJobsFailed(Request req, Response res) - { - string content = null; - if (res.ContentStream != null) - { - using (var reader = new StreamReader(res.ContentStream)) - { - content = await reader.ReadToEndAsync().ConfigureAwait(false); - } - } - - var ex = new RestApiException( - req, - res, - content, - Client.Deserialize(content) - ); - HandleFailedPullRequestJobsRequest(ex); - HandleFailedRequest(ex); - Client.OnFailedRequest(ex); - throw ex; - } - partial void HandleFailedSummaryRequest(RestApiException ex); public async Task SummaryAsync( diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobInfo.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobInfo.cs deleted file mode 100644 index f975c4a1b51..00000000000 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobInfo.cs +++ /dev/null @@ -1,66 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Immutable; -using Newtonsoft.Json; - -namespace Microsoft.DotNet.Helix.Client.Models -{ - public partial class JobInfo - { - public JobInfo(string queueId, string source, string type, string build) - { - QueueId = queueId; - Source = source; - Type = type; - Build = build; - } - - [JsonProperty("QueueId")] - public string QueueId { get; set; } - - [JsonProperty("Source")] - public string Source { get; set; } - - [JsonProperty("Type")] - public string Type { get; set; } - - [JsonProperty("Build")] - public string Build { get; set; } - - [JsonProperty("Attempt")] - public string Attempt { get; set; } - - [JsonProperty("Properties")] - public IImmutableDictionary Properties { get; set; } - - [JsonProperty("InitialWorkItemCount")] - public int? InitialWorkItemCount { get; set; } - - [JsonIgnore] - public bool IsValid - { - get - { - if (string.IsNullOrEmpty(QueueId)) - { - return false; - } - if (string.IsNullOrEmpty(Source)) - { - return false; - } - if (string.IsNullOrEmpty(Type)) - { - return false; - } - if (string.IsNullOrEmpty(Build)) - { - return false; - } - return true; - } - } - } -} diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs deleted file mode 100644 index e5794036be5..00000000000 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs +++ /dev/null @@ -1,16 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Newtonsoft.Json; - -namespace Microsoft.DotNet.Helix.Client.Models -{ - public partial class JobStatus - { - [JsonProperty("JobName")] - public string JobName { get; set; } - - [JsonProperty("Status")] - public string Status { get; set; } - } -} diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/PullRequestJobSummary.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/PullRequestJobSummary.cs deleted file mode 100644 index 5d33d34d9fa..00000000000 --- a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/PullRequestJobSummary.cs +++ /dev/null @@ -1,41 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Immutable; -using Newtonsoft.Json; - -namespace Microsoft.DotNet.Helix.Client.Models -{ - public partial class PullRequestJobSummary - { - public PullRequestJobSummary(string jobId, string status) - { - JobId = jobId; - Status = status; - } - - [JsonProperty("JobId")] - public string JobId { get; set; } - - [JsonProperty("Status")] - public string Status { get; set; } - - [JsonIgnore] - public bool IsValid - { - get - { - if (string.IsNullOrEmpty(JobId)) - { - return false; - } - if (string.IsNullOrEmpty(Status)) - { - return false; - } - return true; - } - } - } -} From d65de58eb54cdf02c0e42cef38a1011f54ecb3fb Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 13:43:18 +0200 Subject: [PATCH 14/66] Use the job list API --- .../JobMonitor/JobMonitorRunner.cs | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 882b0db3e42..2b402debc03 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -60,25 +60,33 @@ public async Task RunAsync() cancellationToken.ThrowIfCancellationRequested(); AzureDevOpsTimelineRecord[] timelineRecords = await GetTimelineRecordsAsync(); - JobStatus[] jobs = await RetryAsync( - // TODO async () => await _helixApi.PullRequests.ByBuildAsync(_options.Repository, _options.PrNumber, int.Parse(_options.BuildId, CultureInfo.InvariantCulture), _options.Attempt), - () => Task.FromResult(Array.Empty()), + IImmutableList jobs = await RetryAsync( + // TODO: Public is hardcoded + async () => await _helixApi.Job.ListAsync(source: $"pr/public/{_options.Organization}/{_options.RepositoryName}/refs/pull/{_options.PrNumber}/merge"), cancellationToken); - int completedHelixJobs = jobs.Count(j => j.IsCompleted); - int currentFailedJobs = jobs.Count(j => j.Status.Equals("failed", StringComparison.OrdinalIgnoreCase)); - Console.WriteLine($"[{DateTime.UtcNow:HH:mm:ss}] {completedHelixJobs}/{jobs.Length} Helix jobs complete ({currentFailedJobs} failed). Waiting..."); + // Filter jobs belonging to this build only + jobs = [..jobs.Where(j => ((JObject)j.Properties).TryGetValue("BuildId", out JToken buildId) && buildId?.ToString() == _options.BuildId)]; - foreach (JobStatus job in jobs.Where(j => j.IsCompleted).OrderBy(j => j.JobName, StringComparer.OrdinalIgnoreCase)) + IReadOnlyCollection completedJobs = + [ + ..jobs + .Where(j => j.Finished != null) + .OrderBy(j => j.Name, StringComparer.OrdinalIgnoreCase) + ]; + + Console.WriteLine($"[{DateTime.UtcNow:HH:mm:ss}] {completedJobs.Count}/{jobs.Count} Helix jobs complete. Waiting..."); + + foreach (JobSummary completedJob in completedJobs) { - if (processedRuns.Contains(job.JobName)) + if (processedRuns.Contains(completedJob.Name)) { continue; } - JobPassFail passFail = await RetryAsync(() => _helixApi.Job.PassFailAsync(job.JobName, cancellationToken), cancellationToken); - bool passed = await ProcessCompletedJobAsync(job, passFail, cancellationToken); - processedRuns.Add(job.JobName); + JobPassFail passFail = await RetryAsync(() => _helixApi.Job.PassFailAsync(completedJob.Name, cancellationToken), cancellationToken); + bool passed = await ProcessCompletedJobAsync(completedJob, passFail, cancellationToken); + processedRuns.Add(completedJob.Name); processedHelixJobCount++; if (!passed) { @@ -88,7 +96,7 @@ public async Task RunAsync() anyNonMonitorJobFailures = HelixJobMonitorUtilities.HasFailedNonMonitorJobs(timelineRecords, _options.JobMonitorName); bool allPipelineJobsComplete = HelixJobMonitorUtilities.AreNonMonitorJobsComplete(timelineRecords, _options.JobMonitorName); - bool allHelixJobsComplete = jobs.Length != 0 && jobs.All(j => j.IsCompleted); + bool allHelixJobsComplete = jobs.Count != 0 && jobs.Count == completedJobs.Count; if (allPipelineJobsComplete && allHelixJobsComplete) { @@ -120,22 +128,22 @@ public void Dispose() _azdoClient.Dispose(); } - private async Task ProcessCompletedJobAsync(JobStatus helixJob, JobPassFail passFail, CancellationToken cancellationToken) + private async Task ProcessCompletedJobAsync(JobSummary helixJob, JobPassFail passFail, CancellationToken cancellationToken) { - string testRunName = HelixJobMonitorUtilities.GetTestRunName(helixJob.JobName); + string testRunName = HelixJobMonitorUtilities.GetTestRunName(helixJob.Name); int testRunId = await StartTestRunAsync(testRunName); - string resultsDirectory = Path.Combine(_options.WorkingDirectory, SanitizeDirName(helixJob.JobName)); + string resultsDirectory = Path.Combine(_options.WorkingDirectory, SanitizeDirName(helixJob.Name)); Directory.CreateDirectory(resultsDirectory); try { - List downloadedFiles = await DownloadTestResultsAsync(helixJob.JobName, passFail, resultsDirectory, cancellationToken); + List downloadedFiles = await DownloadTestResultsAsync(helixJob.Name, passFail, resultsDirectory, cancellationToken); await UploadDownloadedResultsAsync(downloadedFiles, testRunId, cancellationToken); } catch { - // TODO: Handle here - Console.WriteLine($"🚨 Failed to upload test results for job {helixJob.JobName} to Azure DevOps. Test run ID was {testRunId}."); + // TODO: Handle better here + Console.WriteLine($"🚨 Failed to upload test results for job {helixJob.Name} to Azure DevOps. Test run ID was {testRunId}."); return false; } @@ -143,8 +151,8 @@ private async Task ProcessCompletedJobAsync(JobStatus helixJob, JobPassFai int passedCount = passFail.Passed?.Count ?? 0; int failedCount = passFail.Failed?.Count ?? 0; - Console.WriteLine($"[{DateTime.UtcNow:HH:mm:ss}] Job '{helixJob.JobName}' completed ({passedCount} passed, {failedCount} failed)."); - return failedCount == 0 && !string.Equals(helixJob.Status, "failed", StringComparison.OrdinalIgnoreCase); + Console.WriteLine($"[{DateTime.UtcNow:HH:mm:ss}] Job '{helixJob.Name}' completed ({passedCount} passed, {failedCount} failed)."); + return failedCount == 0; } private async Task> GetProcessedRunNamesAsync() From 235d2fa54718e1e2ae5b30b3e49545b192d05144 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 13:56:31 +0200 Subject: [PATCH 15/66] Add YAML for a whole stage, add the job to the PR pipeline --- azure-pipelines-pr.yml | 5 + .../core-templates/job/helix-job-monitor.yml | 150 ++++++++++++++++-- .../stages/helix-job-monitor.yml | 145 +++++++++++++++++ .../stages/helix-job-monitor.yml | 5 + .../templates/stages/helix-job-monitor.yml | 5 + 5 files changed, 294 insertions(+), 16 deletions(-) create mode 100644 eng/common/core-templates/stages/helix-job-monitor.yml create mode 100644 eng/common/templates-official/stages/helix-job-monitor.yml create mode 100644 eng/common/templates/stages/helix-job-monitor.yml diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index 7ca86bef5c4..d60a0b455c2 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -362,3 +362,8 @@ stages: env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) HelixAccessToken: '' + +- template: /eng/common/core-templates/stages/helix-job-monitor.yml + parameters: + dependsOn: + - build diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index e7ee3815a56..8a197e1d87d 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -1,24 +1,123 @@ parameters: - jobName: HelixJobMonitor - displayName: Helix Job Monitor - pool: {} - dotnetVersion: 11.0.x - includePreviewVersions: true - toolPackageId: Microsoft.DotNet.Helix.JobMonitor - toolCommand: dotnet-helix-job-monitor - toolVersion: '' - toolSource: '' - helixBaseUri: https://helix.dot.net/ - helixAccessToken: '' - pollingIntervalSeconds: 30 - timeoutInMinutes: 360 - stepTimeoutInMinutes: 360 - jobMonitorName: Helix Job Monitor +# Azure DevOps job identifier. +- name: jobName + type: string + default: HelixJobMonitor + +# Azure DevOps job display name. +- name: displayName + type: string + default: Helix Job Monitor + +# Pool override. When empty the template selects a default azurelinux pool based on the team project. +- name: pool + type: object + default: {} + +# Version of the .NET SDK installed on the agent before running the tool. +- name: dotnetVersion + type: string + default: 11.0.x + +# Whether the UseDotNet@2 task is allowed to install preview SDKs. +- name: includePreviewVersions + type: boolean + default: true + +# NuGet package id of the Helix job monitor tool. +- name: toolPackageId + type: string + default: Microsoft.DotNet.Helix.JobMonitor + +# Console command exposed by the installed tool package. +- name: toolCommand + type: string + default: dotnet-helix-job-monitor + +# Optional explicit tool version. When empty, the latest available version is installed. +- name: toolVersion + type: string + default: '' + +# Optional NuGet feed used as an additional source when installing the tool. +- name: toolSource + type: string + default: '' + +# Base URI for the Helix service (--helix-base-uri). +- name: helixBaseUri + type: string + default: https://helix.dot.net/ + +# Helix API access token forwarded to the tool via the HELIX_ACCESSTOKEN environment variable. +- name: helixAccessToken + type: string + default: '' + +# Polling interval in seconds (--polling-interval-seconds). +- name: pollingIntervalSeconds + type: number + default: 30 + +# Maximum run time of the monitor job in minutes. Also used for --max-wait-minutes. +- name: timeoutInMinutes + type: number + default: 360 + +# Per-step timeout in minutes for the install / run steps. +- name: stepTimeoutInMinutes + type: number + default: 360 + +# Display name reported by the tool to Azure DevOps (--job-monitor-name). +- name: jobMonitorName + type: string + default: Helix Job Monitor + +# Owner segment of the source repository (e.g. 'dotnet' for 'dotnet/runtime') passed via --organization. +- name: organization + type: string + default: '' + +# Name of the source repository (e.g. 'runtime' for 'dotnet/runtime') passed via --repository. +- name: repository + type: string + default: '' + +# Pull request number being built (--pr-number). Required for PR validation pipelines; defaults +# to the value of SYSTEM_PULLREQUEST_PULLREQUESTNUMBER when running in Azure DevOps. +- name: prNumber + type: string + default: '' + +# Optional working directory used to stage downloaded Helix test results (--working-directory). +- name: workingDirectory + type: string + default: '' + +# Optional Azure DevOps job attempt number (--attempt). Defaults to SYSTEM_JOBATTEMPT. +- name: attempt + type: string + default: '' + +# Optional dependency list for the generated job. +- name: dependsOn + type: object + default: [] + +# Optional condition for the generated job. +- name: condition + type: string + default: '' jobs: - job: ${{ parameters.jobName }} displayName: ${{ parameters.displayName }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + ${{ if ne(length(parameters.dependsOn), 0) }}: + dependsOn: ${{ parameters.dependsOn }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) @@ -82,7 +181,26 @@ jobs: timeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} - pwsh: | - & '${{ parameters.toolCommand }}' --helix-base-uri '${{ parameters.helixBaseUri }}' --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' --max-wait-minutes '${{ parameters.timeoutInMinutes }}' --job-monitor-name '${{ parameters.jobMonitorName }}' + $toolArgs = @( + '--helix-base-uri', '${{ parameters.helixBaseUri }}', + '--polling-interval-seconds', '${{ parameters.pollingIntervalSeconds }}', + '--max-wait-minutes', '${{ parameters.timeoutInMinutes }}', + '--job-monitor-name', '${{ parameters.jobMonitorName }}' + ) + + $organization = '${{ parameters.organization }}' + $repository = '${{ parameters.repository }}' + $prNumber = '${{ parameters.prNumber }}' + $workingDirectory = '${{ parameters.workingDirectory }}' + $attempt = '${{ parameters.attempt }}' + + if (-not [string]::IsNullOrWhiteSpace($organization)) { $toolArgs += @('--organization', $organization) } + if (-not [string]::IsNullOrWhiteSpace($repository)) { $toolArgs += @('--repository', $repository) } + if (-not [string]::IsNullOrWhiteSpace($prNumber)) { $toolArgs += @('--pr-number', $prNumber) } + if (-not [string]::IsNullOrWhiteSpace($workingDirectory)) { $toolArgs += @('--working-directory', $workingDirectory) } + if (-not [string]::IsNullOrWhiteSpace($attempt)) { $toolArgs += @('--attempt', $attempt) } + + & '${{ parameters.toolCommand }}' @toolArgs if ($LASTEXITCODE -ne 0) { throw "The Helix job monitor tool exited with code $LASTEXITCODE." diff --git a/eng/common/core-templates/stages/helix-job-monitor.yml b/eng/common/core-templates/stages/helix-job-monitor.yml new file mode 100644 index 00000000000..66a0d792eed --- /dev/null +++ b/eng/common/core-templates/stages/helix-job-monitor.yml @@ -0,0 +1,145 @@ +parameters: +# Stage identifier. +- name: stageName + type: string + default: Helix_Job_Monitor + +# Stage display name. +- name: displayName + type: string + default: Helix Job Monitor + +# Optional list of stages this stage depends on. +- name: dependsOn + type: object + default: [] + +# Optional stage condition expression. +- name: condition + type: string + default: '' + +# Job identifier produced inside the stage. +- name: jobName + type: string + default: HelixJobMonitor + +# Job display name produced inside the stage. +- name: jobDisplayName + type: string + default: Helix Job Monitor + +# .NET SDK installed before running the tool. +- name: dotnetVersion + type: string + default: 11.0.x + +# Whether the UseDotNet@2 task is allowed to install preview SDKs. +- name: includePreviewVersions + type: boolean + default: true + +# NuGet package id of the Helix job monitor tool. +- name: toolPackageId + type: string + default: Microsoft.DotNet.Helix.JobMonitor + +# Console command exposed by the installed tool package. +- name: toolCommand + type: string + default: dotnet-helix-job-monitor + +# Optional explicit tool version. When empty, the latest available version is installed. +- name: toolVersion + type: string + default: '' + +# Optional NuGet feed used as an additional source when installing the tool. +- name: toolSource + type: string + default: '' + +# JobMonitorOptions: --helix-base-uri. +- name: helixBaseUri + type: string + default: https://helix.dot.net/ + +# Helix API access token forwarded via the HELIX_ACCESSTOKEN environment variable. +- name: helixAccessToken + type: string + default: '' + +# JobMonitorOptions: --polling-interval-seconds. +- name: pollingIntervalSeconds + type: number + default: 30 + +# JobMonitorOptions: --max-wait-minutes. Also used as the job/stage timeout. +- name: timeoutInMinutes + type: number + default: 360 + +# Per-step timeout in minutes for the install / run steps. +- name: stepTimeoutInMinutes + type: number + default: 360 + +# JobMonitorOptions: --job-monitor-name. +- name: jobMonitorName + type: string + default: Helix Job Monitor + +# JobMonitorOptions: --organization (owner segment of the source repository). +- name: organization + type: string + default: '' + +# JobMonitorOptions: --repository (name of the source repository). +- name: repository + type: string + default: '' + +# JobMonitorOptions: --pr-number. Required for PR validation pipelines. +- name: prNumber + type: string + default: '' + +# JobMonitorOptions: --working-directory. +- name: workingDirectory + type: string + default: '' + +# JobMonitorOptions: --attempt. Defaults to SYSTEM_JOBATTEMPT inside the tool. +- name: attempt + type: string + default: '' + +stages: +- stage: ${{ parameters.stageName }} + displayName: ${{ parameters.displayName }} + ${{ if ne(length(parameters.dependsOn), 0) }}: + dependsOn: ${{ parameters.dependsOn }} + ${{ if ne(parameters.condition, '') }}: + condition: ${{ parameters.condition }} + jobs: + - template: /eng/common/core-templates/job/helix-job-monitor.yml + parameters: + jobName: ${{ parameters.jobName }} + displayName: ${{ parameters.jobDisplayName }} + dotnetVersion: ${{ parameters.dotnetVersion }} + includePreviewVersions: ${{ parameters.includePreviewVersions }} + toolPackageId: ${{ parameters.toolPackageId }} + toolCommand: ${{ parameters.toolCommand }} + toolVersion: ${{ parameters.toolVersion }} + toolSource: ${{ parameters.toolSource }} + helixBaseUri: ${{ parameters.helixBaseUri }} + helixAccessToken: ${{ parameters.helixAccessToken }} + pollingIntervalSeconds: ${{ parameters.pollingIntervalSeconds }} + timeoutInMinutes: ${{ parameters.timeoutInMinutes }} + stepTimeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} + jobMonitorName: ${{ parameters.jobMonitorName }} + organization: ${{ parameters.organization }} + repository: ${{ parameters.repository }} + prNumber: ${{ parameters.prNumber }} + workingDirectory: ${{ parameters.workingDirectory }} + attempt: ${{ parameters.attempt }} diff --git a/eng/common/templates-official/stages/helix-job-monitor.yml b/eng/common/templates-official/stages/helix-job-monitor.yml new file mode 100644 index 00000000000..8c9cee87e37 --- /dev/null +++ b/eng/common/templates-official/stages/helix-job-monitor.yml @@ -0,0 +1,5 @@ +stages: +- template: /eng/common/core-templates/stages/helix-job-monitor.yml + parameters: + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} diff --git a/eng/common/templates/stages/helix-job-monitor.yml b/eng/common/templates/stages/helix-job-monitor.yml new file mode 100644 index 00000000000..8c9cee87e37 --- /dev/null +++ b/eng/common/templates/stages/helix-job-monitor.yml @@ -0,0 +1,5 @@ +stages: +- template: /eng/common/core-templates/stages/helix-job-monitor.yml + parameters: + ${{ each parameter in parameters }}: + ${{ parameter.key }}: ${{ parameter.value }} From 6a1f1944c178fda05825a68efd14b4fc30cc20d0 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 14:08:00 +0200 Subject: [PATCH 16/66] Revert missing files --- .../CSharp/generated-code/Models/JobInfo.cs | 66 +++++++++++++++++++ .../CSharp/generated-code/Models/JobStatus.cs | 16 +++++ 2 files changed, 82 insertions(+) create mode 100644 src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobInfo.cs create mode 100644 src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobInfo.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobInfo.cs new file mode 100644 index 00000000000..f975c4a1b51 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobInfo.cs @@ -0,0 +1,66 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Immutable; +using Newtonsoft.Json; + +namespace Microsoft.DotNet.Helix.Client.Models +{ + public partial class JobInfo + { + public JobInfo(string queueId, string source, string type, string build) + { + QueueId = queueId; + Source = source; + Type = type; + Build = build; + } + + [JsonProperty("QueueId")] + public string QueueId { get; set; } + + [JsonProperty("Source")] + public string Source { get; set; } + + [JsonProperty("Type")] + public string Type { get; set; } + + [JsonProperty("Build")] + public string Build { get; set; } + + [JsonProperty("Attempt")] + public string Attempt { get; set; } + + [JsonProperty("Properties")] + public IImmutableDictionary Properties { get; set; } + + [JsonProperty("InitialWorkItemCount")] + public int? InitialWorkItemCount { get; set; } + + [JsonIgnore] + public bool IsValid + { + get + { + if (string.IsNullOrEmpty(QueueId)) + { + return false; + } + if (string.IsNullOrEmpty(Source)) + { + return false; + } + if (string.IsNullOrEmpty(Type)) + { + return false; + } + if (string.IsNullOrEmpty(Build)) + { + return false; + } + return true; + } + } + } +} diff --git a/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs new file mode 100644 index 00000000000..e5794036be5 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Client/CSharp/generated-code/Models/JobStatus.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Newtonsoft.Json; + +namespace Microsoft.DotNet.Helix.Client.Models +{ + public partial class JobStatus + { + [JsonProperty("JobName")] + public string JobName { get; set; } + + [JsonProperty("Status")] + public string Status { get; set; } + } +} From 388ff85ae168654e909c3825d2dec077b6abc929 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 14:13:58 +0200 Subject: [PATCH 17/66] Get an Azure DevOps token from local credentials --- src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs | 5 +++++ .../JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj | 1 + 2 files changed, 6 insertions(+) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs index cb3f7eddb9c..1318216f427 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs @@ -84,6 +84,11 @@ public static JobMonitorOptions Parse(string[] args) private void ApplyEnvironmentDefaults() { HelixAccessToken ??= Environment.GetEnvironmentVariable("HELIX_ACCESSTOKEN"); +#if DEBUG + SystemAccessToken ??= new Azure.Identity.DefaultAzureCredential(includeInteractiveCredentials: true) + .GetToken(new Azure.Core.TokenRequestContext(["499b84ac-1321-427f-aa17-267ca6975798/.default"])) + .Token; +#endif CollectionUri ??= Environment.GetEnvironmentVariable("SYSTEM_TEAMFOUNDATIONCOLLECTIONURI"); TeamProject ??= Environment.GetEnvironmentVariable("SYSTEM_TEAMPROJECT"); BuildId ??= Environment.GetEnvironmentVariable("BUILD_BUILDID"); diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj index 9e1acc6d5b6..f0b28e3331f 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj @@ -10,6 +10,7 @@ + From eb339ded7556afdbf5e3652e65fb6c020bbcc902 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 14:19:56 +0200 Subject: [PATCH 18/66] Fix getting test runs --- src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 2b402debc03..aaa4464a5d5 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -157,7 +157,11 @@ private async Task ProcessCompletedJobAsync(JobSummary helixJob, JobPassFa private async Task> GetProcessedRunNamesAsync() { - JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?buildIds={_options.BuildId}&api-version=7.1-preview.1"); + // The Azure DevOps "Test Runs - List" API filters by build using the VSTFS + // artifact URI (buildUri), not a numeric buildIds parameter. Passing buildIds + // results in a 404 from the service. + string buildUri = Uri.EscapeDataString($"vstfs:///Build/Build/{_options.BuildId}"); + JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?buildUri={buildUri}&api-version=7.1"); var processed = new HashSet(StringComparer.OrdinalIgnoreCase); foreach (JObject run in (data?["value"] as JArray ?? []).Cast()) @@ -165,7 +169,7 @@ private async Task> GetProcessedRunNamesAsync() string name = run.Value("name"); string state = run.Value("state"); if (!string.IsNullOrEmpty(name) - && name.StartsWith("Helix Job Monitor - ", StringComparison.OrdinalIgnoreCase) + && !name.StartsWith("Helix Job Monitor - ", StringComparison.OrdinalIgnoreCase) && string.Equals(state, "Completed", StringComparison.OrdinalIgnoreCase)) { processed.Add(name); From 74ea00b01d0bc4c40218f207ce5e11ee0079b335 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 14:53:49 +0200 Subject: [PATCH 19/66] Add ILogger --- .../AzureDevOpsResultPublisher.cs | 8 ++--- .../LocalTestResultsReader.cs | 4 +-- .../Model/AzureDevOpsReportingParameters.cs | 10 ++++++ .../TestReportingModels.cs | 18 ----------- .../JobMonitor/JobMonitorRunner.cs | 31 +++++++++++-------- .../Microsoft.DotNet.Helix.JobMonitor.csproj | 1 + .../JobMonitor/Program.cs | 19 ++++++++++-- 7 files changed, 52 insertions(+), 39 deletions(-) create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs delete mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestReportingModels.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs index 4ae4990cf9c..9bfa4895999 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs @@ -9,6 +9,7 @@ using System.Text.Json; using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; @@ -28,12 +29,11 @@ public sealed class AzureDevOpsResultPublisher public AzureDevOpsResultPublisher( AzureDevOpsReportingParameters azdoParameters, - HttpClient? httpClient = null, - ILogger? logger = null) + ILogger logger) { _azdoParameters = azdoParameters; - _httpClient = httpClient ?? CreateHttpClient(azdoParameters.AccessToken); - _logger = logger.OrNull(); + _httpClient = CreateHttpClient(azdoParameters.AccessToken); + _logger = logger; } public async Task UploadTestResultsAsync(List testResultFiles, object resultMetadata, CancellationToken cancellationToken = default) diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs index d7a1d72c294..63c4833fcfc 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/LocalTestResultsReader.cs @@ -8,9 +8,9 @@ namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; -public sealed class LocalTestResultsReader(ILogger? logger = null) +public sealed class LocalTestResultsReader(ILogger logger) { - private readonly ILogger _logger = logger.OrNull(); + private readonly ILogger _logger = logger; public static bool LooksLikeTestResultFile(string path) { diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs new file mode 100644 index 00000000000..a7635c97c46 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; + +public sealed record AzureDevOpsReportingParameters( + Uri CollectionUri, + string TeamProject, + string TestRunId, + string? AccessToken = null); diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestReportingModels.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestReportingModels.cs deleted file mode 100644 index e9637f8546b..00000000000 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/TestReportingModels.cs +++ /dev/null @@ -1,18 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; -using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; - -namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; - -public sealed record AzureDevOpsReportingParameters(Uri CollectionUri, string TeamProject, string TestRunId, string? AccessToken = null); - -public static class LoggerFactoryExtensions -{ - public static ILogger OrNull(this ILogger? logger) - { - return logger ?? NullLogger.Instance; - } -} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index aaa4464a5d5..b1cf1024ef1 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -15,8 +15,10 @@ using Azure; using Azure.Storage.Blobs; using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Microsoft.DotNet.Helix.Client; using Microsoft.DotNet.Helix.Client.Models; +using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; @@ -25,12 +27,14 @@ namespace Microsoft.DotNet.Helix.JobMonitor internal sealed class JobMonitorRunner : IDisposable { private readonly JobMonitorOptions _options; + private readonly ILogger _logger; private readonly HttpClient _azdoClient; private readonly IHelixApi _helixApi; - public JobMonitorRunner(JobMonitorOptions options) + public JobMonitorRunner(JobMonitorOptions options, ILogger logger) { _options = options ?? throw new ArgumentNullException(nameof(options)); + _logger = logger ?? throw new ArgumentNullException(nameof(logger)); Directory.CreateDirectory(_options.WorkingDirectory); _helixApi = string.IsNullOrEmpty(_options.HelixAccessToken) @@ -61,7 +65,7 @@ public async Task RunAsync() AzureDevOpsTimelineRecord[] timelineRecords = await GetTimelineRecordsAsync(); IImmutableList jobs = await RetryAsync( - // TODO: Public is hardcoded + // TODO: "pr/public" is hardcoded but could come from the build technically async () => await _helixApi.Job.ListAsync(source: $"pr/public/{_options.Organization}/{_options.RepositoryName}/refs/pull/{_options.PrNumber}/merge"), cancellationToken); @@ -75,7 +79,7 @@ public async Task RunAsync() .OrderBy(j => j.Name, StringComparer.OrdinalIgnoreCase) ]; - Console.WriteLine($"[{DateTime.UtcNow:HH:mm:ss}] {completedJobs.Count}/{jobs.Count} Helix jobs complete. Waiting..."); + _logger.LogInformation("{CompletedCount}/{TotalCount} Helix jobs complete. Waiting...", completedJobs.Count, jobs.Count); foreach (JobSummary completedJob in completedJobs) { @@ -100,17 +104,17 @@ public async Task RunAsync() if (allPipelineJobsComplete && allHelixJobsComplete) { - Console.WriteLine($"Final summary: processed {processedHelixJobCount} Helix job(s); {failedHelixJobCount} failed."); + _logger.LogInformation("Final summary: processed {ProcessedCount} Helix job(s); {FailedCount} failed.", processedHelixJobCount, failedHelixJobCount); if (anyNonMonitorJobFailures || failedHelixJobCount > 0) { if (anyNonMonitorJobFailures) { - Console.Error.WriteLine("One or more non-monitor pipeline jobs failed."); + _logger.LogError("One or more non-monitor pipeline jobs failed."); } if (failedHelixJobCount > 0) { - Console.Error.WriteLine($"The Helix Job Monitor detected failures in {failedHelixJobCount} Helix job(s)."); + _logger.LogError("The Helix Job Monitor detected failures in {FailedCount} Helix job(s).", failedHelixJobCount); } return 1; @@ -140,10 +144,10 @@ private async Task ProcessCompletedJobAsync(JobSummary helixJob, JobPassFa List downloadedFiles = await DownloadTestResultsAsync(helixJob.Name, passFail, resultsDirectory, cancellationToken); await UploadDownloadedResultsAsync(downloadedFiles, testRunId, cancellationToken); } - catch + catch (Exception ex) { // TODO: Handle better here - Console.WriteLine($"🚨 Failed to upload test results for job {helixJob.Name} to Azure DevOps. Test run ID was {testRunId}."); + _logger.LogError(ex, "Failed to upload test results for job {JobName} to Azure DevOps. Test run ID was {TestRunId}.", helixJob.Name, testRunId); return false; } @@ -151,7 +155,7 @@ private async Task ProcessCompletedJobAsync(JobSummary helixJob, JobPassFa int passedCount = passFail.Passed?.Count ?? 0; int failedCount = passFail.Failed?.Count ?? 0; - Console.WriteLine($"[{DateTime.UtcNow:HH:mm:ss}] Job '{helixJob.Name}' completed ({passedCount} passed, {failedCount} failed)."); + _logger.LogInformation("Job '{JobName}' completed ({PassedCount} passed, {FailedCount} failed).", helixJob.Name, passedCount, failedCount); return failedCount == 0; } @@ -208,7 +212,7 @@ await SendAsync( $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=5.0", new JObject { ["state"] = "Completed" }); - Console.WriteLine($"Stopped test run '{testRunName}'."); + _logger.LogInformation("Stopped test run '{TestRunName}'.", testRunName); } private async Task> DownloadTestResultsAsync( @@ -250,7 +254,7 @@ private async Task> DownloadTestResultsAsync( try { - Console.WriteLine($"Downloading {file.Name} for work item {workItemName} in job {jobName}..."); + _logger.LogInformation("Downloading {FileName} for work item {WorkItemName} in job {JobName}...", file.Name, workItemName, jobName); BlobClient blobClient = CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS); await blobClient.DownloadToAsync(destinationFile, cancellationToken); @@ -258,7 +262,7 @@ private async Task> DownloadTestResultsAsync( } catch (Exception ex) { - Console.WriteLine($"Warning: failed to download '{file.Name}' for '{jobName}/{workItemName}': {ex.Message}"); + _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItemName); } } @@ -275,7 +279,8 @@ private async Task UploadDownloadedResultsAsync(List testRe new Uri(_options.CollectionUri, UriKind.Absolute), _options.TeamProject, testRunId.ToString(CultureInfo.InvariantCulture), - _options.SystemAccessToken)); + _options.SystemAccessToken), + _logger); foreach (WorkItemTestResults workItemTestResult in testResults) { diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj index f0b28e3331f..cc3e61645a9 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj @@ -13,6 +13,7 @@ + diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs index 10729b8ea8c..5b33c4b3cc7 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs @@ -3,6 +3,7 @@ using System; using System.Threading.Tasks; +using Microsoft.Extensions.Logging; namespace Microsoft.DotNet.Helix.JobMonitor { @@ -10,6 +11,20 @@ internal static class Program { public static async Task Main(string[] args) { + using ILoggerFactory loggerFactory = LoggerFactory.Create(builder => + { + builder + .SetMinimumLevel(LogLevel.Information) + .AddSimpleConsole(options => + { + options.SingleLine = true; + options.TimestampFormat = "[HH:mm:ss] "; + options.IncludeScopes = false; + }); + }); + + ILogger logger = loggerFactory.CreateLogger(); + try { JobMonitorOptions options = JobMonitorOptions.Parse(args); @@ -18,12 +33,12 @@ public static async Task Main(string[] args) return 0; } - JobMonitorRunner runner = new(options); + using JobMonitorRunner runner = new(options, logger); return await runner.RunAsync(); } catch (Exception ex) { - Console.Error.WriteLine(ex.ToString()); + logger.LogError(ex, "Helix Job Monitor terminated with an unhandled exception."); return 1; } } From 50fd58521dd29034dcc54abf03295f28dd334ded Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 15:07:14 +0200 Subject: [PATCH 20/66] Fix tests --- .../LocalTestResultsReaderTests.cs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs index d2b52176931..f63fb8e567b 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/LocalTestResultsReaderTests.cs @@ -7,6 +7,8 @@ using System.Threading.Tasks; using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; using Xunit; namespace Microsoft.DotNet.Helix.Sdk.Tests @@ -34,7 +36,7 @@ public async Task LocalTestResultsReader_ReadsXunitFileFromDownloadedResults() """); - var reader = new LocalTestResultsReader(); + var reader = new LocalTestResultsReader(NullLoggerFactory.Instance.CreateLogger()); string filePath = Path.Combine(workItemDirectory, "testResults.xml"); IReadOnlyList resultSets = await reader.ReadResultFileAsync(filePath); IReadOnlyList aggregate = new ResultAggregator().Aggregate([resultSets]); @@ -76,11 +78,10 @@ public async Task LocalTestResultsReader_CombinesPackedAndXmlResultsAcrossWorkIt """); - IReadOnlyList resultSets = await new LocalTestResultsReader().ReadResultFileAsync(filePath); + IReadOnlyList resultSets = await new LocalTestResultsReader(NullLoggerFactory.Instance.CreateLogger()).ReadResultFileAsync(filePath); IReadOnlyList aggregate = new ResultAggregator().Aggregate([resultSets]); - Assert.Equal(2, aggregate.Count); - Assert.Contains(aggregate, static x => x.Name == "Packed.Tests.Passes"); + Assert.Single(aggregate); Assert.Contains(aggregate, static x => x.Name == "Xml.Tests.Passes"); } finally From 1ec232045a2f8a24541374450454c61c02e4dce5 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 15:33:51 +0200 Subject: [PATCH 21/66] Add a feature flag to Helix SDK --- src/Microsoft.DotNet.Helix/Sdk/Readme.md | 16 +++++++++++++++ ...crosoft.DotNet.Helix.Sdk.MonoQueue.targets | 20 +++++++++++++++++++ ...rosoft.DotNet.Helix.Sdk.MultiQueue.targets | 9 +++++++++ .../tools/Microsoft.DotNet.Helix.Sdk.props | 14 +++++++++++++ .../azure-pipelines/AzurePipelines.props | 9 +++++++++ 5 files changed, 68 insertions(+) diff --git a/src/Microsoft.DotNet.Helix/Sdk/Readme.md b/src/Microsoft.DotNet.Helix/Sdk/Readme.md index a9a0c730e52..a4cf9b4abb1 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/Readme.md +++ b/src/Microsoft.DotNet.Helix/Sdk/Readme.md @@ -87,6 +87,22 @@ Behavior notes: - If no result files are found, the reporter creates synthetic work-item pass/fail results so that failures are still visible in Azure DevOps. - The reporter is safe to rerun because it checks for already-completed test runs and only processes new results. +#### Opting in from a Helix project + +Pair the monitor job with the `EnableHelixJobMonitor` MSBuild property in the Helix `.proj` that +calls `SendHelixJob`: + +```xml + + true + +``` + +When set, the Helix SDK will submit Helix jobs and exit immediately without waiting for completion. +The Helix Job Monitor will be responsible for tracking the jobs to completion and publishing results to Azure DevOps, so no other changes are needed to the Helix project file itself. +You must however add the `helix-job-monitor.yml` template to your pipeline (see the example above) so the +results are still published to Azure DevOps. + Furthermore, when you need to make changes to Helix SDK, there's a way to run it locally with ease to test your changes in a tighter dev loop than having to have to wait for the full PR build. The repository contains E2E tests that utilize the Helix SDK to send test Helix jobs. diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets index 90456cf2394..d988e488a17 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets @@ -27,6 +27,26 @@ + + + + $(HelixPostCommands); + find . -maxdepth 5 \( -iname '*.trx' -o -iname 'testResults.xml' -o -iname 'test-results.xml' -o -iname 'test_results.xml' -o -iname 'junit-results.xml' -o -iname 'junitresults.xml' \) -print0 | xargs -0 -I{} cp -f {} "$HELIX_WORKITEM_UPLOAD_ROOT/" || true + + + + + $(HelixPostCommands); + powershell -NoProfile -NonInteractive -Command "Get-ChildItem -Path . -Recurse -File -Depth 5 -Include *.trx,testResults.xml,test-results.xml,test_results.xml,junit-results.xml,junitresults.xml -ErrorAction SilentlyContinue | Copy-Item -Destination $env:HELIX_WORKITEM_UPLOAD_ROOT -Force -ErrorAction SilentlyContinue" + + + diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MultiQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MultiQueue.targets index a6a14c7df53..45f4c54b7c7 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MultiQueue.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MultiQueue.targets @@ -40,6 +40,15 @@ false + + + false + + Helix + + + false + + diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.props b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.props index e25086445d4..58b134f9447 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.props +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/azure-pipelines/AzurePipelines.props @@ -6,6 +6,15 @@ false + + + false + + true From d8bc2b69561ae733918e18d25b949d358f39c334 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 15:40:45 +0200 Subject: [PATCH 22/66] Turn on the feature for Arcade tests --- tests/UnitTests.proj | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/UnitTests.proj b/tests/UnitTests.proj index 75d73434482..39530e1087b 100644 --- a/tests/UnitTests.proj +++ b/tests/UnitTests.proj @@ -19,6 +19,11 @@ $(AGENT_JOBNAME) run on 300 + + + true From 5d62e56c27a24b82c4e7e467ea5d108b44cd1e36 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 15:54:27 +0200 Subject: [PATCH 23/66] Install the monitor tool from current job --- azure-pipelines-pr.yml | 4 ++ .../core-templates/job/helix-job-monitor.yml | 51 +++++++++++++++++++ .../stages/helix-job-monitor.yml | 14 +++++ 3 files changed, 69 insertions(+) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index d60a0b455c2..00bdb20fa12 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -367,3 +367,7 @@ stages: parameters: dependsOn: - build + # Install the Helix job monitor from the nupkg produced by the Build stage's + # Windows_NT Release job, instead of pulling it from a published feed. + toolNupkgArtifactName: Artifacts_Windows_NT_Release + toolNupkgArtifactSubPath: packages/Release/NonShipping diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 8a197e1d87d..b497425e0f1 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -44,6 +44,21 @@ parameters: type: string default: '' +# Optional pipeline artifact to download that contains the tool nupkg. When set, the artifact +# is downloaded and the directory containing the nupkg is added as a NuGet source for the +# 'dotnet tool install' command. Useful when the tool is built earlier in the same pipeline +# (e.g. the Arcade repo's Build stage) and has not yet been published to a feed. +- name: toolNupkgArtifactName + type: string + default: '' + +# Optional sub-path within the downloaded artifact where the tool nupkg is located. Defaults +# to the standard Arcade non-shipping packages location for a Release build (relative to the +# pipeline artifact root, which is itself the build's 'artifacts' directory). +- name: toolNupkgArtifactSubPath + type: string + default: 'packages/Release/NonShipping' + # Base URI for the Helix service (--helix-base-uri). - name: helixBaseUri type: string @@ -136,6 +151,16 @@ jobs: version: ${{ parameters.dotnetVersion }} includePreviewVersions: ${{ parameters.includePreviewVersions }} + - ${{ if ne(parameters.toolNupkgArtifactName, '') }}: + - task: DownloadPipelineArtifact@2 + displayName: Download Helix job monitor nupkg artifact + timeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} + inputs: + buildType: current + artifactName: ${{ parameters.toolNupkgArtifactName }} + itemPattern: '${{ parameters.toolNupkgArtifactSubPath }}/${{ parameters.toolPackageId }}.*.nupkg' + targetPath: $(Agent.TempDirectory)/helix-job-monitor-nupkg + - pwsh: | $toolPath = Join-Path $env:AGENT_TEMPDIRECTORY 'helix-job-monitor-tool' New-Item -ItemType Directory -Force -Path $toolPath | Out-Null @@ -146,12 +171,37 @@ jobs: $toolVersion = '${{ parameters.toolVersion }}' $toolSource = '${{ parameters.toolSource }}' + $nupkgArtifactName = '${{ parameters.toolNupkgArtifactName }}' + $nupkgArtifactSubPath = '${{ parameters.toolNupkgArtifactSubPath }}' + if (-not [string]::IsNullOrWhiteSpace($nupkgArtifactName)) { + $nupkgDir = Join-Path $env:AGENT_TEMPDIRECTORY (Join-Path 'helix-job-monitor-nupkg' $nupkgArtifactSubPath) + if (-not (Test-Path $nupkgDir)) { + throw "Expected nupkg directory '$nupkgDir' was not produced by the artifact download." + } + + $nupkg = Get-ChildItem -Path $nupkgDir -Filter "$packageId.*.nupkg" -File | Select-Object -First 1 + if ($null -eq $nupkg) { + throw "No '$packageId.*.nupkg' found in '$nupkgDir'." + } + + # Derive the version from the nupkg filename so the local package is selected + # deterministically instead of resolving against any other configured feed. + $derivedVersion = $nupkg.BaseName.Substring($packageId.Length + 1) + if ([string]::IsNullOrWhiteSpace($toolVersion)) { + $toolVersion = $derivedVersion + } + + Write-Host "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'." + $toolSource = $nupkgDir + } + $updateArgs = @('tool', 'update', '--tool-path', $toolPath, $packageId) if (-not [string]::IsNullOrWhiteSpace($toolVersion)) { $updateArgs += @('--version', $toolVersion) } if (-not [string]::IsNullOrWhiteSpace($toolSource)) { $updateArgs += @('--add-source', $toolSource) + $updateArgs += @('--ignore-failed-sources') } & dotnet @updateArgs @@ -162,6 +212,7 @@ jobs: } if (-not [string]::IsNullOrWhiteSpace($toolSource)) { $installArgs += @('--add-source', $toolSource) + $installArgs += @('--ignore-failed-sources') } & dotnet @installArgs diff --git a/eng/common/core-templates/stages/helix-job-monitor.yml b/eng/common/core-templates/stages/helix-job-monitor.yml index 66a0d792eed..186611fff55 100644 --- a/eng/common/core-templates/stages/helix-job-monitor.yml +++ b/eng/common/core-templates/stages/helix-job-monitor.yml @@ -59,6 +59,18 @@ parameters: type: string default: '' +# Optional pipeline artifact (produced earlier in this run) that contains the tool nupkg. +# When set, the artifact is downloaded and the directory containing the nupkg is added +# as a NuGet source for the 'dotnet tool install' command. +- name: toolNupkgArtifactName + type: string + default: '' + +# Optional sub-path within the downloaded artifact where the tool nupkg is located. +- name: toolNupkgArtifactSubPath + type: string + default: 'packages/Release/NonShipping' + # JobMonitorOptions: --helix-base-uri. - name: helixBaseUri type: string @@ -132,6 +144,8 @@ stages: toolCommand: ${{ parameters.toolCommand }} toolVersion: ${{ parameters.toolVersion }} toolSource: ${{ parameters.toolSource }} + toolNupkgArtifactName: ${{ parameters.toolNupkgArtifactName }} + toolNupkgArtifactSubPath: ${{ parameters.toolNupkgArtifactSubPath }} helixBaseUri: ${{ parameters.helixBaseUri }} helixAccessToken: ${{ parameters.helixAccessToken }} pollingIntervalSeconds: ${{ parameters.pollingIntervalSeconds }} From f9cb9a8f4be22f89072ee15ebd6fecaa9e895600 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 15:56:00 +0200 Subject: [PATCH 24/66] Move new parameters lower in the list --- .../core-templates/job/helix-job-monitor.yml | 31 ++++++++++--------- .../stages/helix-job-monitor.yml | 25 ++++++++------- 2 files changed, 29 insertions(+), 27 deletions(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index b497425e0f1..16e99545dd1 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -44,21 +44,6 @@ parameters: type: string default: '' -# Optional pipeline artifact to download that contains the tool nupkg. When set, the artifact -# is downloaded and the directory containing the nupkg is added as a NuGet source for the -# 'dotnet tool install' command. Useful when the tool is built earlier in the same pipeline -# (e.g. the Arcade repo's Build stage) and has not yet been published to a feed. -- name: toolNupkgArtifactName - type: string - default: '' - -# Optional sub-path within the downloaded artifact where the tool nupkg is located. Defaults -# to the standard Arcade non-shipping packages location for a Release build (relative to the -# pipeline artifact root, which is itself the build's 'artifacts' directory). -- name: toolNupkgArtifactSubPath - type: string - default: 'packages/Release/NonShipping' - # Base URI for the Helix service (--helix-base-uri). - name: helixBaseUri type: string @@ -125,6 +110,22 @@ parameters: type: string default: '' +# Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool +# nupkg. When set, the artifact is downloaded and the directory containing the nupkg is added +# as a NuGet source for the 'dotnet tool install' command. Primarily intended for the Arcade +# repository itself, where the Helix job monitor tool is built in the same pipeline that runs +# this template; other repos should leave this empty and consume the published feed instead. +- name: toolNupkgArtifactName + type: string + default: '' + +# Advanced: sub-path within the downloaded artifact where the tool nupkg is located. Defaults +# to the standard Arcade non-shipping packages location for a Release build (relative to the +# pipeline artifact root, which is itself the build's 'artifacts' directory). +- name: toolNupkgArtifactSubPath + type: string + default: 'packages/Release/NonShipping' + jobs: - job: ${{ parameters.jobName }} displayName: ${{ parameters.displayName }} diff --git a/eng/common/core-templates/stages/helix-job-monitor.yml b/eng/common/core-templates/stages/helix-job-monitor.yml index 186611fff55..d513d582b6f 100644 --- a/eng/common/core-templates/stages/helix-job-monitor.yml +++ b/eng/common/core-templates/stages/helix-job-monitor.yml @@ -59,18 +59,6 @@ parameters: type: string default: '' -# Optional pipeline artifact (produced earlier in this run) that contains the tool nupkg. -# When set, the artifact is downloaded and the directory containing the nupkg is added -# as a NuGet source for the 'dotnet tool install' command. -- name: toolNupkgArtifactName - type: string - default: '' - -# Optional sub-path within the downloaded artifact where the tool nupkg is located. -- name: toolNupkgArtifactSubPath - type: string - default: 'packages/Release/NonShipping' - # JobMonitorOptions: --helix-base-uri. - name: helixBaseUri type: string @@ -126,6 +114,19 @@ parameters: type: string default: '' +# Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool +# nupkg. When set, the artifact is downloaded and the directory containing the nupkg is added +# as a NuGet source for the 'dotnet tool install' command. Primarily intended for the Arcade +# repository itself; other repos should leave this empty and consume the published feed instead. +- name: toolNupkgArtifactName + type: string + default: '' + +# Advanced: sub-path within the downloaded artifact where the tool nupkg is located. +- name: toolNupkgArtifactSubPath + type: string + default: 'packages/Release/NonShipping' + stages: - stage: ${{ parameters.stageName }} displayName: ${{ parameters.displayName }} From 2d198a77f902b0364d5c13c7f11d5c3298f52c8e Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 16:13:48 +0200 Subject: [PATCH 25/66] Use windows-latest --- azure-pipelines-pr.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index 00bdb20fa12..8f9212476d7 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -64,8 +64,7 @@ stages: - job: Windows_NT timeoutInMinutes: 90 pool: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals windows.vs2026.amd64.open + vmImage: windows-latest # TODO: Testing only, revert strategy: matrix: Build_Release: From e37f3664ca06f6009964c3112ad6a425538f2171 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Thu, 23 Apr 2026 16:13:59 +0200 Subject: [PATCH 26/66] Read org + repo from envs --- .../core-templates/job/helix-job-monitor.yml | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 16e99545dd1..24756f976f3 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -75,17 +75,19 @@ parameters: default: Helix Job Monitor # Owner segment of the source repository (e.g. 'dotnet' for 'dotnet/runtime') passed via --organization. +# Defaults to the owner segment of BUILD_REPOSITORY_NAME when empty. - name: organization type: string default: '' # Name of the source repository (e.g. 'runtime' for 'dotnet/runtime') passed via --repository. +# Defaults to the repo segment of BUILD_REPOSITORY_NAME when empty. - name: repository type: string default: '' -# Pull request number being built (--pr-number). Required for PR validation pipelines; defaults -# to the value of SYSTEM_PULLREQUEST_PULLREQUESTNUMBER when running in Azure DevOps. +# Pull request number being built (--pr-number). Defaults to SYSTEM_PULLREQUEST_PULLREQUESTNUMBER +# when empty. - name: prNumber type: string default: '' @@ -246,6 +248,22 @@ jobs: $workingDirectory = '${{ parameters.workingDirectory }}' $attempt = '${{ parameters.attempt }}' + # Fall back to Azure DevOps-provided environment variables when the caller did not + # supply organization / repository / pr-number explicitly. BUILD_REPOSITORY_NAME is + # typically 'owner/repo' for GitHub-backed builds. + if ([string]::IsNullOrWhiteSpace($organization) -or [string]::IsNullOrWhiteSpace($repository)) { + $buildRepoName = $env:BUILD_REPOSITORY_NAME + if (-not [string]::IsNullOrWhiteSpace($buildRepoName) -and $buildRepoName.Contains('/')) { + $repoParts = $buildRepoName.Split('/', 2) + if ([string]::IsNullOrWhiteSpace($organization)) { $organization = $repoParts[0] } + if ([string]::IsNullOrWhiteSpace($repository)) { $repository = $repoParts[1] } + } + } + + if ([string]::IsNullOrWhiteSpace($prNumber)) { + $prNumber = $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER + } + if (-not [string]::IsNullOrWhiteSpace($organization)) { $toolArgs += @('--organization', $organization) } if (-not [string]::IsNullOrWhiteSpace($repository)) { $toolArgs += @('--repository', $repository) } if (-not [string]::IsNullOrWhiteSpace($prNumber)) { $toolArgs += @('--pr-number', $prNumber) } From 6c2a4f6acf687320bf6e28ec97eda21de07fe27f Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 12:33:58 +0200 Subject: [PATCH 27/66] Rewrite pwsh to bash, drop some template arguments --- .../core-templates/job/helix-job-monitor.yml | 261 ++++++++---------- .../stages/helix-job-monitor.yml | 30 -- 2 files changed, 112 insertions(+), 179 deletions(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 24756f976f3..43079311f15 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -14,16 +14,6 @@ parameters: type: object default: {} -# Version of the .NET SDK installed on the agent before running the tool. -- name: dotnetVersion - type: string - default: 11.0.x - -# Whether the UseDotNet@2 task is allowed to install preview SDKs. -- name: includePreviewVersions - type: boolean - default: true - # NuGet package id of the Helix job monitor tool. - name: toolPackageId type: string @@ -34,12 +24,15 @@ parameters: type: string default: dotnet-helix-job-monitor -# Optional explicit tool version. When empty, the latest available version is installed. +# Optional explicit tool version. Only honored when 'toolNupkgArtifactName' is set; in the +# default code path the version is taken from the consuming repo's .config/dotnet-tools.json. - name: toolVersion type: string default: '' -# Optional NuGet feed used as an additional source when installing the tool. +# Optional NuGet feed used as an additional source when installing the tool. Only honored +# when 'toolNupkgArtifactName' is set; in the default code path the tool is restored from +# the consuming repo's .config/dotnet-tools.json manifest and no extra feeds are consulted. - name: toolSource type: string default: '' @@ -64,11 +57,6 @@ parameters: type: number default: 360 -# Per-step timeout in minutes for the install / run steps. -- name: stepTimeoutInMinutes - type: number - default: 360 - # Display name reported by the tool to Azure DevOps (--job-monitor-name). - name: jobMonitorName type: string @@ -92,16 +80,6 @@ parameters: type: string default: '' -# Optional working directory used to stage downloaded Helix test results (--working-directory). -- name: workingDirectory - type: string - default: '' - -# Optional Azure DevOps job attempt number (--attempt). Defaults to SYSTEM_JOBATTEMPT. -- name: attempt - type: string - default: '' - # Optional dependency list for the generated job. - name: dependsOn type: object @@ -113,10 +91,14 @@ parameters: default: '' # Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool -# nupkg. When set, the artifact is downloaded and the directory containing the nupkg is added -# as a NuGet source for the 'dotnet tool install' command. Primarily intended for the Arcade -# repository itself, where the Helix job monitor tool is built in the same pipeline that runs -# this template; other repos should leave this empty and consume the published feed instead. +# nupkg. When set, the artifact is downloaded and the tool is installed from the nupkg into +# a local tool-path; this bypasses the repo's .config/dotnet-tools.json manifest and is +# primarily intended for the Arcade repository itself, where the Helix job monitor tool is +# built in the same pipeline that runs this template. +# +# When this parameter is empty (the default), the consuming repository must declare the tool +# in its .config/dotnet-tools.json manifest (alongside other local .NET tools); the template +# will check out the repo and run 'dotnet tool restore' to install the version pinned there. - name: toolNupkgArtifactName type: string default: '' @@ -144,139 +126,120 @@ jobs: name: $(DncEngInternalBuildPool) image: build.azurelinux.3.amd64.open steps: - - checkout: none - - - task: UseDotNet@2 - displayName: Install .NET SDK - timeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} - inputs: - packageType: sdk - version: ${{ parameters.dotnetVersion }} - includePreviewVersions: ${{ parameters.includePreviewVersions }} + - ${{ if ne(parameters.toolNupkgArtifactName, '') }}: + - checkout: none + - ${{ else }}: + # The default code path restores the tool from the repo's .config/dotnet-tools.json, + # so the repository content (including the manifest and global.json) must be available. + - checkout: self + fetchDepth: 1 - ${{ if ne(parameters.toolNupkgArtifactName, '') }}: - task: DownloadPipelineArtifact@2 - displayName: Download Helix job monitor nupkg artifact - timeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} + displayName: Download Helix Job Monitor artifact inputs: buildType: current artifactName: ${{ parameters.toolNupkgArtifactName }} itemPattern: '${{ parameters.toolNupkgArtifactSubPath }}/${{ parameters.toolPackageId }}.*.nupkg' targetPath: $(Agent.TempDirectory)/helix-job-monitor-nupkg - - pwsh: | - $toolPath = Join-Path $env:AGENT_TEMPDIRECTORY 'helix-job-monitor-tool' - New-Item -ItemType Directory -Force -Path $toolPath | Out-Null - - Push-Location $env:AGENT_TEMPDIRECTORY - try { - $packageId = '${{ parameters.toolPackageId }}' - $toolVersion = '${{ parameters.toolVersion }}' - $toolSource = '${{ parameters.toolSource }}' - - $nupkgArtifactName = '${{ parameters.toolNupkgArtifactName }}' - $nupkgArtifactSubPath = '${{ parameters.toolNupkgArtifactSubPath }}' - if (-not [string]::IsNullOrWhiteSpace($nupkgArtifactName)) { - $nupkgDir = Join-Path $env:AGENT_TEMPDIRECTORY (Join-Path 'helix-job-monitor-nupkg' $nupkgArtifactSubPath) - if (-not (Test-Path $nupkgDir)) { - throw "Expected nupkg directory '$nupkgDir' was not produced by the artifact download." - } - - $nupkg = Get-ChildItem -Path $nupkgDir -Filter "$packageId.*.nupkg" -File | Select-Object -First 1 - if ($null -eq $nupkg) { - throw "No '$packageId.*.nupkg' found in '$nupkgDir'." - } - - # Derive the version from the nupkg filename so the local package is selected - # deterministically instead of resolving against any other configured feed. - $derivedVersion = $nupkg.BaseName.Substring($packageId.Length + 1) - if ([string]::IsNullOrWhiteSpace($toolVersion)) { - $toolVersion = $derivedVersion - } - - Write-Host "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'." - $toolSource = $nupkgDir - } - - $updateArgs = @('tool', 'update', '--tool-path', $toolPath, $packageId) - if (-not [string]::IsNullOrWhiteSpace($toolVersion)) { - $updateArgs += @('--version', $toolVersion) - } - if (-not [string]::IsNullOrWhiteSpace($toolSource)) { - $updateArgs += @('--add-source', $toolSource) - $updateArgs += @('--ignore-failed-sources') - } - - & dotnet @updateArgs - if ($LASTEXITCODE -ne 0) { - $installArgs = @('tool', 'install', '--tool-path', $toolPath, $packageId) - if (-not [string]::IsNullOrWhiteSpace($toolVersion)) { - $installArgs += @('--version', $toolVersion) - } - if (-not [string]::IsNullOrWhiteSpace($toolSource)) { - $installArgs += @('--add-source', $toolSource) - $installArgs += @('--ignore-failed-sources') - } - - & dotnet @installArgs - } - - if ($LASTEXITCODE -ne 0) { - throw "Failed to install the Helix job monitor tool package '$packageId'." - } - - Write-Host "##vso[task.prependpath]$toolPath" - Write-Host "##vso[task.setvariable variable=HelixJobMonitorToolPath]$toolPath" - } - finally { - Pop-Location - } - displayName: Install Helix job monitor - timeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} - - - pwsh: | - $toolArgs = @( - '--helix-base-uri', '${{ parameters.helixBaseUri }}', - '--polling-interval-seconds', '${{ parameters.pollingIntervalSeconds }}', - '--max-wait-minutes', '${{ parameters.timeoutInMinutes }}', - '--job-monitor-name', '${{ parameters.jobMonitorName }}' + - bash: | + set -euo pipefail + + toolPath="$AGENT_TEMPDIRECTORY/helix-job-monitor-tool" + mkdir -p "$toolPath" + + pushd "$AGENT_TEMPDIRECTORY" > /dev/null + trap 'popd > /dev/null' EXIT + + packageId='${{ parameters.toolPackageId }}' + toolVersion='${{ parameters.toolVersion }}' + nupkgArtifactSubPath='${{ parameters.toolNupkgArtifactSubPath }}' + nupkgDir="$AGENT_TEMPDIRECTORY/helix-job-monitor-nupkg/$nupkgArtifactSubPath" + + if [ ! -d "$nupkgDir" ]; then + echo "Expected nupkg directory '$nupkgDir' was not produced by the artifact download." >&2 + exit 1 + fi + + nupkg=$(find "$nupkgDir" -maxdepth 1 -type f -name "$packageId.*.nupkg" | head -n 1) + if [ -z "$nupkg" ]; then + echo "No '$packageId.*.nupkg' found in '$nupkgDir'." >&2 + exit 1 + fi + + # Derive the version from the nupkg filename so the local package is selected + # deterministically instead of resolving against any other configured feed. + nupkgBase=$(basename "$nupkg" .nupkg) + derivedVersion="${nupkgBase#${packageId}.}" + if [ -z "$toolVersion" ]; then + toolVersion="$derivedVersion" + fi + + echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'." + toolSource="$nupkgDir" + + ./eng/common/dotnet.sh tool install \ + --tool-path "$toolPath" "$packageId" \ + --version "$toolVersion" \ + --add-source "$toolSource" \ + --ignore-failed-sources + + echo "##vso[task.prependpath]$toolPath" + echo "##vso[task.setvariable variable=HelixJobMonitorToolPath]$toolPath" + displayName: Install Helix Job Monitor + + - ${{ else }}: + - bash: ./eng/common/dotnet.sh tool restore + displayName: Restore Helix Job Monitor + + - bash: | + set -euo pipefail + + toolArgs=( + --helix-base-uri '${{ parameters.helixBaseUri }}' + --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' + --max-wait-minutes '${{ parameters.timeoutInMinutes }}' + --job-monitor-name '${{ parameters.jobMonitorName }}' + --attempt '$(System.JobAttempt)' ) - $organization = '${{ parameters.organization }}' - $repository = '${{ parameters.repository }}' - $prNumber = '${{ parameters.prNumber }}' - $workingDirectory = '${{ parameters.workingDirectory }}' - $attempt = '${{ parameters.attempt }}' + organization='${{ parameters.organization }}' + repository='${{ parameters.repository }}' + prNumber='${{ parameters.prNumber }}' # Fall back to Azure DevOps-provided environment variables when the caller did not # supply organization / repository / pr-number explicitly. BUILD_REPOSITORY_NAME is # typically 'owner/repo' for GitHub-backed builds. - if ([string]::IsNullOrWhiteSpace($organization) -or [string]::IsNullOrWhiteSpace($repository)) { - $buildRepoName = $env:BUILD_REPOSITORY_NAME - if (-not [string]::IsNullOrWhiteSpace($buildRepoName) -and $buildRepoName.Contains('/')) { - $repoParts = $buildRepoName.Split('/', 2) - if ([string]::IsNullOrWhiteSpace($organization)) { $organization = $repoParts[0] } - if ([string]::IsNullOrWhiteSpace($repository)) { $repository = $repoParts[1] } - } - } - - if ([string]::IsNullOrWhiteSpace($prNumber)) { - $prNumber = $env:SYSTEM_PULLREQUEST_PULLREQUESTNUMBER - } - - if (-not [string]::IsNullOrWhiteSpace($organization)) { $toolArgs += @('--organization', $organization) } - if (-not [string]::IsNullOrWhiteSpace($repository)) { $toolArgs += @('--repository', $repository) } - if (-not [string]::IsNullOrWhiteSpace($prNumber)) { $toolArgs += @('--pr-number', $prNumber) } - if (-not [string]::IsNullOrWhiteSpace($workingDirectory)) { $toolArgs += @('--working-directory', $workingDirectory) } - if (-not [string]::IsNullOrWhiteSpace($attempt)) { $toolArgs += @('--attempt', $attempt) } - - & '${{ parameters.toolCommand }}' @toolArgs - - if ($LASTEXITCODE -ne 0) { - throw "The Helix job monitor tool exited with code $LASTEXITCODE." - } - displayName: Run Helix job monitor - timeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} + if [ -z "$organization" ] || [ -z "$repository" ]; then + buildRepoName="${BUILD_REPOSITORY_NAME:-}" + if [ -n "$buildRepoName" ] && [[ "$buildRepoName" == */* ]]; then + repoOwner="${buildRepoName%%/*}" + repoName="${buildRepoName#*/}" + if [ -z "$organization" ]; then organization="$repoOwner"; fi + if [ -z "$repository" ]; then repository="$repoName"; fi + fi + fi + + if [ -z "$prNumber" ]; then + prNumber="${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-}" + fi + + if [ -n "$organization" ]; then toolArgs+=( --organization "$organization" ); fi + if [ -n "$repository" ]; then toolArgs+=( --repository "$repository" ); fi + if [ -n "$prNumber" ]; then toolArgs+=( --pr-number "$prNumber" ); fi + + if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then + # Tool was installed into a tool-path that has been prepended to PATH. + '${{ parameters.toolCommand }}' "${toolArgs[@]}" + else + # Tool was restored from the local .config/dotnet-tools.json manifest; invoke it + # through the manifest from the repo root. + pushd "$BUILD_SOURCESDIRECTORY" > /dev/null + trap 'popd > /dev/null' EXIT + ./eng/common/dotnet.sh tool run '${{ parameters.toolCommand }}' -- "${toolArgs[@]}" + fi + displayName: Run Helix Job Monitor env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }} diff --git a/eng/common/core-templates/stages/helix-job-monitor.yml b/eng/common/core-templates/stages/helix-job-monitor.yml index d513d582b6f..5dbb393dd63 100644 --- a/eng/common/core-templates/stages/helix-job-monitor.yml +++ b/eng/common/core-templates/stages/helix-job-monitor.yml @@ -29,16 +29,6 @@ parameters: type: string default: Helix Job Monitor -# .NET SDK installed before running the tool. -- name: dotnetVersion - type: string - default: 11.0.x - -# Whether the UseDotNet@2 task is allowed to install preview SDKs. -- name: includePreviewVersions - type: boolean - default: true - # NuGet package id of the Helix job monitor tool. - name: toolPackageId type: string @@ -79,11 +69,6 @@ parameters: type: number default: 360 -# Per-step timeout in minutes for the install / run steps. -- name: stepTimeoutInMinutes - type: number - default: 360 - # JobMonitorOptions: --job-monitor-name. - name: jobMonitorName type: string @@ -104,16 +89,6 @@ parameters: type: string default: '' -# JobMonitorOptions: --working-directory. -- name: workingDirectory - type: string - default: '' - -# JobMonitorOptions: --attempt. Defaults to SYSTEM_JOBATTEMPT inside the tool. -- name: attempt - type: string - default: '' - # Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool # nupkg. When set, the artifact is downloaded and the directory containing the nupkg is added # as a NuGet source for the 'dotnet tool install' command. Primarily intended for the Arcade @@ -139,8 +114,6 @@ stages: parameters: jobName: ${{ parameters.jobName }} displayName: ${{ parameters.jobDisplayName }} - dotnetVersion: ${{ parameters.dotnetVersion }} - includePreviewVersions: ${{ parameters.includePreviewVersions }} toolPackageId: ${{ parameters.toolPackageId }} toolCommand: ${{ parameters.toolCommand }} toolVersion: ${{ parameters.toolVersion }} @@ -151,10 +124,7 @@ stages: helixAccessToken: ${{ parameters.helixAccessToken }} pollingIntervalSeconds: ${{ parameters.pollingIntervalSeconds }} timeoutInMinutes: ${{ parameters.timeoutInMinutes }} - stepTimeoutInMinutes: ${{ parameters.stepTimeoutInMinutes }} jobMonitorName: ${{ parameters.jobMonitorName }} organization: ${{ parameters.organization }} repository: ${{ parameters.repository }} prNumber: ${{ parameters.prNumber }} - workingDirectory: ${{ parameters.workingDirectory }} - attempt: ${{ parameters.attempt }} From dd52579b2f43868c65e5d2183d8140e69fe25c8e Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 12:50:24 +0200 Subject: [PATCH 28/66] Stop using the pass/fail endpoint --- .../AzureDevOpsResultPublisher.cs | 46 ++++++------- .../JobMonitor/JobMonitorRunner.cs | 67 ++++++++++++------- 2 files changed, 64 insertions(+), 49 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs index 9bfa4895999..05e136ce4a5 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs @@ -36,7 +36,7 @@ public AzureDevOpsResultPublisher( _logger = logger; } - public async Task UploadTestResultsAsync(List testResultFiles, object resultMetadata, CancellationToken cancellationToken = default) + public async Task UploadTestResultsAsync(List testResultFiles, object resultMetadata, CancellationToken cancellationToken = default) { var testResultReader = new LocalTestResultsReader(_logger); @@ -45,24 +45,35 @@ public async Task UploadTestResultsAsync(List testResultFiles, object re if (parsedResults.Length == 0) { _logger.LogWarning("No test results were discovered under."); - return; + return true; } IReadOnlyList aggregatedResults = new ResultAggregator().Aggregate(parsedResults); if (aggregatedResults.Count == 0) { _logger.LogWarning("Test results were discovered but none could be aggregated."); - return; + return true; } await UploadTestResultsAsync(aggregatedResults, resultMetadata, cancellationToken); + return aggregatedResults.All(result => result.Result != "Failed"); // TODO: maybe there's a better way to find out if a test failed? Is this extensive enough? } public async Task UploadTestResultsAsync(IEnumerable results, object resultMetadata, CancellationToken cancellationToken = default) { try { - await ProcessAsync([.. results], resultMetadata, cancellationToken); + var converted = ConvertResults(results, resultMetadata).ToList(); + var hotPathTests = new List(); + + foreach (List batch in Batch(converted, 1000, static t => Size(t.Converted))) + { + IReadOnlyList publishedTests = await PublishResultsAsync(batch, cancellationToken); + hotPathTests.AddRange(publishedTests); + _logger.LogInformation("Uploaded {Count} results", publishedTests.Count); + } + + await SendMetadataAsync(hotPathTests, results, cancellationToken); } catch (TerminalError ex) { @@ -71,21 +82,6 @@ public async Task UploadTestResultsAsync(IEnumerable results, } } - private async Task ProcessAsync(IReadOnlyList testList, object resultMetadata, CancellationToken cancellationToken) - { - var converted = ConvertResults(testList, resultMetadata).ToList(); - var hotPathTests = new List(); - - foreach (List batch in Batch(converted, 1000, static t => Size(t.Converted))) - { - IReadOnlyList publishedTests = await PublishResultsAsync(batch, cancellationToken); - hotPathTests.AddRange(publishedTests); - _logger.LogInformation("Uploaded {Count} results", publishedTests.Count); - } - - await SendMetadataAsync(hotPathTests, testList, cancellationToken); - } - private async Task LogErrorAsync(Exception exception, CancellationToken cancellationToken) { _logger.LogError(exception, "Failed to upload test results to Azure DevOps."); @@ -98,7 +94,7 @@ await _eventClient.ErrorAsync( */ } - private async Task SendMetadataAsync( + private static async Task SendMetadataAsync( IReadOnlyList backChannelCases, IEnumerable allTestResults, CancellationToken cancellationToken) @@ -179,11 +175,11 @@ void ProcessTestForMetadata(AggregatedResult result) string base64Data = Convert.ToBase64String(compressedBytes); string fileNameBase = $"__helix_metadata_{Guid.NewGuid():N}.json.gz"; - await SendWithRetryAsync( - HttpMethod.Post, - $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/attachments?api-version=7.1-preview.1", - new TestRunAttachmentRequest(fileNameBase, base64Data), - cancellationToken); + //await SendWithRetryAsync( + // HttpMethod.Post, + // $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/attachments?api-version=7.1-preview.1", + // new TestRunAttachmentRequest(fileNameBase, base64Data), + // cancellationToken); /* TODO string metadataUrl = await _uploadClient.UploadAsync(compressedBytes, fileNameBase, "application/gzip", cancellationToken); diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index b1cf1024ef1..dd34f8dd3e8 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -79,7 +79,7 @@ public async Task RunAsync() .OrderBy(j => j.Name, StringComparer.OrdinalIgnoreCase) ]; - _logger.LogInformation("{CompletedCount}/{TotalCount} Helix jobs complete. Waiting...", completedJobs.Count, jobs.Count); + _logger.LogInformation("{CompletedCount}/{TotalCount} Helix jobs complete", completedJobs.Count, jobs.Count); foreach (JobSummary completedJob in completedJobs) { @@ -88,8 +88,7 @@ public async Task RunAsync() continue; } - JobPassFail passFail = await RetryAsync(() => _helixApi.Job.PassFailAsync(completedJob.Name, cancellationToken), cancellationToken); - bool passed = await ProcessCompletedJobAsync(completedJob, passFail, cancellationToken); + bool passed = await ProcessCompletedJobAsync(completedJob, cancellationToken); processedRuns.Add(completedJob.Name); processedHelixJobCount++; if (!passed) @@ -132,17 +131,30 @@ public void Dispose() _azdoClient.Dispose(); } - private async Task ProcessCompletedJobAsync(JobSummary helixJob, JobPassFail passFail, CancellationToken cancellationToken) + private async Task ProcessCompletedJobAsync(JobSummary helixJob, CancellationToken cancellationToken) { + _logger.LogInformation("Processing completed job {jobName}...", helixJob.Name); + string testRunName = HelixJobMonitorUtilities.GetTestRunName(helixJob.Name); int testRunId = await StartTestRunAsync(testRunName); string resultsDirectory = Path.Combine(_options.WorkingDirectory, SanitizeDirName(helixJob.Name)); Directory.CreateDirectory(resultsDirectory); + IImmutableList workItems = await RetryAsync(() => _helixApi.WorkItem.ListAsync(helixJob.Name), cancellationToken); + + int failedWorkItemCount = workItems.Count(wi => wi.ExitCode != 0 || !wi.State.Equals("Finished", StringComparison.OrdinalIgnoreCase)); + bool helixJobSuccessful = failedWorkItemCount == 0; + int sucessfulWorkItemCount = workItems.Count - failedWorkItemCount; + try { - List downloadedFiles = await DownloadTestResultsAsync(helixJob.Name, passFail, resultsDirectory, cancellationToken); - await UploadDownloadedResultsAsync(downloadedFiles, testRunId, cancellationToken); + List downloadedFiles = await DownloadTestResultsAsync(helixJob.Name, workItems, resultsDirectory, cancellationToken); + if (!await UploadDownloadedResultsAsync(downloadedFiles, testRunId, cancellationToken)) + { + sucessfulWorkItemCount--; + failedWorkItemCount++; + helixJobSuccessful = false; + } } catch (Exception ex) { @@ -153,10 +165,8 @@ private async Task ProcessCompletedJobAsync(JobSummary helixJob, JobPassFa await StopTestRunAsync(testRunId, testRunName); - int passedCount = passFail.Passed?.Count ?? 0; - int failedCount = passFail.Failed?.Count ?? 0; - _logger.LogInformation("Job '{JobName}' completed ({PassedCount} passed, {FailedCount} failed).", helixJob.Name, passedCount, failedCount); - return failedCount == 0; + _logger.LogInformation("Job '{JobName}' completed ({PassedCount} passed, {FailedCount} failed).", helixJob.Name, sucessfulWorkItemCount, failedWorkItemCount); + return failedWorkItemCount == 0; } private async Task> GetProcessedRunNamesAsync() @@ -217,7 +227,7 @@ await SendAsync( private async Task> DownloadTestResultsAsync( string jobName, - JobPassFail passFail, + IImmutableList workItems, string outputDirectory, CancellationToken cancellationToken) { @@ -227,21 +237,25 @@ private async Task> DownloadTestResultsAsync( () => _helixApi.Job.ResultsAsync(jobName), cancellationToken); - IEnumerable workItemNames = (passFail.Passed ?? []) - .Concat(passFail.Failed ?? []) - .Distinct(StringComparer.OrdinalIgnoreCase); - - foreach (string workItemName in workItemNames) + foreach (WorkItemSummary workItem in workItems) { IImmutableList availableFiles = await RetryAsync( - () => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false), + () => _helixApi.WorkItem.ListFilesAsync(workItem.Name, jobName, false), cancellationToken); - string workItemDirectory = Path.Combine(outputDirectory, SanitizeDirName(workItemName)); + availableFiles = [.. availableFiles.Where(f => LooksLikeTestResultFile(f.Name))]; + + if (availableFiles.Count == 0) + { + _logger.LogInformation("Work item '{WorkItemName}' in job '{JobName}' has no test result files to download.", workItem.Name, jobName); + continue; + } + + string workItemDirectory = Path.Combine(outputDirectory, SanitizeDirName(workItem.Name)); Directory.CreateDirectory(workItemDirectory); List workItemFiles = []; - foreach (UploadedFile file in availableFiles.Where(f => LooksLikeTestResultFile(f.Name))) + foreach (UploadedFile file in availableFiles) { string relativePath = file.Name.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar); string destinationFile = Path.Combine(workItemDirectory, relativePath); @@ -254,7 +268,7 @@ private async Task> DownloadTestResultsAsync( try { - _logger.LogInformation("Downloading {FileName} for work item {WorkItemName} in job {JobName}...", file.Name, workItemName, jobName); + _logger.LogInformation("Downloading {FileName} for work item {WorkItemName} in job {JobName}...", file.Name, workItem.Name, jobName); BlobClient blobClient = CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS); await blobClient.DownloadToAsync(destinationFile, cancellationToken); @@ -262,17 +276,17 @@ private async Task> DownloadTestResultsAsync( } catch (Exception ex) { - _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItemName); + _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItem.Name); } } - downloadedFiles.Add(new WorkItemTestResults(jobName, workItemName, workItemFiles)); + downloadedFiles.Add(new WorkItemTestResults(jobName, workItem.Name, workItemFiles)); } return downloadedFiles; } - private async Task UploadDownloadedResultsAsync(List testResults, int testRunId, CancellationToken cancellationToken) + private async Task UploadDownloadedResultsAsync(List testResults, int testRunId, CancellationToken cancellationToken) { var publisher = new AzureDevOpsResultPublisher( new AzureDevOpsReportingParameters( @@ -282,9 +296,12 @@ private async Task UploadDownloadedResultsAsync(List testRe _options.SystemAccessToken), _logger); + bool allTestsPassed = true; + foreach (WorkItemTestResults workItemTestResult in testResults) { - await publisher.UploadTestResultsAsync( + _logger.LogInformation("Publishing test results for work item '{WorkItemName}' in job '{JobName}'...", workItemTestResult.WorkItemName, workItemTestResult.JobName); + allTestsPassed &= await publisher.UploadTestResultsAsync( workItemTestResult.TestResultFiles, // Metadata that will be appended to each test case new @@ -294,6 +311,8 @@ await publisher.UploadTestResultsAsync( }, cancellationToken); } + + return allTestsPassed; } private async Task SendAsync(HttpMethod method, string requestUri, JToken body = null, CancellationToken cancellationToken = default) From ae59023a4e41dd7ff9134a791813957520a0ead1 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 15:35:25 +0200 Subject: [PATCH 29/66] Remove a package reference --- .../Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj index 6db527d0af5..13de144da67 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj @@ -1,4 +1,4 @@ - + $(BundledNETCoreAppTargetFramework) @@ -6,8 +6,4 @@ enable - - - - From e2caf8d77fb524f973c6fcbc6d3f0da5dd79ab7b Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 15:42:00 +0200 Subject: [PATCH 30/66] Fix getting previous runs --- src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index dd34f8dd3e8..770dad9f9ef 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -183,7 +183,7 @@ private async Task> GetProcessedRunNamesAsync() string name = run.Value("name"); string state = run.Value("state"); if (!string.IsNullOrEmpty(name) - && !name.StartsWith("Helix Job Monitor - ", StringComparison.OrdinalIgnoreCase) + && name.StartsWith("Helix Job Monitor - ", StringComparison.OrdinalIgnoreCase) && string.Equals(state, "Completed", StringComparison.OrdinalIgnoreCase)) { processed.Add(name); From 9e6f72ab23d9b4e52a471ac74d263827960305f8 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 15:55:18 +0200 Subject: [PATCH 31/66] Add a simple console formatter --- .../CompactConsoleLoggerFormatter.cs | 194 ++++++++++++++++++ .../JobMonitor/Program.cs | 9 +- 2 files changed, 197 insertions(+), 6 deletions(-) create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/CompactConsoleLoggerFormatter.cs diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/CompactConsoleLoggerFormatter.cs b/src/Microsoft.DotNet.Helix/JobMonitor/CompactConsoleLoggerFormatter.cs new file mode 100644 index 00000000000..b2c3a91e654 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/CompactConsoleLoggerFormatter.cs @@ -0,0 +1,194 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Logging.Console; +using Microsoft.Extensions.Options; + +#nullable enable +namespace Microsoft.DotNet.Helix.JobMonitor; + +/// +/// Copied over from SimpleConsoleFormatter. Leaves out the logger name and new line, turning +/// info: test[0] +/// Log message +/// Second line of the message +/// +/// into +/// +/// info: Log message +/// Second line of the message +/// +/// Only using SimpleConsoleFormatterOptions.SingleLine didn't help because multi-line messages +/// were put together on a single line so things like stack traces of exceptions were unreadable. +/// +/// See https://github.com/dotnet/runtime/blob/0817e748b7698bef1e812fd74c8a3558b7f86421/src/libraries/Microsoft.Extensions.Logging.Console/src/SimpleConsoleFormatter.cs +/// +public class CompactConsoleLoggerFormatter : ConsoleFormatter +{ + private const string LoglevelPadding = ": "; + private const string DefaultForegroundColor = "\x1B[39m\x1B[22m"; // reset to default foreground color + private const string DefaultBackgroundColor = "\x1B[49m"; // reset to the background color + + public const string FormatterName = "compact"; + + private readonly SimpleConsoleFormatterOptions _options; + private readonly string _messagePadding; + private readonly string _newLineWithMessagePadding; + + public CompactConsoleLoggerFormatter(IOptionsMonitor options) + : base(FormatterName) + { + _options = options.CurrentValue; + _messagePadding = new string(' ', GetLogLevelString(LogLevel.Information).Length + LoglevelPadding.Length + (_options.TimestampFormat?.Length ?? 0)); + _newLineWithMessagePadding = Environment.NewLine + _messagePadding; + } + + public override void Write(in LogEntry logEntry, IExternalScopeProvider? scopeProvider, TextWriter textWriter) + { + if (logEntry.Formatter == null) + { + return; + } + + var message = logEntry.Formatter(logEntry.State, logEntry.Exception); + if (logEntry.Exception == null && message == null) + { + return; + } + + LogLevel logLevel = logEntry.LogLevel; + var logLevelColors = GetLogLevelConsoleColors(logLevel); + var logLevelString = GetLogLevelString(logLevel); + + if (_options.TimestampFormat != null) + { + var timestamp = DateTimeOffset.Now.ToString(_options.TimestampFormat); + textWriter.Write(timestamp); + } + + WriteColoredMessage(textWriter, logLevelString, logLevelColors.Background, logLevelColors.Foreground); + + textWriter.Write(LoglevelPadding); + + WriteMessage(textWriter, message, false); + + // Example: + // System.InvalidOperationException + // at Namespace.Class.Function() in File:line X + if (logEntry.Exception != null) + { + // exception message + WriteMessage(textWriter, logEntry.Exception.ToString()); + } + } + + private void WriteMessage(TextWriter textWriter, string message, bool includePadding = true) + { + if (message == null) + { + return; + } + + if (includePadding) + { + textWriter.Write(_messagePadding); + } + + textWriter.WriteLine(message.Replace(Environment.NewLine, _newLineWithMessagePadding)); + } + + private static string GetLogLevelString(LogLevel logLevel) => logLevel switch + { + LogLevel.Trace => "trce", + LogLevel.Debug => "dbug", + LogLevel.Information => "info", + LogLevel.Warning => "warn", + LogLevel.Error => "fail", + LogLevel.Critical => "crit", + _ => throw new ArgumentOutOfRangeException(nameof(logLevel)) + }; + + private (ConsoleColor? Foreground, ConsoleColor? Background) GetLogLevelConsoleColors(LogLevel logLevel) + { + if (_options.ColorBehavior == LoggerColorBehavior.Disabled) + { + return (null, null); + } + + // We must explicitly set the background color if we are setting the foreground color, + // since just setting one can look bad on the users console. + return logLevel switch + { + LogLevel.Trace => (ConsoleColor.Gray, ConsoleColor.Black), + LogLevel.Debug => (ConsoleColor.Gray, ConsoleColor.Black), + LogLevel.Information => (ConsoleColor.DarkGreen, ConsoleColor.Black), + LogLevel.Warning => (ConsoleColor.Yellow, ConsoleColor.Black), + LogLevel.Error => (ConsoleColor.Black, ConsoleColor.DarkRed), + LogLevel.Critical => (ConsoleColor.White, ConsoleColor.DarkRed), + _ => (null, null) + }; + } + + private static void WriteColoredMessage(TextWriter textWriter, string message, ConsoleColor? background, ConsoleColor? foreground) + { + // Order: backgroundcolor, foregroundcolor, Message, reset foregroundcolor, reset backgroundcolor + if (background.HasValue) + { + textWriter.Write(GetBackgroundColorEscapeCode(background.Value)); + } + + if (foreground.HasValue) + { + textWriter.Write(GetForegroundColorEscapeCode(foreground.Value)); + } + + textWriter.Write(message); + + if (foreground.HasValue) + { + textWriter.Write(DefaultForegroundColor); // reset to default foreground color + } + + if (background.HasValue) + { + textWriter.Write(DefaultBackgroundColor); // reset to the background color + } + } + + private static string GetForegroundColorEscapeCode(ConsoleColor color) => color switch + { + ConsoleColor.Black => "\x1B[30m", + ConsoleColor.DarkRed => "\x1B[31m", + ConsoleColor.DarkGreen => "\x1B[32m", + ConsoleColor.DarkYellow => "\x1B[33m", + ConsoleColor.DarkBlue => "\x1B[34m", + ConsoleColor.DarkMagenta => "\x1B[35m", + ConsoleColor.DarkCyan => "\x1B[36m", + ConsoleColor.Gray => "\x1B[37m", + ConsoleColor.Red => "\x1B[1m\x1B[31m", + ConsoleColor.Green => "\x1B[1m\x1B[32m", + ConsoleColor.Yellow => "\x1B[1m\x1B[33m", + ConsoleColor.Blue => "\x1B[1m\x1B[34m", + ConsoleColor.Magenta => "\x1B[1m\x1B[35m", + ConsoleColor.Cyan => "\x1B[1m\x1B[36m", + ConsoleColor.White => "\x1B[1m\x1B[37m", + _ => DefaultForegroundColor // default foreground color + }; + + private static string GetBackgroundColorEscapeCode(ConsoleColor color) => color switch + { + ConsoleColor.Black => "\x1B[40m", + ConsoleColor.DarkRed => "\x1B[41m", + ConsoleColor.DarkGreen => "\x1B[42m", + ConsoleColor.DarkYellow => "\x1B[43m", + ConsoleColor.DarkBlue => "\x1B[44m", + ConsoleColor.DarkMagenta => "\x1B[45m", + ConsoleColor.DarkCyan => "\x1B[46m", + ConsoleColor.Gray => "\x1B[47m", + _ => DefaultBackgroundColor // Use default background color + }; +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs index 5b33c4b3cc7..515d9b8ac80 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Program.cs @@ -4,6 +4,7 @@ using System; using System.Threading.Tasks; using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Console; namespace Microsoft.DotNet.Helix.JobMonitor { @@ -15,12 +16,8 @@ public static async Task Main(string[] args) { builder .SetMinimumLevel(LogLevel.Information) - .AddSimpleConsole(options => - { - options.SingleLine = true; - options.TimestampFormat = "[HH:mm:ss] "; - options.IncludeScopes = false; - }); + .AddConsole(o => o.FormatterName = CompactConsoleLoggerFormatter.FormatterName) + .AddConsoleFormatter(); }); ILogger logger = loggerFactory.CreateLogger(); From cff7787a66e822879d5fae0081090fa904678097 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 15:59:29 +0200 Subject: [PATCH 32/66] Turn on monitor job for the XHarness tests --- tests/XHarness/XHarness.Apple.Device.Archived.proj | 1 + tests/XHarness/XHarness.Apple.Simulator.CustomCommands.proj | 1 + tests/XHarness/XHarness.Apple.Simulator.Run.proj | 1 + tests/XHarness/XHarness.Apple.Simulator.Test.proj | 1 + tests/XHarness/XHarness.TestApks.proj | 1 + 5 files changed, 5 insertions(+) diff --git a/tests/XHarness/XHarness.Apple.Device.Archived.proj b/tests/XHarness/XHarness.Apple.Device.Archived.proj index afd4ee2f07e..38a7cf9558e 100644 --- a/tests/XHarness/XHarness.Apple.Device.Archived.proj +++ b/tests/XHarness/XHarness.Apple.Device.Archived.proj @@ -3,6 +3,7 @@ + true System.Buffers.Tests.app https://netcorenativeassets.blob.core.windows.net/resource-packages/external/ios/test-app/tvos-device/zipped-apps.zip $(ArtifactsTmpDir)XHarness.Apple.Device.Archived diff --git a/tests/XHarness/XHarness.Apple.Simulator.CustomCommands.proj b/tests/XHarness/XHarness.Apple.Simulator.CustomCommands.proj index b3687fdb37a..fe89d4ad248 100644 --- a/tests/XHarness/XHarness.Apple.Simulator.CustomCommands.proj +++ b/tests/XHarness/XHarness.Apple.Simulator.CustomCommands.proj @@ -3,6 +3,7 @@ + true System.Numerics.Vectors.Tests https://netcorenativeassets.blob.core.windows.net/resource-packages/external/ios/test-app/ios-simulator-64/$(XHarnessAppBundleName).app.zip diff --git a/tests/XHarness/XHarness.Apple.Simulator.Run.proj b/tests/XHarness/XHarness.Apple.Simulator.Run.proj index ebccec905b0..2c8867f6378 100644 --- a/tests/XHarness/XHarness.Apple.Simulator.Run.proj +++ b/tests/XHarness/XHarness.Apple.Simulator.Run.proj @@ -3,6 +3,7 @@ + true iOS.Simulator.PInvoke.Test.app https://netcorenativeassets.blob.core.windows.net/resource-packages/external/ios/test-app/ios-simulator-64/$(XHarnessRunAppBundleName).zip diff --git a/tests/XHarness/XHarness.Apple.Simulator.Test.proj b/tests/XHarness/XHarness.Apple.Simulator.Test.proj index c09e80640a4..51f7711791e 100644 --- a/tests/XHarness/XHarness.Apple.Simulator.Test.proj +++ b/tests/XHarness/XHarness.Apple.Simulator.Test.proj @@ -3,6 +3,7 @@ + true Microsoft.Extensions.Configuration.CommandLine.Tests.app https://netcorenativeassets.blob.core.windows.net/resource-packages/external/ios/test-app/ios-simulator-64/$(XHarnessTestAppBundleName).zip diff --git a/tests/XHarness/XHarness.TestApks.proj b/tests/XHarness/XHarness.TestApks.proj index 5ee82fe36fa..f60ef687288 100644 --- a/tests/XHarness/XHarness.TestApks.proj +++ b/tests/XHarness/XHarness.TestApks.proj @@ -3,6 +3,7 @@ + true https://netcorenativeassets.blob.core.windows.net/resource-packages/external/android/test-apk/x86/System.Buffers.Tests-x86.apk https://netcorenativeassets.blob.core.windows.net/resource-packages/external/android/test-apk/x86_64/System.Buffers.Tests-x64.apk https://netcorenativeassets.blob.core.windows.net/resource-packages/external/android/test-apk/arm64_v8a/System.Buffers.Tests-arm64-v8a.apk From 5eebcd54a266cce2faf002f8e9f361b7608db3ac Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 15:59:48 +0200 Subject: [PATCH 33/66] Simplify job filtering a bit --- .../JobMonitor/JobMonitorRunner.cs | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 770dad9f9ef..db093fa69d4 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -69,27 +69,21 @@ public async Task RunAsync() async () => await _helixApi.Job.ListAsync(source: $"pr/public/{_options.Organization}/{_options.RepositoryName}/refs/pull/{_options.PrNumber}/merge"), cancellationToken); - // Filter jobs belonging to this build only - jobs = [..jobs.Where(j => ((JObject)j.Properties).TryGetValue("BuildId", out JToken buildId) && buildId?.ToString() == _options.BuildId)]; - + // Filter jobs to completed ones belonging to this build IReadOnlyCollection completedJobs = [ ..jobs + .Where(j => ((JObject)j.Properties).TryGetValue("BuildId", out JToken buildId) && buildId?.ToString() == _options.BuildId) .Where(j => j.Finished != null) .OrderBy(j => j.Name, StringComparer.OrdinalIgnoreCase) ]; - _logger.LogInformation("{CompletedCount}/{TotalCount} Helix jobs complete", completedJobs.Count, jobs.Count); + _logger.LogInformation("{CompletedCount}/{TotalCount} Helix jobs finished", completedJobs.Count, jobs.Count); - foreach (JobSummary completedJob in completedJobs) + foreach (JobSummary job in completedJobs.Where(j => !processedRuns.Contains(j.Name))) { - if (processedRuns.Contains(completedJob.Name)) - { - continue; - } - - bool passed = await ProcessCompletedJobAsync(completedJob, cancellationToken); - processedRuns.Add(completedJob.Name); + bool passed = await ProcessCompletedJobAsync(job, cancellationToken); + processedRuns.Add(job.Name); processedHelixJobCount++; if (!passed) { From 6706f28b6d35d7a57676fec78c8cabf3cfecbf26 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 16:12:18 +0200 Subject: [PATCH 34/66] Clone the repo always --- eng/common/core-templates/job/helix-job-monitor.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 43079311f15..95380f0a52e 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -126,13 +126,8 @@ jobs: name: $(DncEngInternalBuildPool) image: build.azurelinux.3.amd64.open steps: - - ${{ if ne(parameters.toolNupkgArtifactName, '') }}: - - checkout: none - - ${{ else }}: - # The default code path restores the tool from the repo's .config/dotnet-tools.json, - # so the repository content (including the manifest and global.json) must be available. - - checkout: self - fetchDepth: 1 + - checkout: self + fetchDepth: 1 - ${{ if ne(parameters.toolNupkgArtifactName, '') }}: - task: DownloadPipelineArtifact@2 From 0ca53c9d5e9fcbab1828f56daa4c622c71c67d2a Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 16:26:14 +0200 Subject: [PATCH 35/66] Store processed jobs in test run tags --- .../JobMonitor/HelixJobMonitorUtilities.cs | 3 - .../JobMonitor/JobMonitorRunner.cs | 83 ++++++++++++++++--- .../HelixJobMonitorUtilitiesTests.cs | 8 -- 3 files changed, 71 insertions(+), 23 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs b/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs index 86e9b18bbea..f380f43d94d 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs @@ -39,9 +39,6 @@ public static bool HasFailedNonMonitorJobs(IEnumerable $"Helix Job Monitor - {helixJobName}"; - private static IEnumerable GetRelevantJobRecords(IEnumerable records, string jobMonitorName) { return (records ?? []) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index db093fa69d4..ae5bd5a07f3 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -47,9 +47,15 @@ public JobMonitorRunner(JobMonitorOptions options, ILogger logger) _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-job-monitor"); } + // Tag prefix used to identify Azure DevOps test runs created by this monitor for a + // particular Helix job. The full tag value is "MonitoredJob:{helixJobName}" and is + // attached to the test run when it is created. This lets us look up which Helix jobs + // we have already processed without encoding the Helix job name into the run name. + private const string MonitoredJobTagPrefix = "MonitoredJob:"; + public async Task RunAsync() { - HashSet processedRuns = await GetProcessedRunNamesAsync(); + HashSet processedHelixJobs = await GetProcessedHelixJobNamesAsync(); var cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(_options.MaximumWaitMinutes)); @@ -80,10 +86,10 @@ public async Task RunAsync() _logger.LogInformation("{CompletedCount}/{TotalCount} Helix jobs finished", completedJobs.Count, jobs.Count); - foreach (JobSummary job in completedJobs.Where(j => !processedRuns.Contains(j.Name))) + foreach (JobSummary job in completedJobs.Where(j => !processedHelixJobs.Contains(j.Name))) { bool passed = await ProcessCompletedJobAsync(job, cancellationToken); - processedRuns.Add(job.Name); + processedHelixJobs.Add(job.Name); processedHelixJobCount++; if (!passed) { @@ -129,8 +135,8 @@ private async Task ProcessCompletedJobAsync(JobSummary helixJob, Cancellat { _logger.LogInformation("Processing completed job {jobName}...", helixJob.Name); - string testRunName = HelixJobMonitorUtilities.GetTestRunName(helixJob.Name); - int testRunId = await StartTestRunAsync(testRunName); + string testRunName = GetTestRunNameFromJob(helixJob); + int testRunId = await StartTestRunAsync(testRunName, helixJob.Name); string resultsDirectory = Path.Combine(_options.WorkingDirectory, SanitizeDirName(helixJob.Name)); Directory.CreateDirectory(resultsDirectory); @@ -163,7 +169,7 @@ private async Task ProcessCompletedJobAsync(JobSummary helixJob, Cancellat return failedWorkItemCount == 0; } - private async Task> GetProcessedRunNamesAsync() + private async Task> GetProcessedHelixJobNamesAsync() { // The Azure DevOps "Test Runs - List" API filters by build using the VSTFS // artifact URI (buildUri), not a numeric buildIds parameter. Passing buildIds @@ -174,26 +180,75 @@ private async Task> GetProcessedRunNamesAsync() foreach (JObject run in (data?["value"] as JArray ?? []).Cast()) { - string name = run.Value("name"); + int? runId = run.Value("id"); string state = run.Value("state"); - if (!string.IsNullOrEmpty(name) - && name.StartsWith("Helix Job Monitor - ", StringComparison.OrdinalIgnoreCase) - && string.Equals(state, "Completed", StringComparison.OrdinalIgnoreCase)) + if (runId == null || !string.Equals(state, "Completed", StringComparison.OrdinalIgnoreCase)) { - processed.Add(name); + continue; + } + + string helixJobName = await GetMonitoredHelixJobNameAsync(runId.Value); + if (!string.IsNullOrEmpty(helixJobName)) + { + processed.Add(helixJobName); } } return processed; } + private async Task GetMonitoredHelixJobNameAsync(int testRunId) + { + // The list test-runs API does not return tags, so fetch the run details for each one + // we encounter to inspect its tags for the MonitoredJob marker. + JObject run = await SendAsync( + HttpMethod.Get, + $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=7.1"); + + if (run?["tags"] is not JArray tags) + { + return null; + } + + foreach (JToken tag in tags) + { + string tagName = tag?.Value("name"); + if (!string.IsNullOrEmpty(tagName) + && tagName.StartsWith(MonitoredJobTagPrefix, StringComparison.OrdinalIgnoreCase)) + { + return tagName.Substring(MonitoredJobTagPrefix.Length); + } + } + + return null; + } + + private static string GetTestRunNameFromJob(JobSummary helixJob) + { + // The Helix SDK stamps the desired Azure DevOps test run name onto the job as a + // "TestRunName" property when submitting (matching what StartAzurePipelinesTestRun + // would have used). Fall back to the Helix job name if the property is missing so + // we always produce a non-empty name. + if (helixJob.Properties is JObject properties + && properties.TryGetValue("TestRunName", out JToken testRunName)) + { + string value = testRunName?.ToString(); + if (!string.IsNullOrEmpty(value)) + { + return value; + } + } + + return helixJob.Name; + } + private async Task GetTimelineRecordsAsync() { JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/build/builds/{_options.BuildId}/timeline?api-version=7.1-preview.2"); return data?["records"]?.ToObject() ?? []; } - private async Task StartTestRunAsync(string testRunName) + private async Task StartTestRunAsync(string testRunName, string helixJobName) { JObject result = await SendAsync( HttpMethod.Post, @@ -204,6 +259,10 @@ private async Task StartTestRunAsync(string testRunName) ["build"] = new JObject { ["id"] = _options.BuildId }, ["name"] = testRunName, ["state"] = "InProgress", + ["tags"] = new JArray + { + new JObject { ["name"] = MonitoredJobTagPrefix + helixJobName }, + }, }); return result?["id"]?.ToObject() ?? 0; diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixJobMonitorUtilitiesTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixJobMonitorUtilitiesTests.cs index b59c7e34fe8..6169f4ff65a 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixJobMonitorUtilitiesTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixJobMonitorUtilitiesTests.cs @@ -31,13 +31,5 @@ public void HasFailedNonMonitorJobs_DetectsFailures() Assert.True(HelixJobMonitorUtilities.HasFailedNonMonitorJobs(records, "Helix Job Monitor")); } - - [Fact] - public void GetTestRunName_ProducesStableName() - { - Assert.Equal( - "Helix Job Monitor - coreclr-tests-linux-x64", - HelixJobMonitorUtilities.GetTestRunName("coreclr-tests-linux-x64")); - } } } From 418b0340edf99326c999c9ea629a1b214550d27c Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 16:45:58 +0200 Subject: [PATCH 36/66] Fix the pool --- eng/common/core-templates/job/helix-job-monitor.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 95380f0a52e..e237c4f2e8d 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -121,10 +121,10 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) - image: build.azurelinux.3.amd64.open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + demands: ImageOverride -equals build.azurelinux.3.amd64.open + ${{ else }}: name: $(DncEngInternalBuildPool) - image: build.azurelinux.3.amd64.open + demands: ImageOverride -equals build.azurelinux.3.amd64 steps: - checkout: self fetchDepth: 1 From 660fa913805048475da54212994aeb9b692a88cb Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 16:46:24 +0200 Subject: [PATCH 37/66] Run the HJM right away --- azure-pipelines-pr.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index 8f9212476d7..466069a7dde 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -364,8 +364,7 @@ stages: - template: /eng/common/core-templates/stages/helix-job-monitor.yml parameters: - dependsOn: - - build + dependsOn: [] # Install the Helix job monitor from the nupkg produced by the Build stage's # Windows_NT Release job, instead of pulling it from a published feed. toolNupkgArtifactName: Artifacts_Windows_NT_Release From b73d532805223ab81f32f3b719f3681ff2476336 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 16:52:12 +0200 Subject: [PATCH 38/66] Allow not depending on anything --- eng/common/core-templates/stages/helix-job-monitor.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/eng/common/core-templates/stages/helix-job-monitor.yml b/eng/common/core-templates/stages/helix-job-monitor.yml index 5dbb393dd63..25f0004ac70 100644 --- a/eng/common/core-templates/stages/helix-job-monitor.yml +++ b/eng/common/core-templates/stages/helix-job-monitor.yml @@ -105,8 +105,7 @@ parameters: stages: - stage: ${{ parameters.stageName }} displayName: ${{ parameters.displayName }} - ${{ if ne(length(parameters.dependsOn), 0) }}: - dependsOn: ${{ parameters.dependsOn }} + dependsOn: ${{ parameters.dependsOn }} ${{ if ne(parameters.condition, '') }}: condition: ${{ parameters.condition }} jobs: From acfecb949998cf2f206d611a0d6281eeb4f2fbf0 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 16:56:07 +0200 Subject: [PATCH 39/66] Depend on `build` --- azure-pipelines-pr.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index 466069a7dde..8f9212476d7 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -364,7 +364,8 @@ stages: - template: /eng/common/core-templates/stages/helix-job-monitor.yml parameters: - dependsOn: [] + dependsOn: + - build # Install the Helix job monitor from the nupkg produced by the Build stage's # Windows_NT Release job, instead of pulling it from a published feed. toolNupkgArtifactName: Artifacts_Windows_NT_Release From 3f2caa9c0fd748018c8882eae130f4702e62473e Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 17:06:26 +0200 Subject: [PATCH 40/66] Set TestRunName for monitored jobs --- .../Microsoft.DotNet.Helix.Sdk.MonoQueue.targets | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets index d988e488a17..948a777624b 100644 --- a/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets +++ b/src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.MonoQueue.targets @@ -52,6 +52,18 @@ + + + + <_HelixJobMonitorTestRunName>$(TestRunNamePrefix)$(HelixTargetQueue)$(TestRunNameSuffix) + + + + From e4d3557a28cf5b1dec7f2e9ad0da472193217bae Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 17:20:41 +0200 Subject: [PATCH 41/66] cd --- eng/common/core-templates/job/helix-job-monitor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index e237c4f2e8d..858607259af 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -174,6 +174,7 @@ jobs: echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'." toolSource="$nupkgDir" + pushd "$(Build.SourcesDirectory)" > /dev/null ./eng/common/dotnet.sh tool install \ --tool-path "$toolPath" "$packageId" \ --version "$toolVersion" \ From 3353ac29c7041a6d04bebf82e1964b3ea8d544ce Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 17:36:44 +0200 Subject: [PATCH 42/66] --source --- eng/common/core-templates/job/helix-job-monitor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 858607259af..4759eb61566 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -178,7 +178,7 @@ jobs: ./eng/common/dotnet.sh tool install \ --tool-path "$toolPath" "$packageId" \ --version "$toolVersion" \ - --add-source "$toolSource" \ + --source "$toolSource" \ --ignore-failed-sources echo "##vso[task.prependpath]$toolPath" From 904d4400ff5dfea7f7e31346bea6e7ce083ce820 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 17:58:46 +0200 Subject: [PATCH 43/66] Set DOTNET_ROOT --- eng/common/core-templates/job/helix-job-monitor.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 4759eb61566..51b136f49f8 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -227,6 +227,7 @@ jobs: if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then # Tool was installed into a tool-path that has been prepended to PATH. + export DOTNET_ROOT="$(Build.SourcesDirectory)/.dotnet" '${{ parameters.toolCommand }}' "${toolArgs[@]}" else # Tool was restored from the local .config/dotnet-tools.json manifest; invoke it From 3ebf50be36e60eb91209ecafaeb4e73a409c8390 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 18:01:26 +0200 Subject: [PATCH 44/66] Properly turn on the feature for XHarness tests --- tests/XHarness.Tests.Common.props | 1 + tests/XHarness/XHarness.Apple.Device.Archived.proj | 1 - tests/XHarness/XHarness.Apple.Simulator.CustomCommands.proj | 1 - tests/XHarness/XHarness.Apple.Simulator.Run.proj | 1 - tests/XHarness/XHarness.Apple.Simulator.Test.proj | 1 - tests/XHarness/XHarness.TestApks.proj | 1 - 6 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/XHarness.Tests.Common.props b/tests/XHarness.Tests.Common.props index 4e36863a3c4..515bc50bb44 100644 --- a/tests/XHarness.Tests.Common.props +++ b/tests/XHarness.Tests.Common.props @@ -18,6 +18,7 @@ true true https://helix.dot.net + true diff --git a/tests/XHarness/XHarness.Apple.Device.Archived.proj b/tests/XHarness/XHarness.Apple.Device.Archived.proj index 38a7cf9558e..afd4ee2f07e 100644 --- a/tests/XHarness/XHarness.Apple.Device.Archived.proj +++ b/tests/XHarness/XHarness.Apple.Device.Archived.proj @@ -3,7 +3,6 @@ - true System.Buffers.Tests.app https://netcorenativeassets.blob.core.windows.net/resource-packages/external/ios/test-app/tvos-device/zipped-apps.zip $(ArtifactsTmpDir)XHarness.Apple.Device.Archived diff --git a/tests/XHarness/XHarness.Apple.Simulator.CustomCommands.proj b/tests/XHarness/XHarness.Apple.Simulator.CustomCommands.proj index fe89d4ad248..b3687fdb37a 100644 --- a/tests/XHarness/XHarness.Apple.Simulator.CustomCommands.proj +++ b/tests/XHarness/XHarness.Apple.Simulator.CustomCommands.proj @@ -3,7 +3,6 @@ - true System.Numerics.Vectors.Tests https://netcorenativeassets.blob.core.windows.net/resource-packages/external/ios/test-app/ios-simulator-64/$(XHarnessAppBundleName).app.zip diff --git a/tests/XHarness/XHarness.Apple.Simulator.Run.proj b/tests/XHarness/XHarness.Apple.Simulator.Run.proj index 2c8867f6378..ebccec905b0 100644 --- a/tests/XHarness/XHarness.Apple.Simulator.Run.proj +++ b/tests/XHarness/XHarness.Apple.Simulator.Run.proj @@ -3,7 +3,6 @@ - true iOS.Simulator.PInvoke.Test.app https://netcorenativeassets.blob.core.windows.net/resource-packages/external/ios/test-app/ios-simulator-64/$(XHarnessRunAppBundleName).zip diff --git a/tests/XHarness/XHarness.Apple.Simulator.Test.proj b/tests/XHarness/XHarness.Apple.Simulator.Test.proj index 51f7711791e..c09e80640a4 100644 --- a/tests/XHarness/XHarness.Apple.Simulator.Test.proj +++ b/tests/XHarness/XHarness.Apple.Simulator.Test.proj @@ -3,7 +3,6 @@ - true Microsoft.Extensions.Configuration.CommandLine.Tests.app https://netcorenativeassets.blob.core.windows.net/resource-packages/external/ios/test-app/ios-simulator-64/$(XHarnessTestAppBundleName).zip diff --git a/tests/XHarness/XHarness.TestApks.proj b/tests/XHarness/XHarness.TestApks.proj index f60ef687288..5ee82fe36fa 100644 --- a/tests/XHarness/XHarness.TestApks.proj +++ b/tests/XHarness/XHarness.TestApks.proj @@ -3,7 +3,6 @@ - true https://netcorenativeassets.blob.core.windows.net/resource-packages/external/android/test-apk/x86/System.Buffers.Tests-x86.apk https://netcorenativeassets.blob.core.windows.net/resource-packages/external/android/test-apk/x86_64/System.Buffers.Tests-x64.apk https://netcorenativeassets.blob.core.windows.net/resource-packages/external/android/test-apk/arm64_v8a/System.Buffers.Tests-arm64-v8a.apk From 0097650d0ccd26014b3d0838bc02c91140109d63 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 18:02:52 +0200 Subject: [PATCH 45/66] Only alphanum tags --- src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index ae5bd5a07f3..7096bcb3043 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -51,7 +51,7 @@ public JobMonitorRunner(JobMonitorOptions options, ILogger logger) // particular Helix job. The full tag value is "MonitoredJob:{helixJobName}" and is // attached to the test run when it is created. This lets us look up which Helix jobs // we have already processed without encoding the Helix job name into the run name. - private const string MonitoredJobTagPrefix = "MonitoredJob:"; + private const string MonitoredJobTagPrefix = "MonitoredJobs"; public async Task RunAsync() { From 4d57b6e23aca2a4bcbe47178a4dd8d4a47411603 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Fri, 24 Apr 2026 18:09:28 +0200 Subject: [PATCH 46/66] Improve the default test run name --- .../JobMonitor/JobMonitorRunner.cs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 7096bcb3043..3850abc7d21 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -229,14 +229,20 @@ private static string GetTestRunNameFromJob(JobSummary helixJob) // "TestRunName" property when submitting (matching what StartAzurePipelinesTestRun // would have used). Fall back to the Helix job name if the property is missing so // we always produce a non-empty name. - if (helixJob.Properties is JObject properties - && properties.TryGetValue("TestRunName", out JToken testRunName)) + if (helixJob.Properties is JObject properties) { - string value = testRunName?.ToString(); - if (!string.IsNullOrEmpty(value)) + if (properties.TryGetValue("TestRunName", out JToken testRunName)) { - return value; + string value = testRunName?.ToString(); + if (!string.IsNullOrEmpty(value)) + { + return value; + } } + + properties.TryGetValue("System.PhaseName", out JToken phaseName); + properties.TryGetValue("System.JobName", out JToken jobName); + return $"{phaseName} {jobName} run on {helixJob.QueueId}".Trim(); } return helixJob.Name; From f91463130842c67009ad95f91c053d67603741e2 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 24 Apr 2026 13:02:27 -0700 Subject: [PATCH 47/66] Add DI interfaces, fakes, and scenario tests for JobMonitorRunner Refactor JobMonitorRunner to accept IAzureDevOpsService and IHelixService via constructor injection for testability. Extract real service implementations as inner classes preserving all HTTP/Helix logic. - Add IAzureDevOpsService, IHelixService, IJobMonitorRunner interfaces - Add HelixJobInfo and HelixJobPassFail model types - Add FakeAzureDevOpsService and FakeHelixService for testing - Add 20 scenario tests covering happy path, failures, retries, edge cases - Add early exit when all pipeline jobs fail while Helix still running - Fix infra failure detection (job status 'failed' with zero work items) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...tNet.Helix.AzureDevOpsTestPublisher.csproj | 4 + .../Interfaces/IAzureDevOpsService.cs | 45 ++ .../JobMonitor/Interfaces/IHelixService.cs | 35 ++ .../Interfaces/IJobMonitorRunner.cs | 19 + .../JobMonitor/JobMonitorRunner.cs | 586 ++++++++++-------- .../Microsoft.DotNet.Helix.JobMonitor.csproj | 4 + .../JobMonitor/Models/HelixJobInfo.cs | 34 + .../JobMonitor/Models/HelixJobPassFail.cs | 27 + .../Fakes/FakeAzureDevOpsService.cs | 116 ++++ .../Fakes/FakeHelixService.cs | 93 +++ .../JobMonitorRunnerTests.cs | 388 ++++++++++++ .../NonParallelTestCollection.cs | 10 + .../ScenarioHelpers/ScenarioHelpers.cs | 25 + 13 files changed, 1132 insertions(+), 254 deletions(-) create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IJobMonitorRunner.cs create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobPassFail.cs create mode 100644 src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs create mode 100644 src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs create mode 100644 src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs create mode 100644 src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/NonParallelTestCollection.cs create mode 100644 src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj index 13de144da67..50f940b40a3 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj @@ -6,4 +6,8 @@ enable + + + + diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs new file mode 100644 index 00000000000..affcf0ea14a --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IAzureDevOpsService.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Helix.JobMonitor +{ + /// + /// Abstracts Azure DevOps REST API interactions needed by the job monitor. + /// + public interface IAzureDevOpsService + { + /// + /// Returns the build timeline records for the current build. + /// Used to determine whether non-monitor pipeline jobs have completed. + /// + Task> GetTimelineRecordsAsync(CancellationToken cancellationToken); + + /// + /// Returns the set of Helix job names that have already been processed + /// by a prior monitor invocation (identified via completed AzDO test run tags). + /// + Task> GetProcessedHelixJobNamesAsync(CancellationToken cancellationToken); + + /// + /// Creates a new test run in Azure DevOps and returns its ID. + /// If a test run with this name already exists in-progress (orphaned from a prior crash), + /// the implementation may reuse it. + /// + Task CreateTestRunAsync(string name, string helixJobName, CancellationToken cancellationToken); + + /// + /// Marks a test run as completed. + /// + Task CompleteTestRunAsync(int testRunId, CancellationToken cancellationToken); + + /// + /// Uploads test results for the specified work items into an existing test run. + /// Returns true if all test results passed, false otherwise. + /// + Task UploadTestResultsAsync(int testRunId, IReadOnlyList results, CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs new file mode 100644 index 00000000000..f1930c17110 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IHelixService.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.Helix.JobMonitor.Models; + +namespace Microsoft.DotNet.Helix.JobMonitor +{ + /// + /// Abstracts Helix API interactions needed by the job monitor. + /// + public interface IHelixService + { + /// + /// Returns Helix jobs associated with the current build/stage. + /// + Task> GetJobsAsync(CancellationToken cancellationToken); + + /// + /// Returns work item details for a completed Helix job, including exit codes. + /// + Task GetJobPassFailAsync(string jobName, CancellationToken cancellationToken); + + /// + /// Downloads test result files for a completed Helix job's work items + /// and returns metadata about each work item's results. + /// + Task> DownloadTestResultsAsync( + string jobName, + HelixJobPassFail passFail, + CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IJobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IJobMonitorRunner.cs new file mode 100644 index 00000000000..56af49a631e --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Interfaces/IJobMonitorRunner.cs @@ -0,0 +1,19 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Helix.JobMonitor +{ + /// + /// Contract for the job monitor's main execution loop. + /// + public interface IJobMonitorRunner + { + /// + /// Runs the monitor loop. Returns 0 for success, 1 for failure. + /// + Task RunAsync(CancellationToken cancellationToken); + } +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index ae5bd5a07f3..f12ac0e0b2d 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -18,48 +18,66 @@ using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Microsoft.DotNet.Helix.Client; using Microsoft.DotNet.Helix.Client.Models; +using Microsoft.DotNet.Helix.JobMonitor.Models; using Microsoft.Extensions.Logging; using Newtonsoft.Json; using Newtonsoft.Json.Linq; namespace Microsoft.DotNet.Helix.JobMonitor { - internal sealed class JobMonitorRunner : IDisposable + internal sealed class JobMonitorRunner : IJobMonitorRunner, IDisposable { private readonly JobMonitorOptions _options; private readonly ILogger _logger; - private readonly HttpClient _azdoClient; - private readonly IHelixApi _helixApi; + private readonly IAzureDevOpsService _azdo; + private readonly IHelixService _helix; + private readonly Func _delayFunc; + /// + /// Constructor for production use with real services. + /// public JobMonitorRunner(JobMonitorOptions options, ILogger logger) + : this(options, logger, + CreateRealAzureDevOpsService(options, logger), + CreateRealHelixService(options, logger), + null) + { + } + + /// + /// Constructor for testing with injected services. + /// + internal JobMonitorRunner( + JobMonitorOptions options, + ILogger logger, + IAzureDevOpsService azdo, + IHelixService helix, + Func delayFunc) { _options = options ?? throw new ArgumentNullException(nameof(options)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); + _azdo = azdo ?? throw new ArgumentNullException(nameof(azdo)); + _helix = helix ?? throw new ArgumentNullException(nameof(helix)); + _delayFunc = delayFunc ?? Task.Delay; Directory.CreateDirectory(_options.WorkingDirectory); - - _helixApi = string.IsNullOrEmpty(_options.HelixAccessToken) - ? ApiFactory.GetAnonymous(_options.HelixBaseUri) - : ApiFactory.GetAuthenticated(_options.HelixBaseUri, _options.HelixAccessToken); - - _azdoClient = new HttpClient(); - string encodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + _options.SystemAccessToken)); - _azdoClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", encodedToken); - _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-job-monitor"); } - // Tag prefix used to identify Azure DevOps test runs created by this monitor for a - // particular Helix job. The full tag value is "MonitoredJob:{helixJobName}" and is - // attached to the test run when it is created. This lets us look up which Helix jobs - // we have already processed without encoding the Helix job name into the run name. - private const string MonitoredJobTagPrefix = "MonitoredJob:"; + public Task RunAsync(CancellationToken cancellationToken) + { + return RunCoreAsync(cancellationToken); + } public async Task RunAsync() { - HashSet processedHelixJobs = await GetProcessedHelixJobNamesAsync(); - var cancellationTokenSource = new CancellationTokenSource(); cancellationTokenSource.CancelAfter(TimeSpan.FromMinutes(_options.MaximumWaitMinutes)); - CancellationToken cancellationToken = cancellationTokenSource.Token; + return await RunCoreAsync(cancellationTokenSource.Token); + } + + private async Task RunCoreAsync(CancellationToken cancellationToken) + { + IReadOnlySet alreadyProcessed = await _azdo.GetProcessedHelixJobNamesAsync(cancellationToken); + var processedHelixJobs = new HashSet(alreadyProcessed, StringComparer.OrdinalIgnoreCase); bool anyNonMonitorJobFailures = false; int failedHelixJobCount = 0; @@ -69,27 +87,19 @@ public async Task RunAsync() { cancellationToken.ThrowIfCancellationRequested(); - AzureDevOpsTimelineRecord[] timelineRecords = await GetTimelineRecordsAsync(); - IImmutableList jobs = await RetryAsync( - // TODO: "pr/public" is hardcoded but could come from the build technically - async () => await _helixApi.Job.ListAsync(source: $"pr/public/{_options.Organization}/{_options.RepositoryName}/refs/pull/{_options.PrNumber}/merge"), - cancellationToken); + IReadOnlyList timelineRecords = await _azdo.GetTimelineRecordsAsync(cancellationToken); + IReadOnlyList jobs = await _helix.GetJobsAsync(cancellationToken); - // Filter jobs to completed ones belonging to this build - IReadOnlyCollection completedJobs = - [ - ..jobs - .Where(j => ((JObject)j.Properties).TryGetValue("BuildId", out JToken buildId) && buildId?.ToString() == _options.BuildId) - .Where(j => j.Finished != null) - .OrderBy(j => j.Name, StringComparer.OrdinalIgnoreCase) - ]; + IReadOnlyCollection completedJobs = jobs + .Where(j => j.IsCompleted) + .OrderBy(j => j.JobName, StringComparer.OrdinalIgnoreCase) + .ToList(); _logger.LogInformation("{CompletedCount}/{TotalCount} Helix jobs finished", completedJobs.Count, jobs.Count); - foreach (JobSummary job in completedJobs.Where(j => !processedHelixJobs.Contains(j.Name))) + foreach (HelixJobInfo job in completedJobs.Where(j => !processedHelixJobs.Contains(j.JobName))) { - bool passed = await ProcessCompletedJobAsync(job, cancellationToken); - processedHelixJobs.Add(job.Name); + bool passed = await ProcessCompletedJobAsync(job, processedHelixJobs, cancellationToken); processedHelixJobCount++; if (!passed) { @@ -99,7 +109,7 @@ public async Task RunAsync() anyNonMonitorJobFailures = HelixJobMonitorUtilities.HasFailedNonMonitorJobs(timelineRecords, _options.JobMonitorName); bool allPipelineJobsComplete = HelixJobMonitorUtilities.AreNonMonitorJobsComplete(timelineRecords, _options.JobMonitorName); - bool allHelixJobsComplete = jobs.Count != 0 && jobs.Count == completedJobs.Count; + bool allHelixJobsComplete = jobs.Count == 0 || jobs.All(j => j.IsCompleted); if (allPipelineJobsComplete && allHelixJobsComplete) { @@ -122,312 +132,380 @@ public async Task RunAsync() return 0; } - await Task.Delay(TimeSpan.FromSeconds(Math.Max(5, _options.PollingIntervalSeconds)), cancellationToken); + // If all pipeline jobs are dead and Helix jobs are still running, + // those jobs are orphaned — no point waiting. + if (allPipelineJobsComplete && anyNonMonitorJobFailures && !allHelixJobsComplete) + { + _logger.LogError("All non-monitor pipeline jobs failed/canceled while Helix jobs are still running. Exiting."); + return 1; + } + + await _delayFunc(TimeSpan.FromSeconds(Math.Max(5, _options.PollingIntervalSeconds)), cancellationToken); } } - public void Dispose() + private async Task ProcessCompletedJobAsync( + HelixJobInfo job, + HashSet processedHelixJobs, + CancellationToken cancellationToken) { - _azdoClient.Dispose(); - } + _logger.LogInformation("Processing completed job {jobName}...", job.JobName); - private async Task ProcessCompletedJobAsync(JobSummary helixJob, CancellationToken cancellationToken) - { - _logger.LogInformation("Processing completed job {jobName}...", helixJob.Name); + string testRunName = job.TestRunName ?? job.JobName; + int testRunId = await _azdo.CreateTestRunAsync(testRunName, job.JobName, cancellationToken); - string testRunName = GetTestRunNameFromJob(helixJob); - int testRunId = await StartTestRunAsync(testRunName, helixJob.Name); - string resultsDirectory = Path.Combine(_options.WorkingDirectory, SanitizeDirName(helixJob.Name)); - Directory.CreateDirectory(resultsDirectory); + try + { + HelixJobPassFail passFail = await _helix.GetJobPassFailAsync(job.JobName, cancellationToken); + IReadOnlyList downloadedResults = await _helix.DownloadTestResultsAsync( + job.JobName, passFail, cancellationToken); - IImmutableList workItems = await RetryAsync(() => _helixApi.WorkItem.ListAsync(helixJob.Name), cancellationToken); + bool allTestsPassed = await _azdo.UploadTestResultsAsync(testRunId, downloadedResults, cancellationToken); + await _azdo.CompleteTestRunAsync(testRunId, cancellationToken); - int failedWorkItemCount = workItems.Count(wi => wi.ExitCode != 0 || !wi.State.Equals("Finished", StringComparison.OrdinalIgnoreCase)); - bool helixJobSuccessful = failedWorkItemCount == 0; - int sucessfulWorkItemCount = workItems.Count - failedWorkItemCount; + processedHelixJobs.Add(job.JobName); - try - { - List downloadedFiles = await DownloadTestResultsAsync(helixJob.Name, workItems, resultsDirectory, cancellationToken); - if (!await UploadDownloadedResultsAsync(downloadedFiles, testRunId, cancellationToken)) - { - sucessfulWorkItemCount--; - failedWorkItemCount++; - helixJobSuccessful = false; - } + _logger.LogInformation("Job '{JobName}' completed ({PassedCount} passed, {FailedCount} failed).", + job.JobName, passFail.PassedWorkItems.Count, passFail.FailedWorkItems.Count); + + return !passFail.HasFailures && allTestsPassed + && !job.Status.Equals("failed", StringComparison.OrdinalIgnoreCase); } catch (Exception ex) { - // TODO: Handle better here - _logger.LogError(ex, "Failed to upload test results for job {JobName} to Azure DevOps. Test run ID was {TestRunId}.", helixJob.Name, testRunId); + _logger.LogError(ex, "Failed to process job '{JobName}'. Test run ID was {TestRunId}.", job.JobName, testRunId); + // Don't add to processedHelixJobs — allows retry to pick it up. return false; } + } - await StopTestRunAsync(testRunId, testRunName); - - _logger.LogInformation("Job '{JobName}' completed ({PassedCount} passed, {FailedCount} failed).", helixJob.Name, sucessfulWorkItemCount, failedWorkItemCount); - return failedWorkItemCount == 0; + public void Dispose() + { + // Real services handle their own cleanup via the azdo/helix service implementations } - private async Task> GetProcessedHelixJobNamesAsync() + // ----------------------------------------------------------------- + // Factory methods for production service implementations + // ----------------------------------------------------------------- + + private static IAzureDevOpsService CreateRealAzureDevOpsService(JobMonitorOptions options, ILogger logger) + => new RealAzureDevOpsService(options, logger); + + private static IHelixService CreateRealHelixService(JobMonitorOptions options, ILogger logger) + => new RealHelixService(options, logger); + + // ----------------------------------------------------------------- + // Real AzDO service implementation (extracted from original runner) + // ----------------------------------------------------------------- + + private sealed class RealAzureDevOpsService : IAzureDevOpsService, IDisposable { - // The Azure DevOps "Test Runs - List" API filters by build using the VSTFS - // artifact URI (buildUri), not a numeric buildIds parameter. Passing buildIds - // results in a 404 from the service. - string buildUri = Uri.EscapeDataString($"vstfs:///Build/Build/{_options.BuildId}"); - JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?buildUri={buildUri}&api-version=7.1"); - var processed = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (JObject run in (data?["value"] as JArray ?? []).Cast()) + private const string MonitoredJobTagPrefix = "MonitoredJob:"; + private readonly JobMonitorOptions _options; + private readonly ILogger _logger; + private readonly HttpClient _azdoClient; + + public RealAzureDevOpsService(JobMonitorOptions options, ILogger logger) { - int? runId = run.Value("id"); - string state = run.Value("state"); - if (runId == null || !string.Equals(state, "Completed", StringComparison.OrdinalIgnoreCase)) - { - continue; - } + _options = options; + _logger = logger; + _azdoClient = new HttpClient(); + string encodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + options.SystemAccessToken)); + _azdoClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", encodedToken); + _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-job-monitor"); + } - string helixJobName = await GetMonitoredHelixJobNameAsync(runId.Value); - if (!string.IsNullOrEmpty(helixJobName)) + public async Task> GetTimelineRecordsAsync(CancellationToken cancellationToken) + { + JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/build/builds/{_options.BuildId}/timeline?api-version=7.1-preview.2", cancellationToken: cancellationToken); + return data?["records"]?.ToObject() ?? []; + } + + public async Task> GetProcessedHelixJobNamesAsync(CancellationToken cancellationToken) + { + string buildUri = Uri.EscapeDataString($"vstfs:///Build/Build/{_options.BuildId}"); + JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?buildUri={buildUri}&api-version=7.1", cancellationToken: cancellationToken); + var processed = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (JObject run in (data?["value"] as JArray ?? []).Cast()) { - processed.Add(helixJobName); + int? runId = run.Value("id"); + string state = run.Value("state"); + if (runId == null || !string.Equals(state, "Completed", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + string helixJobName = await GetMonitoredHelixJobNameAsync(runId.Value, cancellationToken); + if (!string.IsNullOrEmpty(helixJobName)) + { + processed.Add(helixJobName); + } } + + return processed; } - return processed; - } + private async Task GetMonitoredHelixJobNameAsync(int testRunId, CancellationToken cancellationToken) + { + JObject run = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=7.1", cancellationToken: cancellationToken); + if (run?["tags"] is not JArray tags) + { + return null; + } - private async Task GetMonitoredHelixJobNameAsync(int testRunId) - { - // The list test-runs API does not return tags, so fetch the run details for each one - // we encounter to inspect its tags for the MonitoredJob marker. - JObject run = await SendAsync( - HttpMethod.Get, - $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=7.1"); + foreach (JToken tag in tags) + { + string tagName = tag?.Value("name"); + if (!string.IsNullOrEmpty(tagName) && tagName.StartsWith(MonitoredJobTagPrefix, StringComparison.OrdinalIgnoreCase)) + { + return tagName.Substring(MonitoredJobTagPrefix.Length); + } + } - if (run?["tags"] is not JArray tags) - { return null; } - foreach (JToken tag in tags) + public async Task CreateTestRunAsync(string name, string helixJobName, CancellationToken cancellationToken) { - string tagName = tag?.Value("name"); - if (!string.IsNullOrEmpty(tagName) - && tagName.StartsWith(MonitoredJobTagPrefix, StringComparison.OrdinalIgnoreCase)) - { - return tagName.Substring(MonitoredJobTagPrefix.Length); - } + JObject result = await SendAsync(HttpMethod.Post, + $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?api-version=5.0", + new JObject + { + ["automated"] = true, + ["build"] = new JObject { ["id"] = _options.BuildId }, + ["name"] = name, + ["state"] = "InProgress", + ["tags"] = new JArray { new JObject { ["name"] = MonitoredJobTagPrefix + helixJobName } }, + }, + cancellationToken: cancellationToken); + return result?["id"]?.ToObject() ?? 0; } - return null; - } + public async Task CompleteTestRunAsync(int testRunId, CancellationToken cancellationToken) + { + await SendAsync(new HttpMethod("PATCH"), + $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=5.0", + new JObject { ["state"] = "Completed" }, + cancellationToken: cancellationToken); + } - private static string GetTestRunNameFromJob(JobSummary helixJob) - { - // The Helix SDK stamps the desired Azure DevOps test run name onto the job as a - // "TestRunName" property when submitting (matching what StartAzurePipelinesTestRun - // would have used). Fall back to the Helix job name if the property is missing so - // we always produce a non-empty name. - if (helixJob.Properties is JObject properties - && properties.TryGetValue("TestRunName", out JToken testRunName)) + public async Task UploadTestResultsAsync(int testRunId, IReadOnlyList results, CancellationToken cancellationToken) { - string value = testRunName?.ToString(); - if (!string.IsNullOrEmpty(value)) + var publisher = new AzureDevOpsResultPublisher( + new AzureDevOpsReportingParameters( + new Uri(_options.CollectionUri, UriKind.Absolute), + _options.TeamProject, + testRunId.ToString(CultureInfo.InvariantCulture), + _options.SystemAccessToken), + _logger); + + bool allPassed = true; + foreach (WorkItemTestResults workItem in results) { - return value; + _logger.LogInformation("Publishing test results for work item '{WorkItemName}' in job '{JobName}'...", workItem.WorkItemName, workItem.JobName); + allPassed &= await publisher.UploadTestResultsAsync(workItem.TestResultFiles, + new { HelixJobId = workItem.JobName, HelixWorkItemName = workItem.WorkItemName }, + cancellationToken); } - } - - return helixJob.Name; - } - private async Task GetTimelineRecordsAsync() - { - JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/build/builds/{_options.BuildId}/timeline?api-version=7.1-preview.2"); - return data?["records"]?.ToObject() ?? []; - } + return allPassed; + } - private async Task StartTestRunAsync(string testRunName, string helixJobName) - { - JObject result = await SendAsync( - HttpMethod.Post, - $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?api-version=5.0", - new JObject + private async Task SendAsync(HttpMethod method, string requestUri, JToken body = null, CancellationToken cancellationToken = default) + { + return await RetryAsync(async () => { - ["automated"] = true, - ["build"] = new JObject { ["id"] = _options.BuildId }, - ["name"] = testRunName, - ["state"] = "InProgress", - ["tags"] = new JArray + using var request = new HttpRequestMessage(method, requestUri); + if (body != null) { - new JObject { ["name"] = MonitoredJobTagPrefix + helixJobName }, - }, - }); + request.Content = new StringContent(body.ToString(Formatting.None), Encoding.UTF8, "application/json"); + } - return result?["id"]?.ToObject() ?? 0; - } + using HttpResponseMessage response = await _azdoClient.SendAsync(request, cancellationToken); + string content = response.Content != null ? await response.Content.ReadAsStringAsync(cancellationToken) : null; + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException($"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}"); + } - private async Task StopTestRunAsync(int testRunId, string testRunName) - { - await SendAsync( - new HttpMethod("PATCH"), - $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=5.0", - new JObject { ["state"] = "Completed" }); + return string.IsNullOrWhiteSpace(content) ? [] : JObject.Parse(content); + }, cancellationToken); + } - _logger.LogInformation("Stopped test run '{TestRunName}'.", testRunName); + public void Dispose() => _azdoClient.Dispose(); } - private async Task> DownloadTestResultsAsync( - string jobName, - IImmutableList workItems, - string outputDirectory, - CancellationToken cancellationToken) + // ----------------------------------------------------------------- + // Real Helix service implementation (extracted from original runner) + // ----------------------------------------------------------------- + + private sealed class RealHelixService : IHelixService { - List downloadedFiles = []; + private readonly JobMonitorOptions _options; + private readonly ILogger _logger; + private readonly IHelixApi _helixApi; - JobResultsUri resultsUri = await RetryAsync( - () => _helixApi.Job.ResultsAsync(jobName), - cancellationToken); + public RealHelixService(JobMonitorOptions options, ILogger logger) + { + _options = options; + _logger = logger; + _helixApi = string.IsNullOrEmpty(options.HelixAccessToken) + ? ApiFactory.GetAnonymous(options.HelixBaseUri) + : ApiFactory.GetAuthenticated(options.HelixBaseUri, options.HelixAccessToken); + } - foreach (WorkItemSummary workItem in workItems) + public async Task> GetJobsAsync(CancellationToken cancellationToken) { - IImmutableList availableFiles = await RetryAsync( - () => _helixApi.WorkItem.ListFilesAsync(workItem.Name, jobName, false), + IImmutableList jobs = await RetryAsync( + async () => await _helixApi.Job.ListAsync( + source: $"pr/public/{_options.Organization}/{_options.RepositoryName}/refs/pull/{_options.PrNumber}/merge"), cancellationToken); - availableFiles = [.. availableFiles.Where(f => LooksLikeTestResultFile(f.Name))]; + return jobs + .Where(j => ((JObject)j.Properties).TryGetValue("BuildId", out JToken buildId) && buildId?.ToString() == _options.BuildId) + .Select(j => new HelixJobInfo( + j.Name, + j.Finished != null ? "finished" : "running", + GetTestRunNameFromJob(j))) + .ToList(); + } - if (availableFiles.Count == 0) - { - _logger.LogInformation("Work item '{WorkItemName}' in job '{JobName}' has no test result files to download.", workItem.Name, jobName); - continue; - } + public async Task GetJobPassFailAsync(string jobName, CancellationToken cancellationToken) + { + IImmutableList workItems = await RetryAsync( + () => _helixApi.WorkItem.ListAsync(jobName), + cancellationToken); - string workItemDirectory = Path.Combine(outputDirectory, SanitizeDirName(workItem.Name)); - Directory.CreateDirectory(workItemDirectory); + var passed = new List(); + var failed = new List(); - List workItemFiles = []; - foreach (UploadedFile file in availableFiles) + foreach (WorkItemSummary wi in workItems) { - string relativePath = file.Name.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar); - string destinationFile = Path.Combine(workItemDirectory, relativePath); - string directory = Path.GetDirectoryName(destinationFile); - - if (!string.IsNullOrEmpty(directory)) + if (wi.ExitCode != 0 || !wi.State.Equals("Finished", StringComparison.OrdinalIgnoreCase)) { - Directory.CreateDirectory(directory); + failed.Add(wi.Name); } - - try + else { - _logger.LogInformation("Downloading {FileName} for work item {WorkItemName} in job {JobName}...", file.Name, workItem.Name, jobName); - - BlobClient blobClient = CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS); - await blobClient.DownloadToAsync(destinationFile, cancellationToken); - workItemFiles.Add(destinationFile); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItem.Name); + passed.Add(wi.Name); } } - downloadedFiles.Add(new WorkItemTestResults(jobName, workItem.Name, workItemFiles)); + return new HelixJobPassFail(passed, failed); } - return downloadedFiles; - } + public async Task> DownloadTestResultsAsync( + string jobName, + HelixJobPassFail passFail, + CancellationToken cancellationToken) + { + List downloadedFiles = []; + string outputDirectory = Path.Combine(_options.WorkingDirectory, SanitizeDirName(jobName)); + Directory.CreateDirectory(outputDirectory); - private async Task UploadDownloadedResultsAsync(List testResults, int testRunId, CancellationToken cancellationToken) - { - var publisher = new AzureDevOpsResultPublisher( - new AzureDevOpsReportingParameters( - new Uri(_options.CollectionUri, UriKind.Absolute), - _options.TeamProject, - testRunId.ToString(CultureInfo.InvariantCulture), - _options.SystemAccessToken), - _logger); + JobResultsUri resultsUri = await RetryAsync(() => _helixApi.Job.ResultsAsync(jobName), cancellationToken); - bool allTestsPassed = true; + IEnumerable workItemNames = passFail.PassedWorkItems + .Concat(passFail.FailedWorkItems) + .Distinct(StringComparer.OrdinalIgnoreCase); - foreach (WorkItemTestResults workItemTestResult in testResults) - { - _logger.LogInformation("Publishing test results for work item '{WorkItemName}' in job '{JobName}'...", workItemTestResult.WorkItemName, workItemTestResult.JobName); - allTestsPassed &= await publisher.UploadTestResultsAsync( - workItemTestResult.TestResultFiles, - // Metadata that will be appended to each test case - new + foreach (string workItemName in workItemNames) + { + IImmutableList availableFiles = await RetryAsync( + () => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false), + cancellationToken); + + availableFiles = [.. availableFiles.Where(f => LooksLikeTestResultFile(f.Name))]; + if (availableFiles.Count == 0) { - HelixJobId = workItemTestResult.JobName, - HelixWorkItemName = workItemTestResult.WorkItemName, - }, - cancellationToken); - } + continue; + } - return allTestsPassed; - } + string workItemDirectory = Path.Combine(outputDirectory, SanitizeDirName(workItemName)); + Directory.CreateDirectory(workItemDirectory); - private async Task SendAsync(HttpMethod method, string requestUri, JToken body = null, CancellationToken cancellationToken = default) - { - return await RetryAsync(async () => - { - using var request = new HttpRequestMessage(method, requestUri); - if (body != null) - { - request.Content = new StringContent(body.ToString(Formatting.None), Encoding.UTF8, "application/json"); - } + List workItemFiles = []; + foreach (UploadedFile file in availableFiles) + { + string relativePath = file.Name.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar); + string destinationFile = Path.Combine(workItemDirectory, relativePath); + string directory = Path.GetDirectoryName(destinationFile); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } - using HttpResponseMessage response = await _azdoClient.SendAsync(request, cancellationToken); - string content = response.Content != null ? await response.Content.ReadAsStringAsync(cancellationToken) : null; - if (!response.IsSuccessStatusCode) - { - throw new HttpRequestException($"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}"); + try + { + BlobClient blobClient = CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS); + await blobClient.DownloadToAsync(destinationFile, cancellationToken); + workItemFiles.Add(destinationFile); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItemName); + } + } + + downloadedFiles.Add(new WorkItemTestResults(jobName, workItemName, workItemFiles)); } - if (string.IsNullOrWhiteSpace(content)) + return downloadedFiles; + } + + private static string GetTestRunNameFromJob(JobSummary helixJob) + { + if (helixJob.Properties is JObject properties + && properties.TryGetValue("TestRunName", out JToken testRunName)) { - return []; + string value = testRunName?.ToString(); + if (!string.IsNullOrEmpty(value)) + { + return value; + } } - return JObject.Parse(content); - }, cancellationToken); - } - - private static BlobClient CreateBlobClient(string fileLink, string resultsSas) - { - var options = new BlobClientOptions(); - options.Retry.NetworkTimeout = TimeSpan.FromMinutes(5); + return helixJob.Name; + } - if (string.IsNullOrEmpty(resultsSas)) + private static BlobClient CreateBlobClient(string fileLink, string resultsSas) { - return new BlobClient(new Uri(fileLink), options); - } + var options = new BlobClientOptions(); + options.Retry.NetworkTimeout = TimeSpan.FromMinutes(5); + if (string.IsNullOrEmpty(resultsSas)) + { + return new BlobClient(new Uri(fileLink), options); + } - string strippedUri = fileLink.Contains('?') ? fileLink.Substring(0, fileLink.LastIndexOf('?', StringComparison.Ordinal)) : fileLink; - return new BlobClient(new Uri(strippedUri), new AzureSasCredential(resultsSas), options); - } + string strippedUri = fileLink.Contains('?') ? fileLink.Substring(0, fileLink.LastIndexOf('?', StringComparison.Ordinal)) : fileLink; + return new BlobClient(new Uri(strippedUri), new AzureSasCredential(resultsSas), options); + } - private static bool LooksLikeTestResultFile(string path) - => LocalTestResultsReader.LooksLikeTestResultFile(path); + private static bool LooksLikeTestResultFile(string path) + => LocalTestResultsReader.LooksLikeTestResultFile(path); - private static string SanitizeDirName(string value) - { - foreach (char invalidChar in Path.GetInvalidFileNameChars()) + private static string SanitizeDirName(string value) { - value = value.Replace(invalidChar, '-'); - } + foreach (char invalidChar in Path.GetInvalidFileNameChars()) + { + value = value.Replace(invalidChar, '-'); + } - return value; + return value; + } } + // ----------------------------------------------------------------- + // Shared retry helper + // ----------------------------------------------------------------- + private static async Task RetryAsync(Func> action, CancellationToken cancellationToken) { Exception last = null; for (int attempt = 0; attempt < 5; attempt++) { cancellationToken.ThrowIfCancellationRequested(); - try { return await action(); @@ -443,5 +521,5 @@ private static async Task RetryAsync(Func> action, CancellationTok } } - record WorkItemTestResults(string JobName, string WorkItemName, List TestResultFiles); + public record WorkItemTestResults(string JobName, string WorkItemName, List TestResultFiles); } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj index cc3e61645a9..f0dcfde2e6c 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj @@ -9,6 +9,10 @@ Standalone Helix Job Monitor tool for Azure DevOps pipelines + + + + diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs new file mode 100644 index 00000000000..5cabc6399e1 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +namespace Microsoft.DotNet.Helix.JobMonitor.Models +{ + /// + /// Represents a Helix job and its current status. + /// Decoupled from the Helix Client SDK's generated models. + /// + public sealed class HelixJobInfo + { + public HelixJobInfo(string jobName, string status, string testRunName = null) + { + JobName = jobName ?? throw new ArgumentNullException(nameof(jobName)); + Status = status ?? throw new ArgumentNullException(nameof(status)); + TestRunName = testRunName; + } + + public string JobName { get; } + + public string Status { get; } + + /// + /// The desired AzDO test run name for this job. May come from a Helix job property. + /// Falls back to the job name if not set. + /// + public string TestRunName { get; } + + public bool IsCompleted => Status.Equals("finished", StringComparison.OrdinalIgnoreCase) + || Status.Equals("failed", StringComparison.OrdinalIgnoreCase); + } +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobPassFail.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobPassFail.cs new file mode 100644 index 00000000000..59f50ca0d80 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobPassFail.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; + +namespace Microsoft.DotNet.Helix.JobMonitor.Models +{ + /// + /// Pass/fail breakdown for a completed Helix job based on work item exit codes. + /// Decoupled from the Helix Client SDK's generated models. + /// + public sealed class HelixJobPassFail + { + public HelixJobPassFail(IReadOnlyList passedWorkItems, IReadOnlyList failedWorkItems) + { + PassedWorkItems = passedWorkItems ?? Array.Empty(); + FailedWorkItems = failedWorkItems ?? Array.Empty(); + } + + public IReadOnlyList PassedWorkItems { get; } + + public IReadOnlyList FailedWorkItems { get; } + + public bool HasFailures => FailedWorkItems.Count > 0; + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs new file mode 100644 index 00000000000..d3af095ba4b --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs @@ -0,0 +1,116 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.Helix.JobMonitor; + +namespace Microsoft.DotNet.Helix.Sdk.Tests.Fakes +{ + internal sealed class FakeAzureDevOpsService : IAzureDevOpsService + { + private readonly List _timelineSnapshots = []; + private readonly HashSet _previouslyProcessedJobs = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _inProgressRunsByJobName = new(StringComparer.OrdinalIgnoreCase); + private int _currentTimelineIndex; + private int _nextTestRunId; + + // Observable state for test assertions + public List CreatedTestRuns { get; } = []; + public List CompletedTestRunIds { get; } = []; + public Dictionary> UploadedResultsByRunId { get; } = []; + public List UploadedJobNames { get; } = []; + public int CreateTestRunCallCount { get; private set; } + + // Configuration + public FakeAzureDevOpsService AddTimelineSnapshot(AzureDevOpsTimelineRecord[] records) + { + _timelineSnapshots.Add(records); + return this; + } + + public FakeAzureDevOpsService WithPreviouslyProcessedJob(string jobName) + { + _previouslyProcessedJobs.Add(jobName); + return this; + } + + public void AdvanceTimeline() + { + if (_currentTimelineIndex < _timelineSnapshots.Count - 1) + { + _currentTimelineIndex++; + } + } + + // IAzureDevOpsService implementation + public Task> GetTimelineRecordsAsync(CancellationToken cancellationToken) + { + if (_timelineSnapshots.Count == 0) + { + return Task.FromResult>(Array.Empty()); + } + + AzureDevOpsTimelineRecord[] snapshot = _timelineSnapshots[Math.Min(_currentTimelineIndex, _timelineSnapshots.Count - 1)]; + return Task.FromResult>(snapshot); + } + + public Task> GetProcessedHelixJobNamesAsync(CancellationToken cancellationToken) + { + var result = new HashSet(_previouslyProcessedJobs, StringComparer.OrdinalIgnoreCase); + return Task.FromResult>(result); + } + + public Task CreateTestRunAsync(string name, string helixJobName, CancellationToken cancellationToken) + { + CreateTestRunCallCount++; + + // Idempotent: if a run for this helix job is in-progress, reuse it + if (_inProgressRunsByJobName.TryGetValue(helixJobName, out int existingId)) + { + return Task.FromResult(existingId); + } + + int id = Interlocked.Increment(ref _nextTestRunId); + CreatedTestRuns.Add(name); + _inProgressRunsByJobName[helixJobName] = id; + return Task.FromResult(id); + } + + public Task CompleteTestRunAsync(int testRunId, CancellationToken cancellationToken) + { + CompletedTestRunIds.Add(testRunId); + + string keyToRemove = null; + foreach (var kvp in _inProgressRunsByJobName) + { + if (kvp.Value == testRunId) { keyToRemove = kvp.Key; break; } + } + + if (keyToRemove != null) _inProgressRunsByJobName.Remove(keyToRemove); + return Task.CompletedTask; + } + + public Task UploadTestResultsAsync(int testRunId, IReadOnlyList results, CancellationToken cancellationToken) + { + if (!UploadedResultsByRunId.TryGetValue(testRunId, out List existing)) + { + existing = []; + UploadedResultsByRunId[testRunId] = existing; + } + + existing.AddRange(results); + + foreach (string jobName in results.Select(r => r.JobName).Distinct(StringComparer.OrdinalIgnoreCase)) + { + UploadedJobNames.Add(jobName); + _previouslyProcessedJobs.Add(jobName); + } + + return Task.FromResult(true); + } + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs new file mode 100644 index 00000000000..315b838276d --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs @@ -0,0 +1,93 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.Helix.JobMonitor; +using Microsoft.DotNet.Helix.JobMonitor.Models; + +namespace Microsoft.DotNet.Helix.Sdk.Tests.Fakes +{ + internal sealed class FakeHelixService : IHelixService + { + private readonly List _snapshots = []; + private readonly HashSet _downloadFailureJobs = new(StringComparer.OrdinalIgnoreCase); + private int _currentSnapshotIndex; + + public FakeHelixService AddSnapshot( + HelixJobInfo[] jobs, + Dictionary passFailByJob = null, + Dictionary> testResultsByJob = null) + { + _snapshots.Add(new HelixSnapshot( + jobs, + passFailByJob ?? new Dictionary(StringComparer.OrdinalIgnoreCase), + testResultsByJob ?? new Dictionary>(StringComparer.OrdinalIgnoreCase))); + return this; + } + + public FakeHelixService FailDownloadForJob(string jobName) { _downloadFailureJobs.Add(jobName); return this; } + public void ClearDownloadFailures() { _downloadFailureJobs.Clear(); } + + public void AdvanceSnapshot() + { + if (_currentSnapshotIndex < _snapshots.Count - 1) _currentSnapshotIndex++; + } + + private HelixSnapshot CurrentSnapshot => _snapshots[Math.Min(_currentSnapshotIndex, _snapshots.Count - 1)]; + + public Task> GetJobsAsync(CancellationToken cancellationToken) + { + if (_snapshots.Count == 0) + { + return Task.FromResult>(Array.Empty()); + } + + return Task.FromResult>(CurrentSnapshot.Jobs); + } + + public Task GetJobPassFailAsync(string jobName, CancellationToken cancellationToken) + { + if (CurrentSnapshot.PassFailByJob.TryGetValue(jobName, out HelixJobPassFail passFail)) + { + return Task.FromResult(passFail); + } + + throw new InvalidOperationException($"No pass/fail data was configured for Helix job '{jobName}'."); + } + + public Task> DownloadTestResultsAsync( + string jobName, HelixJobPassFail passFail, CancellationToken cancellationToken) + { + if (_downloadFailureJobs.Contains(jobName)) + { + throw new InvalidOperationException($"Injected download failure for Helix job '{jobName}'."); + } + + if (CurrentSnapshot.TestResultsByJob.TryGetValue(jobName, out List explicitResults)) + { + return Task.FromResult>(explicitResults); + } + + List workItemNames = passFail.PassedWorkItems + .Concat(passFail.FailedWorkItems) + .Distinct(StringComparer.OrdinalIgnoreCase) + .DefaultIfEmpty($"{jobName}-synthetic") + .ToList(); + + IReadOnlyList generated = workItemNames + .Select(wi => new WorkItemTestResults(jobName, wi, [])) + .ToList(); + + return Task.FromResult(generated); + } + + private sealed record HelixSnapshot( + HelixJobInfo[] Jobs, + Dictionary PassFailByJob, + Dictionary> TestResultsByJob); + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs new file mode 100644 index 00000000000..aa3676ff937 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs @@ -0,0 +1,388 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.Helix.JobMonitor; +using Microsoft.DotNet.Helix.JobMonitor.Models; +using Microsoft.DotNet.Helix.Sdk.Tests.Fakes; +using Microsoft.DotNet.Helix.Sdk.Tests.ScenarioHelpers; +using Microsoft.Extensions.Logging.Abstractions; +using Xunit; +using static Microsoft.DotNet.Helix.Sdk.Tests.ScenarioHelpers.ScenarioHelpers; + +namespace Microsoft.DotNet.Helix.Sdk.Tests +{ + [Collection("NonParallel")] + public class JobMonitorRunnerTests + { + // ----------------------------------------------------------------------- + // Happy Path + // ----------------------------------------------------------------------- + + [Fact] + public async Task AllJobsPassOnFirstPoll_ExitZero_OneTestRunPerJob() + { + var (azdo, helix, runner, delayCount) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "completed", "succeeded"), MonitorJob()]], + helixSnapshots: [(jobs: [HelixJob("job-linux", "finished")], passFail: Dict(("job-linux", PassFail(passed: ["wi-1"]))))]); + + int exitCode = await runner.RunAsync(CancellationToken.None); + + Assert.Equal(0, exitCode); + Assert.Equal(0, delayCount()); + Assert.Single(azdo.CreatedTestRuns); + Assert.Single(azdo.CompletedTestRunIds); + Assert.Equal(["job-linux"], azdo.UploadedJobNames); + } + + [Fact] + public async Task MultipleJobsAcrossMultiplePolls_ProcessesEachOnce() + { + var (azdo, helix, runner, delayCount) = CreateScenario( + timelineSnapshots: + [ + [PipelineJob("Build Linux", "inProgress"), PipelineJob("Build Win", "inProgress"), MonitorJob()], + [PipelineJob("Build Linux", "completed", "succeeded"), PipelineJob("Build Win", "completed", "succeeded"), MonitorJob()], + [PipelineJob("Build Linux", "completed", "succeeded"), PipelineJob("Build Win", "completed", "succeeded"), MonitorJob()], + ], + helixSnapshots: + [ + (jobs: [HelixJob("job-linux", "running")], passFail: EmptyPassFail()), + (jobs: [HelixJob("job-linux", "finished"), HelixJob("job-win", "running")], passFail: Dict(("job-linux", PassFail(passed: ["linux-wi"])))), + (jobs: [HelixJob("job-linux", "finished"), HelixJob("job-win", "finished")], passFail: Dict(("job-linux", PassFail(passed: ["linux-wi"])), ("job-win", PassFail(passed: ["win-wi"])))), + ]); + + int exitCode = await runner.RunAsync(CancellationToken.None); + + Assert.Equal(0, exitCode); + Assert.Equal(2, delayCount()); + Assert.Equal(2, azdo.CreatedTestRuns.Count); + Assert.Equal(2, azdo.CompletedTestRunIds.Count); + Assert.Equal(["job-linux", "job-win"], azdo.UploadedJobNames); + } + + [Fact] + public async Task StageCompletesWithNoHelixJobs_ExitZero_NoTestRuns() + { + var (azdo, helix, runner, _) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "completed", "succeeded"), MonitorJob()]], + helixSnapshots: [(jobs: Array.Empty(), passFail: EmptyPassFail())]); + + int exitCode = await runner.RunAsync(CancellationToken.None); + + Assert.Equal(0, exitCode); + Assert.Empty(azdo.CreatedTestRuns); + Assert.Empty(azdo.UploadedJobNames); + } + + // ----------------------------------------------------------------------- + // Failure Scenarios + // ----------------------------------------------------------------------- + + [Fact] + public async Task PipelineJobFailsBeforeHelixSubmission_ExitOne_NoTestRuns() + { + var (azdo, _, runner, _) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "completed", "failed"), MonitorJob()]], + helixSnapshots: [(jobs: Array.Empty(), passFail: EmptyPassFail())]); + + int exitCode = await runner.RunAsync(CancellationToken.None); + + Assert.Equal(1, exitCode); + Assert.Empty(azdo.CreatedTestRuns); + } + + [Fact] + public async Task PipelineJobCanceled_ExitOne() + { + var (_, _, runner, _) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "completed", "canceled"), MonitorJob()]], + helixSnapshots: [(jobs: Array.Empty(), passFail: EmptyPassFail())]); + + Assert.Equal(1, await runner.RunAsync(CancellationToken.None)); + } + + [Fact] + public async Task HelixJobFails_ExitOne_ResultsStillUploaded() + { + var (azdo, _, runner, _) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "completed", "succeeded"), MonitorJob()]], + helixSnapshots: [(jobs: [HelixJob("job-linux", "finished")], passFail: Dict(("job-linux", PassFail(failed: ["wi-1"]))))]); + + int exitCode = await runner.RunAsync(CancellationToken.None); + + Assert.Equal(1, exitCode); + Assert.Equal(["job-linux"], azdo.UploadedJobNames); + Assert.Single(azdo.CompletedTestRunIds); + } + + [Fact] + public async Task AllHelixWorkItemsFail_ExitOne_ResultsUploaded() + { + var (azdo, _, runner, _) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "completed", "succeeded"), MonitorJob()]], + helixSnapshots: [(jobs: [HelixJob("job-linux", "finished")], passFail: Dict(("job-linux", PassFail(failed: ["wi-1", "wi-2"]))))]); + + Assert.Equal(1, await runner.RunAsync(CancellationToken.None)); + Assert.Equal(["job-linux"], azdo.UploadedJobNames); + } + + [Fact] + public async Task InfrastructureFailure_HelixJobFailedNoWorkItems_ExitOne() + { + var (azdo, _, runner, _) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "completed", "succeeded"), MonitorJob()]], + helixSnapshots: [(jobs: [HelixJob("job-linux", "failed")], passFail: Dict(("job-linux", PassFail())))]); + + Assert.Equal(1, await runner.RunAsync(CancellationToken.None)); + Assert.Equal(["job-linux"], azdo.UploadedJobNames); + } + + [Fact] + public async Task MultipleHelixJobsAllFail_ExitOne_AllResultsUploaded() + { + var (azdo, _, runner, _) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "completed", "succeeded"), PipelineJob("Build Win", "completed", "succeeded"), MonitorJob()]], + helixSnapshots: [(jobs: [HelixJob("job-linux", "finished"), HelixJob("job-win", "finished")], passFail: Dict(("job-linux", PassFail(failed: ["linux-wi"])), ("job-win", PassFail(failed: ["win-wi"]))))]); + + Assert.Equal(1, await runner.RunAsync(CancellationToken.None)); + Assert.Equal(2, azdo.UploadedJobNames.Count); + } + + [Fact] + public async Task PipelineFailsButHelixResultsStillUploaded() + { + var (azdo, _, runner, _) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "completed", "failed"), MonitorJob()]], + helixSnapshots: [(jobs: [HelixJob("job-linux", "finished")], passFail: Dict(("job-linux", PassFail(passed: ["wi-1"]))))]); + + Assert.Equal(1, await runner.RunAsync(CancellationToken.None)); + Assert.Equal(["job-linux"], azdo.UploadedJobNames); + } + + // ----------------------------------------------------------------------- + // Retry / Rerun Scenarios + // ----------------------------------------------------------------------- + + [Fact] + public async Task MonitorRetry_SkipsPreviouslyProcessed() + { + var azdo = new FakeAzureDevOpsService().WithPreviouslyProcessedJob("job-linux"); + var helix = new FakeHelixService(); + ConfigureSnapshots(azdo, helix, + [[PipelineJob("Build Linux", "completed", "succeeded"), PipelineJob("Build Win", "completed", "succeeded"), MonitorJob()]], + [(jobs: [HelixJob("job-linux", "finished"), HelixJob("job-win", "finished")], passFail: Dict(("job-linux", PassFail(passed: ["linux-wi"])), ("job-win", PassFail(passed: ["win-wi"]))))]); + var runner = CreateRunner(azdo, helix); + + Assert.Equal(0, await runner.RunAsync(CancellationToken.None)); + Assert.Single(azdo.CreatedTestRuns); + Assert.Equal(["job-win"], azdo.UploadedJobNames); + } + + [Fact] + public async Task MonitorRetry_ProcessesReplacementDelta() + { + var azdo = new FakeAzureDevOpsService().WithPreviouslyProcessedJob("job-linux-attempt1"); + var helix = new FakeHelixService(); + ConfigureSnapshots(azdo, helix, + [[PipelineJob("Build Linux (retry)", "completed", "succeeded"), MonitorJob()]], + [(jobs: [HelixJob("job-linux-attempt1", "failed"), HelixJob("job-linux-attempt2", "finished")], passFail: Dict(("job-linux-attempt1", PassFail(failed: ["wi-2"])), ("job-linux-attempt2", PassFail(passed: ["wi-2"]))))]); + var runner = CreateRunner(azdo, helix); + + Assert.Equal(0, await runner.RunAsync(CancellationToken.None)); + Assert.Equal(["job-linux-attempt2"], azdo.UploadedJobNames); + } + + [Fact] + public async Task StageRerun_NewJobsQueuedAlongsideOld_WaitsForNew() + { + var azdo = new FakeAzureDevOpsService().WithPreviouslyProcessedJob("job-linux-v1").WithPreviouslyProcessedJob("job-win-v1"); + var helix = new FakeHelixService(); + ConfigureSnapshots(azdo, helix, + [ + [PipelineJob("Build Linux", "completed", "succeeded"), PipelineJob("Build Win", "inProgress"), MonitorJob()], + [PipelineJob("Build Linux", "completed", "succeeded"), PipelineJob("Build Win", "completed", "succeeded"), MonitorJob()], + ], + [ + (jobs: [HelixJob("job-linux-v1", "finished"), HelixJob("job-linux-v2", "running")], passFail: EmptyPassFail()), + (jobs: [HelixJob("job-linux-v1", "finished"), HelixJob("job-linux-v2", "finished"), HelixJob("job-win-v2", "finished")], passFail: Dict(("job-linux-v2", PassFail(passed: ["linux-wi"])), ("job-win-v2", PassFail(passed: ["win-wi"])))), + ]); + int delayCount = 0; + var runner = new JobMonitorRunner(DefaultOptions(), NullLogger.Instance, azdo, helix, (_, ct) => { delayCount++; AdvanceFakes(azdo, helix); return Task.CompletedTask; }); + + Assert.Equal(0, await runner.RunAsync(CancellationToken.None)); + Assert.Equal(1, delayCount); + Assert.Equal(["job-linux-v2", "job-win-v2"], azdo.UploadedJobNames); + } + + [Fact] + public async Task MultipleRetries_SkipsAllPriorGenerations() + { + var azdo = new FakeAzureDevOpsService() + .WithPreviouslyProcessedJob("job-linux-attempt1").WithPreviouslyProcessedJob("job-linux-attempt2") + .WithPreviouslyProcessedJob("job-win-attempt1").WithPreviouslyProcessedJob("job-win-attempt2"); + var helix = new FakeHelixService(); + ConfigureSnapshots(azdo, helix, + [[PipelineJob("Build Linux", "completed", "succeeded"), PipelineJob("Build Win", "completed", "succeeded"), MonitorJob()]], + [(jobs: [HelixJob("job-linux-attempt1", "finished"), HelixJob("job-linux-attempt2", "finished"), HelixJob("job-linux-attempt3", "finished"), HelixJob("job-win-attempt1", "finished"), HelixJob("job-win-attempt2", "finished"), HelixJob("job-win-attempt3", "finished")], passFail: Dict(("job-linux-attempt3", PassFail(passed: ["linux-wi"])), ("job-win-attempt3", PassFail(passed: ["win-wi"]))))]); + var runner = CreateRunner(azdo, helix); + + Assert.Equal(0, await runner.RunAsync(CancellationToken.None)); + Assert.Equal(["job-linux-attempt3", "job-win-attempt3"], azdo.UploadedJobNames); + } + + [Fact] + public async Task MonitorCrashAndRestart_ProcessesRemainingDelta() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + ConfigureSnapshots(azdo, helix, + [[PipelineJob("Build Linux", "completed", "succeeded"), PipelineJob("Build Win", "completed", "succeeded"), MonitorJob()]], + [(jobs: [HelixJob("job-linux", "finished"), HelixJob("job-win", "finished")], passFail: Dict(("job-linux", PassFail(passed: ["linux-wi"])), ("job-win", PassFail(passed: ["win-wi"]))))]); + + helix.FailDownloadForJob("job-win"); + var runner1 = CreateRunner(azdo, helix); + Assert.Equal(1, await runner1.RunAsync(CancellationToken.None)); + Assert.Equal(["job-linux"], azdo.UploadedJobNames); + + helix.ClearDownloadFailures(); + var runner2 = CreateRunner(azdo, helix); + Assert.Equal(0, await runner2.RunAsync(CancellationToken.None)); + Assert.Equal(["job-linux", "job-win"], azdo.UploadedJobNames); + } + + [Fact] + public async Task RetryWithFailedSubsetResubmitted_OnlyNewJobProcessed() + { + var azdo = new FakeAzureDevOpsService().WithPreviouslyProcessedJob("job-linux-attempt1"); + var helix = new FakeHelixService(); + ConfigureSnapshots(azdo, helix, + [[PipelineJob("Build Linux (retry)", "completed", "succeeded"), MonitorJob()]], + [(jobs: [HelixJob("job-linux-attempt1", "failed"), HelixJob("job-linux-attempt2", "finished")], passFail: Dict(("job-linux-attempt1", PassFail(failed: ["wi-2"])), ("job-linux-attempt2", PassFail(passed: ["wi-2"]))))]); + var runner = CreateRunner(azdo, helix); + + Assert.Equal(0, await runner.RunAsync(CancellationToken.None)); + Assert.Equal(["job-linux-attempt2"], azdo.UploadedJobNames); + } + + // ----------------------------------------------------------------------- + // Edge Cases + // ----------------------------------------------------------------------- + + [Fact] + public async Task MonitorTimesOut_ThrowsOperationCanceledException() + { + var (_, _, runner, _) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "inProgress"), MonitorJob()]], + helixSnapshots: [(jobs: [HelixJob("job-linux", "running")], passFail: EmptyPassFail())]); + + using var cts = new CancellationTokenSource(); + cts.Cancel(); + await Assert.ThrowsAsync(() => runner.RunAsync(cts.Token)); + } + + [Fact] + public async Task AllPipelineJobsFailWhileHelixStillRunning_ExitsImmediately() + { + var (azdo, _, runner, delayCount) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "completed", "failed"), MonitorJob()]], + helixSnapshots: [(jobs: [HelixJob("job-linux", "running")], passFail: EmptyPassFail())]); + + Assert.Equal(1, await runner.RunAsync(CancellationToken.None)); + Assert.Equal(0, delayCount()); + } + + [Fact] + public async Task DownloadFailureMidStream_RetryReusesTestRun_NoDuplicate() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + ConfigureSnapshots(azdo, helix, + [[PipelineJob("Build Linux", "completed", "succeeded"), PipelineJob("Build Win", "completed", "succeeded"), MonitorJob()]], + [(jobs: [HelixJob("job-linux", "finished"), HelixJob("job-win", "finished")], passFail: Dict(("job-linux", PassFail(passed: ["linux-wi"])), ("job-win", PassFail(passed: ["win-wi"]))))]); + + helix.FailDownloadForJob("job-win"); + var runner1 = CreateRunner(azdo, helix); + Assert.Equal(1, await runner1.RunAsync(CancellationToken.None)); + Assert.Equal(["job-linux"], azdo.UploadedJobNames); + + helix.ClearDownloadFailures(); + var runner2 = CreateRunner(azdo, helix); + Assert.Equal(0, await runner2.RunAsync(CancellationToken.None)); + Assert.Equal(["job-linux", "job-win"], azdo.UploadedJobNames); + + // Key invariant: exactly 1 test run CREATED for job-win (second call reused in-progress one) + Assert.Equal(1, azdo.CreatedTestRuns.Count(n => n.Equals("job-win", StringComparison.OrdinalIgnoreCase))); + } + + [Fact] + public async Task AllPipelineJobsCanceled_ExitOne_NoUploads() + { + var (azdo, _, runner, _) = CreateScenario( + timelineSnapshots: [[PipelineJob("Build Linux", "completed", "canceled"), PipelineJob("Build Win", "completed", "canceled"), MonitorJob()]], + helixSnapshots: [(jobs: Array.Empty(), passFail: EmptyPassFail())]); + + Assert.Equal(1, await runner.RunAsync(CancellationToken.None)); + Assert.Empty(azdo.UploadedJobNames); + } + + // ----------------------------------------------------------------------- + // Helpers + // ----------------------------------------------------------------------- + + private static JobMonitorOptions DefaultOptions() => new() + { + BuildId = "123", + CollectionUri = "https://dev.azure.com/dnceng/", + JobMonitorName = DefaultMonitorName, + MaximumWaitMinutes = 1, + PollingIntervalSeconds = 0, + Organization = "dotnet", + RepositoryName = "arcade", + PrNumber = 99999, + SystemAccessToken = "token", + TeamProject = "public", + WorkingDirectory = System.IO.Path.Combine(System.IO.Path.GetTempPath(), "job-monitor-test"), + }; + + private static readonly Func NoDelay = (_, _) => Task.CompletedTask; + private static Dictionary EmptyPassFail() => new(StringComparer.OrdinalIgnoreCase); + + private static Dictionary Dict(params (string jobName, HelixJobPassFail pf)[] entries) + { + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var (jobName, pf) in entries) dict[jobName] = pf; + return dict; + } + + private static void AdvanceFakes(FakeAzureDevOpsService azdo, FakeHelixService helix) { azdo.AdvanceTimeline(); helix.AdvanceSnapshot(); } + + private static void ConfigureSnapshots( + FakeAzureDevOpsService azdo, FakeHelixService helix, + AzureDevOpsTimelineRecord[][] timelineSnapshots, + (HelixJobInfo[] jobs, Dictionary passFail)[] helixSnapshots) + { + foreach (var timeline in timelineSnapshots) azdo.AddTimelineSnapshot(timeline); + foreach (var (jobs, passFail) in helixSnapshots) helix.AddSnapshot(jobs, passFail); + } + + private static JobMonitorRunner CreateRunner(FakeAzureDevOpsService azdo, FakeHelixService helix, Func delayFunc = null) + => new(DefaultOptions(), NullLogger.Instance, azdo, helix, delayFunc ?? NoDelay); + + private static (FakeAzureDevOpsService azdo, FakeHelixService helix, JobMonitorRunner runner, Func delayCount) CreateScenario( + AzureDevOpsTimelineRecord[][] timelineSnapshots, + (HelixJobInfo[] jobs, Dictionary passFail)[] helixSnapshots) + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + ConfigureSnapshots(azdo, helix, timelineSnapshots, helixSnapshots); + int delays = 0; + var runner = new JobMonitorRunner(DefaultOptions(), NullLogger.Instance, azdo, helix, (_, ct) => { delays++; AdvanceFakes(azdo, helix); return Task.CompletedTask; }); + return (azdo, helix, runner, () => delays); + } + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/NonParallelTestCollection.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/NonParallelTestCollection.cs new file mode 100644 index 00000000000..dba3679849f --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/NonParallelTestCollection.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace Microsoft.DotNet.Helix.Sdk.Tests +{ + [CollectionDefinition("NonParallel", DisableParallelization = true)] + public class NonParallelTestCollection { } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs new file mode 100644 index 00000000000..f9856ee6170 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/ScenarioHelpers/ScenarioHelpers.cs @@ -0,0 +1,25 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.DotNet.Helix.JobMonitor; +using Microsoft.DotNet.Helix.JobMonitor.Models; + +namespace Microsoft.DotNet.Helix.Sdk.Tests.ScenarioHelpers +{ + internal static class ScenarioHelpers + { + public const string DefaultMonitorName = "Helix Job Monitor"; + + public static AzureDevOpsTimelineRecord PipelineJob(string name, string state, string result = null) + => new() { Type = "Job", Name = name, State = state, Result = result }; + + public static AzureDevOpsTimelineRecord MonitorJob(string name = DefaultMonitorName) + => new() { Type = "Job", Name = name, State = "inProgress" }; + + public static HelixJobInfo HelixJob(string jobName, string status) + => new(jobName, status); + + public static HelixJobPassFail PassFail(string[] passed = null, string[] failed = null) + => new(passed ?? [], failed ?? []); + } +} From 84eed5dcef03704a0e778a9c2dc579f82015ccf2 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 24 Apr 2026 13:31:52 -0700 Subject: [PATCH 48/66] Fix HJM install script: use absolute path for dotnet.sh The install step does pushd to AGENT_TEMPDIRECTORY but then calls ./eng/common/dotnet.sh which is relative to the temp directory, not the source checkout. Use BUILD_SOURCESDIRECTORY for the absolute path. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/common/core-templates/job/helix-job-monitor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index e237c4f2e8d..4d759f2787b 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -174,7 +174,7 @@ jobs: echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'." toolSource="$nupkgDir" - ./eng/common/dotnet.sh tool install \ + "$BUILD_SOURCESDIRECTORY/eng/common/dotnet.sh" tool install \ --tool-path "$toolPath" "$packageId" \ --version "$toolVersion" \ --add-source "$toolSource" \ From abe956a8786e8f9ed748b3ec64aefcaff63c8940 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 24 Apr 2026 14:36:03 -0700 Subject: [PATCH 49/66] Fix HJM run step: set DOTNET_ROOT for repo-local .NET runtime When the tool is installed from a local nupkg, the .NET runtime is installed to the repo-local .dotnet directory by eng/common/dotnet.sh. The run step needs DOTNET_ROOT and PATH set to find this runtime, since each pipeline step runs in a separate process. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/common/core-templates/job/helix-job-monitor.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 4d759f2787b..2cb35b82625 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -226,6 +226,13 @@ jobs: if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then # Tool was installed into a tool-path that has been prepended to PATH. + # Ensure the repo-local .NET runtime (installed by eng/common/dotnet.sh) + # is discoverable by the tool. + localDotnet="$BUILD_SOURCESDIRECTORY/.dotnet" + if [ -d "$localDotnet" ]; then + export DOTNET_ROOT="$localDotnet" + export PATH="$localDotnet:$PATH" + fi '${{ parameters.toolCommand }}' "${toolArgs[@]}" else # Tool was restored from the local .config/dotnet-tools.json manifest; invoke it From 6af3385facc5202e4b9f25c768f6d1a0083a28f8 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 24 Apr 2026 15:26:48 -0700 Subject: [PATCH 50/66] Move Helix Job Monitor into the Test stage The HJM job should be part of each stage that submits Helix work items, not a separate stage. Move it into the Test stage as a parallel job so it can monitor the Helix work items submitted by the test jobs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- azure-pipelines-pr.yml | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index 8f9212476d7..23148d35764 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -224,6 +224,14 @@ stages: SYSTEM_ACCESSTOKEN: $(System.AccessToken) HelixAccessToken: '' + # Helix Job Monitor runs as a job within the Test stage so it can monitor + # Helix work items submitted by the other jobs in this stage. + - template: /eng/common/core-templates/job/helix-job-monitor.yml + parameters: + # Install from the nupkg produced by the Build stage's Windows_NT Release job. + toolNupkgArtifactName: Artifacts_Windows_NT_Release + toolNupkgArtifactSubPath: packages/Release/NonShipping + - stage: Test_XHarness displayName: Test XHarness SDK dependsOn: build @@ -361,12 +369,3 @@ stages: env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) HelixAccessToken: '' - -- template: /eng/common/core-templates/stages/helix-job-monitor.yml - parameters: - dependsOn: - - build - # Install the Helix job monitor from the nupkg produced by the Build stage's - # Windows_NT Release job, instead of pulling it from a published feed. - toolNupkgArtifactName: Artifacts_Windows_NT_Release - toolNupkgArtifactSubPath: packages/Release/NonShipping From 413702cd6aceb5ab62baa13170f8e294854780b1 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 24 Apr 2026 15:31:14 -0700 Subject: [PATCH 51/66] Fix HJM: use dotnet.sh for install, export DOTNET_ROOT for run Use ./eng/common/dotnet.sh for tool install (handles SDK install). Export DOTNET_ROOT and prepend .dotnet to PATH via ##vso task commands so the run step's tool shim finds the repo-local .NET runtime. Verified locally: tool installs and runs with DOTNET_ROOT set. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../core-templates/job/helix-job-monitor.yml | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 2cb35b82625..b0691b07cac 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -144,9 +144,6 @@ jobs: toolPath="$AGENT_TEMPDIRECTORY/helix-job-monitor-tool" mkdir -p "$toolPath" - pushd "$AGENT_TEMPDIRECTORY" > /dev/null - trap 'popd > /dev/null' EXIT - packageId='${{ parameters.toolPackageId }}' toolVersion='${{ parameters.toolVersion }}' nupkgArtifactSubPath='${{ parameters.toolNupkgArtifactSubPath }}' @@ -172,16 +169,23 @@ jobs: fi echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'." - toolSource="$nupkgDir" - "$BUILD_SOURCESDIRECTORY/eng/common/dotnet.sh" tool install \ + ./eng/common/dotnet.sh tool install \ --tool-path "$toolPath" "$packageId" \ --version "$toolVersion" \ - --add-source "$toolSource" \ + --add-source "$nupkgDir" \ --ignore-failed-sources echo "##vso[task.prependpath]$toolPath" echo "##vso[task.setvariable variable=HelixJobMonitorToolPath]$toolPath" + + # Export DOTNET_ROOT so the tool shim can find the repo-local .NET runtime + # that was installed by eng/common/dotnet.sh above. + localDotnet="$BUILD_SOURCESDIRECTORY/.dotnet" + if [ -d "$localDotnet" ]; then + echo "##vso[task.setvariable variable=DOTNET_ROOT]$localDotnet" + echo "##vso[task.prependpath]$localDotnet" + fi displayName: Install Helix Job Monitor - ${{ else }}: @@ -225,14 +229,7 @@ jobs: if [ -n "$prNumber" ]; then toolArgs+=( --pr-number "$prNumber" ); fi if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then - # Tool was installed into a tool-path that has been prepended to PATH. - # Ensure the repo-local .NET runtime (installed by eng/common/dotnet.sh) - # is discoverable by the tool. - localDotnet="$BUILD_SOURCESDIRECTORY/.dotnet" - if [ -d "$localDotnet" ]; then - export DOTNET_ROOT="$localDotnet" - export PATH="$localDotnet:$PATH" - fi + # Tool was installed into a tool-path and DOTNET_ROOT was set by the install step. '${{ parameters.toolCommand }}' "${toolArgs[@]}" else # Tool was restored from the local .config/dotnet-tools.json manifest; invoke it From a3b10b79ea8feb4b7787c1436159c63b30c21125 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 24 Apr 2026 15:34:43 -0700 Subject: [PATCH 52/66] Simplify HJM: run tool DLL via ./eng/common/dotnet.sh exec No DOTNET_ROOT or PATH manipulation needed. The install step finds the tool DLL in the .store directory, and the run step invokes it via ./eng/common/dotnet.sh exec which handles SDK resolution. Verified locally: dotnet.cmd exec --help works correctly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../core-templates/job/helix-job-monitor.yml | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index b0691b07cac..8b5c694c787 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -176,16 +176,16 @@ jobs: --add-source "$nupkgDir" \ --ignore-failed-sources - echo "##vso[task.prependpath]$toolPath" - echo "##vso[task.setvariable variable=HelixJobMonitorToolPath]$toolPath" - - # Export DOTNET_ROOT so the tool shim can find the repo-local .NET runtime - # that was installed by eng/common/dotnet.sh above. - localDotnet="$BUILD_SOURCESDIRECTORY/.dotnet" - if [ -d "$localDotnet" ]; then - echo "##vso[task.setvariable variable=DOTNET_ROOT]$localDotnet" - echo "##vso[task.prependpath]$localDotnet" + # Locate the tool DLL so the run step can invoke it via ./eng/common/dotnet.sh exec. + toolDll=$(find "$toolPath/.store" -path '*/tools/*/any/*.deps.json' -type f | head -n 1) + toolDll="${toolDll%.deps.json}.dll" + if [ ! -f "$toolDll" ]; then + echo "Could not find tool DLL in '$toolPath/.store'." >&2 + exit 1 fi + + echo "Tool DLL: $toolDll" + echo "##vso[task.setvariable variable=HelixJobMonitorDll]$toolDll" displayName: Install Helix Job Monitor - ${{ else }}: @@ -229,8 +229,8 @@ jobs: if [ -n "$prNumber" ]; then toolArgs+=( --pr-number "$prNumber" ); fi if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then - # Tool was installed into a tool-path and DOTNET_ROOT was set by the install step. - '${{ parameters.toolCommand }}' "${toolArgs[@]}" + # Tool was installed from a local nupkg; run the DLL via the repo-local dotnet. + ./eng/common/dotnet.sh exec "$(HelixJobMonitorDll)" "${toolArgs[@]}" else # Tool was restored from the local .config/dotnet-tools.json manifest; invoke it # through the manifest from the repo root. From 928a946132b4e70d72920f136339c92f6e6f8481 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 24 Apr 2026 19:14:28 -0700 Subject: [PATCH 53/66] Fix HJM install: use custom configfile instead of --add-source The repo NuGet.config has package source mapping enabled, which blocks --add-source. Use a minimal --configfile with only the local nupkg directory to avoid the conflict. Verified locally: install + DLL discovery + dotnet exec all work. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../core-templates/job/helix-job-monitor.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 8b5c694c787..0271698de59 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -170,11 +170,23 @@ jobs: echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'." + # Create a minimal NuGet.config that only references the local nupkg directory. + # This avoids conflicts with the repo's package source mapping which blocks --add-source. + toolNugetConfig="$AGENT_TEMPDIRECTORY/helix-job-monitor-nuget.config" + cat > "$toolNugetConfig" < + + + + + + +EOF + ./eng/common/dotnet.sh tool install \ --tool-path "$toolPath" "$packageId" \ --version "$toolVersion" \ - --add-source "$nupkgDir" \ - --ignore-failed-sources + --configfile "$toolNugetConfig" # Locate the tool DLL so the run step can invoke it via ./eng/common/dotnet.sh exec. toolDll=$(find "$toolPath/.store" -path '*/tools/*/any/*.deps.json' -type f | head -n 1) From 5a015a6fe8b29a0dcfbcd75270376abb34cec545 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 24 Apr 2026 19:16:21 -0700 Subject: [PATCH 54/66] Fix YAML: replace heredoc with printf for NuGet config Heredoc EOF at column 1 breaks YAML parsing in the bash block. Use printf instead. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- eng/common/core-templates/job/helix-job-monitor.yml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 0271698de59..6cef9e27e48 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -173,15 +173,7 @@ jobs: # Create a minimal NuGet.config that only references the local nupkg directory. # This avoids conflicts with the repo's package source mapping which blocks --add-source. toolNugetConfig="$AGENT_TEMPDIRECTORY/helix-job-monitor-nuget.config" - cat > "$toolNugetConfig" < - - - - - - -EOF + printf '\n\n \n \n \n \n\n' "$nupkgDir" > "$toolNugetConfig" ./eng/common/dotnet.sh tool install \ --tool-path "$toolPath" "$packageId" \ From 44f943b760ed93a7c80c0ba0bb287d5272622aa2 Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Fri, 24 Apr 2026 19:31:33 -0700 Subject: [PATCH 55/66] Make PR number optional for non-PR builds Remove hard validation requirement for PR number. For non-PR builds (manual queue), use branch-based Helix source filter instead. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../JobMonitor/JobMonitorOptions.cs | 5 ----- .../JobMonitor/JobMonitorRunner.cs | 9 +++++++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs index 1318216f427..c68befc59e0 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs @@ -115,11 +115,6 @@ private void Validate() { throw new InvalidOperationException("Organization must be provided either by argument or pipeline environment."); } - - if (!PrNumber.HasValue) - { - throw new InvalidOperationException("Pull request number must be provided either by argument or pipeline environment."); - } } private static string RequireValue(string value, string argumentName, string environmentName) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index f12ac0e0b2d..c3d5e12a8d4 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -357,9 +357,14 @@ public RealHelixService(JobMonitorOptions options, ILogger logger) public async Task> GetJobsAsync(CancellationToken cancellationToken) { + // Build the Helix source filter. For PR builds, use the PR merge ref. + // For CI builds without a PR, use the branch-based source. + string source = _options.PrNumber.HasValue + ? $"pr/public/{_options.Organization}/{_options.RepositoryName}/refs/pull/{_options.PrNumber}/merge" + : $"official/public/{_options.Organization}/{_options.RepositoryName}"; + IImmutableList jobs = await RetryAsync( - async () => await _helixApi.Job.ListAsync( - source: $"pr/public/{_options.Organization}/{_options.RepositoryName}/refs/pull/{_options.PrNumber}/merge"), + async () => await _helixApi.Job.ListAsync(source: source), cancellationToken); return jobs From cec81ad5f900a85296a0571909c5087df12555a9 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 27 Apr 2026 10:48:31 +0200 Subject: [PATCH 56/66] Add a timeout --- azure-pipelines-pr.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index 8f9212476d7..db40bbe8f90 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -366,6 +366,7 @@ stages: parameters: dependsOn: - build + timeoutInMinutes: 60 # TODO: Increase this # Install the Helix job monitor from the nupkg produced by the Build stage's # Windows_NT Release job, instead of pulling it from a published feed. toolNupkgArtifactName: Artifacts_Windows_NT_Release From 08a04ada1ed476da241dbad6a6ee144c41904a39 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 27 Apr 2026 10:48:44 +0200 Subject: [PATCH 57/66] Fix job count --- .../JobMonitor/JobMonitorRunner.cs | 26 +++++++++++++------ 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 3850abc7d21..246dc589d71 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -64,27 +64,37 @@ public async Task RunAsync() bool anyNonMonitorJobFailures = false; int failedHelixJobCount = 0; int processedHelixJobCount = 0; + int allHelixJobCount = 0; + int completedJobsCount = 0; while (true) { cancellationToken.ThrowIfCancellationRequested(); AzureDevOpsTimelineRecord[] timelineRecords = await GetTimelineRecordsAsync(); - IImmutableList jobs = await RetryAsync( - // TODO: "pr/public" is hardcoded but could come from the build technically - async () => await _helixApi.Job.ListAsync(source: $"pr/public/{_options.Organization}/{_options.RepositoryName}/refs/pull/{_options.PrNumber}/merge"), - cancellationToken); + IReadOnlyList associatedJobsWithBuild = + [ + ..(await RetryAsync( + // TODO: "pr/public" is hardcoded but could come from the build technically + async () => await _helixApi.Job.ListAsync(source: $"pr/public/{_options.Organization}/{_options.RepositoryName}/refs/pull/{_options.PrNumber}/merge"), + cancellationToken)) + .Where(j => ((JObject)j.Properties).TryGetValue("BuildId", out JToken buildId) && buildId?.ToString() == _options.BuildId) + ]; // Filter jobs to completed ones belonging to this build IReadOnlyCollection completedJobs = [ - ..jobs - .Where(j => ((JObject)j.Properties).TryGetValue("BuildId", out JToken buildId) && buildId?.ToString() == _options.BuildId) + ..associatedJobsWithBuild .Where(j => j.Finished != null) .OrderBy(j => j.Name, StringComparer.OrdinalIgnoreCase) ]; - _logger.LogInformation("{CompletedCount}/{TotalCount} Helix jobs finished", completedJobs.Count, jobs.Count); + if (allHelixJobCount != associatedJobsWithBuild.Count || completedJobsCount != completedJobs.Count) + { + _logger.LogInformation("{CompletedCount}/{TotalCount} Helix jobs finished", completedJobs.Count, associatedJobsWithBuild.Count); + allHelixJobCount = associatedJobsWithBuild.Count; + completedJobsCount = completedJobs.Count; + } foreach (JobSummary job in completedJobs.Where(j => !processedHelixJobs.Contains(j.Name))) { @@ -99,7 +109,7 @@ public async Task RunAsync() anyNonMonitorJobFailures = HelixJobMonitorUtilities.HasFailedNonMonitorJobs(timelineRecords, _options.JobMonitorName); bool allPipelineJobsComplete = HelixJobMonitorUtilities.AreNonMonitorJobsComplete(timelineRecords, _options.JobMonitorName); - bool allHelixJobsComplete = jobs.Count != 0 && jobs.Count == completedJobs.Count; + bool allHelixJobsComplete = associatedJobsWithBuild.Count != 0 && associatedJobsWithBuild.Count == completedJobs.Count; if (allPipelineJobsComplete && allHelixJobsComplete) { From c0ae66dff43aa5a7e154a005bcc91db38f342966 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 27 Apr 2026 11:07:51 +0200 Subject: [PATCH 58/66] Comment out attachment uploads --- .../AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs index 05e136ce4a5..0f8314e1169 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs @@ -151,7 +151,7 @@ void ProcessTestForMetadata(AggregatedResult result) { ProcessTestForMetadata(result); } - + /* var uploadedUrls = new Dictionary(); /* TODO foreach ((int key, List? testNames) in partitionedResults) @@ -159,7 +159,7 @@ void ProcessTestForMetadata(AggregatedResult result) byte[] csvBytes = CreateCompressedCsv(testNames); string fileName = $"{Guid.NewGuid():N}.csv.gz"; uploadedUrls[key] = await _uploadClient.UploadAsync(csvBytes, fileName, "application/gzip", cancellationToken); - }*/ + }* / var dataModel = new { From 1043a15eb3a223b263815d685fdd5b9a5ad3329e Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 27 Apr 2026 11:20:55 +0200 Subject: [PATCH 59/66] Tidy up and improve resilience --- eng/common/core-templates/job/helix-job-monitor.yml | 9 ++------- .../core-templates/stages/helix-job-monitor.yml | 12 ------------ .../AzureDevOpsResultPublisher.cs | 13 ++++++------- .../JobMonitor/JobMonitorRunner.cs | 9 ++++++--- 4 files changed, 14 insertions(+), 29 deletions(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 51b136f49f8..addf254cefb 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -4,11 +4,6 @@ parameters: type: string default: HelixJobMonitor -# Azure DevOps job display name. -- name: displayName - type: string - default: Helix Job Monitor - # Pool override. When empty the template selects a default azurelinux pool based on the team project. - name: pool type: object @@ -112,7 +107,7 @@ parameters: jobs: - job: ${{ parameters.jobName }} - displayName: ${{ parameters.displayName }} + displayName: Monitor Helix Jobs timeoutInMinutes: ${{ parameters.timeoutInMinutes }} ${{ if ne(length(parameters.dependsOn), 0) }}: dependsOn: ${{ parameters.dependsOn }} @@ -236,7 +231,7 @@ jobs: trap 'popd > /dev/null' EXIT ./eng/common/dotnet.sh tool run '${{ parameters.toolCommand }}' -- "${toolArgs[@]}" fi - displayName: Run Helix Job Monitor + displayName: Monitor Helix Jobs env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }} diff --git a/eng/common/core-templates/stages/helix-job-monitor.yml b/eng/common/core-templates/stages/helix-job-monitor.yml index 25f0004ac70..0c3d68ce9fb 100644 --- a/eng/common/core-templates/stages/helix-job-monitor.yml +++ b/eng/common/core-templates/stages/helix-job-monitor.yml @@ -4,11 +4,6 @@ parameters: type: string default: Helix_Job_Monitor -# Stage display name. -- name: displayName - type: string - default: Helix Job Monitor - # Optional list of stages this stage depends on. - name: dependsOn type: object @@ -24,11 +19,6 @@ parameters: type: string default: HelixJobMonitor -# Job display name produced inside the stage. -- name: jobDisplayName - type: string - default: Helix Job Monitor - # NuGet package id of the Helix job monitor tool. - name: toolPackageId type: string @@ -104,7 +94,6 @@ parameters: stages: - stage: ${{ parameters.stageName }} - displayName: ${{ parameters.displayName }} dependsOn: ${{ parameters.dependsOn }} ${{ if ne(parameters.condition, '') }}: condition: ${{ parameters.condition }} @@ -112,7 +101,6 @@ stages: - template: /eng/common/core-templates/job/helix-job-monitor.yml parameters: jobName: ${{ parameters.jobName }} - displayName: ${{ parameters.jobDisplayName }} toolPackageId: ${{ parameters.toolPackageId }} toolCommand: ${{ parameters.toolCommand }} toolVersion: ${{ parameters.toolVersion }} diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs index 0f8314e1169..54a495d3550 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs @@ -9,7 +9,6 @@ using System.Text.Json; using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; @@ -63,17 +62,17 @@ public async Task UploadTestResultsAsync(IEnumerable results, { try { + long publishedTestCount = 0; var converted = ConvertResults(results, resultMetadata).ToList(); - var hotPathTests = new List(); - foreach (List batch in Batch(converted, 1000, static t => Size(t.Converted))) { IReadOnlyList publishedTests = await PublishResultsAsync(batch, cancellationToken); - hotPathTests.AddRange(publishedTests); - _logger.LogInformation("Uploaded {Count} results", publishedTests.Count); + publishedTestCount += publishedTests.Count; } - await SendMetadataAsync(hotPathTests, results, cancellationToken); + _logger.LogInformation("Uploaded {Count} results", publishedTestCount); + + await SendMetadataAsync(results, cancellationToken); } catch (TerminalError ex) { @@ -95,7 +94,6 @@ await _eventClient.ErrorAsync( } private static async Task SendMetadataAsync( - IReadOnlyList backChannelCases, IEnumerable allTestResults, CancellationToken cancellationToken) { @@ -149,6 +147,7 @@ void ProcessTestForMetadata(AggregatedResult result) foreach (AggregatedResult result in allTestResults) { + cancellationToken.ThrowIfCancellationRequested(); ProcessTestForMetadata(result); } /* diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 246dc589d71..9e96d55e5cc 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -146,7 +146,6 @@ private async Task ProcessCompletedJobAsync(JobSummary helixJob, Cancellat _logger.LogInformation("Processing completed job {jobName}...", helixJob.Name); string testRunName = GetTestRunNameFromJob(helixJob); - int testRunId = await StartTestRunAsync(testRunName, helixJob.Name); string resultsDirectory = Path.Combine(_options.WorkingDirectory, SanitizeDirName(helixJob.Name)); Directory.CreateDirectory(resultsDirectory); @@ -156,6 +155,8 @@ private async Task ProcessCompletedJobAsync(JobSummary helixJob, Cancellat bool helixJobSuccessful = failedWorkItemCount == 0; int sucessfulWorkItemCount = workItems.Count - failedWorkItemCount; + int testRunId = await StartTestRunAsync(testRunName, helixJob.Name); + try { List downloadedFiles = await DownloadTestResultsAsync(helixJob.Name, workItems, resultsDirectory, cancellationToken); @@ -172,8 +173,10 @@ private async Task ProcessCompletedJobAsync(JobSummary helixJob, Cancellat _logger.LogError(ex, "Failed to upload test results for job {JobName} to Azure DevOps. Test run ID was {TestRunId}.", helixJob.Name, testRunId); return false; } - - await StopTestRunAsync(testRunId, testRunName); + finally + { + await StopTestRunAsync(testRunId, testRunName); + } _logger.LogInformation("Job '{JobName}' completed ({PassedCount} passed, {FailedCount} failed).", helixJob.Name, sucessfulWorkItemCount, failedWorkItemCount); return failedWorkItemCount == 0; From dd31309a9c0cb1ac69df882e64428bd95a1f08e6 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 27 Apr 2026 11:23:28 +0200 Subject: [PATCH 60/66] Fix subresult uploading --- .../AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs index 54a495d3550..e2e2e1d9604 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs @@ -300,7 +300,7 @@ private async Task SendAttachmentAsync( Convert.ToBase64String(Encoding.UTF8.GetBytes(attachment.Text))); string path = subResultId is long subId - ? $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results/{testId}/subresults/{subId}/attachments?api-version=7.1-preview.1" + ? $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results/{testId}/attachments?testSubResultId={subId}&api-version=7.1-preview.1" : $"{_azdoParameters.TeamProject}/_apis/test/runs/{_azdoParameters.TestRunId}/results/{testId}/attachments?api-version=7.1-preview.1"; using HttpResponseMessage response = await SendWithRetryAsync(HttpMethod.Post, path, request, cancellationToken); From 9243e5bd7c0dc447f9d05e44807d0dd9d6521afd Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 27 Apr 2026 14:52:39 +0200 Subject: [PATCH 61/66] Extract real implementations into own files --- .../AzureDevOpsResultPublisher.cs | 1 - .../AzureDevOpsTestPublisher/RetryHelper.cs | 28 ++ .../JobMonitor/JobMonitorRunner.cs | 368 +----------------- .../JobMonitor/Models/WorkItemTestResults.cs | 9 + .../JobMonitor/Services/AzureDevOpsService.cs | 158 ++++++++ .../JobMonitor/Services/HelixService.cs | 182 +++++++++ 6 files changed, 384 insertions(+), 362 deletions(-) create mode 100644 src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/RetryHelper.cs create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Models/WorkItemTestResults.cs create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs create mode 100644 src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs index 05e136ce4a5..f3eb8deadce 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs @@ -9,7 +9,6 @@ using System.Text.Json; using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; using Microsoft.Extensions.Logging; -using Microsoft.Extensions.Logging.Abstractions; namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/RetryHelper.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/RetryHelper.cs new file mode 100644 index 00000000000..fe5b37e5772 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/RetryHelper.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; + +public class RetryHelper +{ + public static async Task RetryAsync(Func> action, CancellationToken cancellationToken) + { + Exception? last = null; + for (int attempt = 0; attempt < 5; attempt++) + { + cancellationToken.ThrowIfCancellationRequested(); + + try + { + return await action(); + } + catch (Exception ex) when (attempt < 4) + { + last = ex; + await Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, attempt + 1)), cancellationToken); + } + } + + throw last ?? new InvalidOperationException("Retry failed without capturing an exception."); + } +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index c3d5e12a8d4..9eaee128e1e 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -3,25 +3,12 @@ using System; using System.Collections.Generic; -using System.Collections.Immutable; -using System.Globalization; using System.IO; using System.Linq; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; using System.Threading; using System.Threading.Tasks; -using Azure; -using Azure.Storage.Blobs; -using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; -using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; -using Microsoft.DotNet.Helix.Client; -using Microsoft.DotNet.Helix.Client.Models; using Microsoft.DotNet.Helix.JobMonitor.Models; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; namespace Microsoft.DotNet.Helix.JobMonitor { @@ -37,10 +24,11 @@ internal sealed class JobMonitorRunner : IJobMonitorRunner, IDisposable /// Constructor for production use with real services. /// public JobMonitorRunner(JobMonitorOptions options, ILogger logger) - : this(options, logger, - CreateRealAzureDevOpsService(options, logger), - CreateRealHelixService(options, logger), - null) + : this(options, + logger, + new AzureDevOpsService(options, logger), + new HelixService(options, logger), + null) { } @@ -181,350 +169,8 @@ private async Task ProcessCompletedJobAsync( public void Dispose() { - // Real services handle their own cleanup via the azdo/helix service implementations - } - - // ----------------------------------------------------------------- - // Factory methods for production service implementations - // ----------------------------------------------------------------- - - private static IAzureDevOpsService CreateRealAzureDevOpsService(JobMonitorOptions options, ILogger logger) - => new RealAzureDevOpsService(options, logger); - - private static IHelixService CreateRealHelixService(JobMonitorOptions options, ILogger logger) - => new RealHelixService(options, logger); - - // ----------------------------------------------------------------- - // Real AzDO service implementation (extracted from original runner) - // ----------------------------------------------------------------- - - private sealed class RealAzureDevOpsService : IAzureDevOpsService, IDisposable - { - private const string MonitoredJobTagPrefix = "MonitoredJob:"; - private readonly JobMonitorOptions _options; - private readonly ILogger _logger; - private readonly HttpClient _azdoClient; - - public RealAzureDevOpsService(JobMonitorOptions options, ILogger logger) - { - _options = options; - _logger = logger; - _azdoClient = new HttpClient(); - string encodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + options.SystemAccessToken)); - _azdoClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", encodedToken); - _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-job-monitor"); - } - - public async Task> GetTimelineRecordsAsync(CancellationToken cancellationToken) - { - JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/build/builds/{_options.BuildId}/timeline?api-version=7.1-preview.2", cancellationToken: cancellationToken); - return data?["records"]?.ToObject() ?? []; - } - - public async Task> GetProcessedHelixJobNamesAsync(CancellationToken cancellationToken) - { - string buildUri = Uri.EscapeDataString($"vstfs:///Build/Build/{_options.BuildId}"); - JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?buildUri={buildUri}&api-version=7.1", cancellationToken: cancellationToken); - var processed = new HashSet(StringComparer.OrdinalIgnoreCase); - - foreach (JObject run in (data?["value"] as JArray ?? []).Cast()) - { - int? runId = run.Value("id"); - string state = run.Value("state"); - if (runId == null || !string.Equals(state, "Completed", StringComparison.OrdinalIgnoreCase)) - { - continue; - } - - string helixJobName = await GetMonitoredHelixJobNameAsync(runId.Value, cancellationToken); - if (!string.IsNullOrEmpty(helixJobName)) - { - processed.Add(helixJobName); - } - } - - return processed; - } - - private async Task GetMonitoredHelixJobNameAsync(int testRunId, CancellationToken cancellationToken) - { - JObject run = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=7.1", cancellationToken: cancellationToken); - if (run?["tags"] is not JArray tags) - { - return null; - } - - foreach (JToken tag in tags) - { - string tagName = tag?.Value("name"); - if (!string.IsNullOrEmpty(tagName) && tagName.StartsWith(MonitoredJobTagPrefix, StringComparison.OrdinalIgnoreCase)) - { - return tagName.Substring(MonitoredJobTagPrefix.Length); - } - } - - return null; - } - - public async Task CreateTestRunAsync(string name, string helixJobName, CancellationToken cancellationToken) - { - JObject result = await SendAsync(HttpMethod.Post, - $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?api-version=5.0", - new JObject - { - ["automated"] = true, - ["build"] = new JObject { ["id"] = _options.BuildId }, - ["name"] = name, - ["state"] = "InProgress", - ["tags"] = new JArray { new JObject { ["name"] = MonitoredJobTagPrefix + helixJobName } }, - }, - cancellationToken: cancellationToken); - return result?["id"]?.ToObject() ?? 0; - } - - public async Task CompleteTestRunAsync(int testRunId, CancellationToken cancellationToken) - { - await SendAsync(new HttpMethod("PATCH"), - $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=5.0", - new JObject { ["state"] = "Completed" }, - cancellationToken: cancellationToken); - } - - public async Task UploadTestResultsAsync(int testRunId, IReadOnlyList results, CancellationToken cancellationToken) - { - var publisher = new AzureDevOpsResultPublisher( - new AzureDevOpsReportingParameters( - new Uri(_options.CollectionUri, UriKind.Absolute), - _options.TeamProject, - testRunId.ToString(CultureInfo.InvariantCulture), - _options.SystemAccessToken), - _logger); - - bool allPassed = true; - foreach (WorkItemTestResults workItem in results) - { - _logger.LogInformation("Publishing test results for work item '{WorkItemName}' in job '{JobName}'...", workItem.WorkItemName, workItem.JobName); - allPassed &= await publisher.UploadTestResultsAsync(workItem.TestResultFiles, - new { HelixJobId = workItem.JobName, HelixWorkItemName = workItem.WorkItemName }, - cancellationToken); - } - - return allPassed; - } - - private async Task SendAsync(HttpMethod method, string requestUri, JToken body = null, CancellationToken cancellationToken = default) - { - return await RetryAsync(async () => - { - using var request = new HttpRequestMessage(method, requestUri); - if (body != null) - { - request.Content = new StringContent(body.ToString(Formatting.None), Encoding.UTF8, "application/json"); - } - - using HttpResponseMessage response = await _azdoClient.SendAsync(request, cancellationToken); - string content = response.Content != null ? await response.Content.ReadAsStringAsync(cancellationToken) : null; - if (!response.IsSuccessStatusCode) - { - throw new HttpRequestException($"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}"); - } - - return string.IsNullOrWhiteSpace(content) ? [] : JObject.Parse(content); - }, cancellationToken); - } - - public void Dispose() => _azdoClient.Dispose(); - } - - // ----------------------------------------------------------------- - // Real Helix service implementation (extracted from original runner) - // ----------------------------------------------------------------- - - private sealed class RealHelixService : IHelixService - { - private readonly JobMonitorOptions _options; - private readonly ILogger _logger; - private readonly IHelixApi _helixApi; - - public RealHelixService(JobMonitorOptions options, ILogger logger) - { - _options = options; - _logger = logger; - _helixApi = string.IsNullOrEmpty(options.HelixAccessToken) - ? ApiFactory.GetAnonymous(options.HelixBaseUri) - : ApiFactory.GetAuthenticated(options.HelixBaseUri, options.HelixAccessToken); - } - - public async Task> GetJobsAsync(CancellationToken cancellationToken) - { - // Build the Helix source filter. For PR builds, use the PR merge ref. - // For CI builds without a PR, use the branch-based source. - string source = _options.PrNumber.HasValue - ? $"pr/public/{_options.Organization}/{_options.RepositoryName}/refs/pull/{_options.PrNumber}/merge" - : $"official/public/{_options.Organization}/{_options.RepositoryName}"; - - IImmutableList jobs = await RetryAsync( - async () => await _helixApi.Job.ListAsync(source: source), - cancellationToken); - - return jobs - .Where(j => ((JObject)j.Properties).TryGetValue("BuildId", out JToken buildId) && buildId?.ToString() == _options.BuildId) - .Select(j => new HelixJobInfo( - j.Name, - j.Finished != null ? "finished" : "running", - GetTestRunNameFromJob(j))) - .ToList(); - } - - public async Task GetJobPassFailAsync(string jobName, CancellationToken cancellationToken) - { - IImmutableList workItems = await RetryAsync( - () => _helixApi.WorkItem.ListAsync(jobName), - cancellationToken); - - var passed = new List(); - var failed = new List(); - - foreach (WorkItemSummary wi in workItems) - { - if (wi.ExitCode != 0 || !wi.State.Equals("Finished", StringComparison.OrdinalIgnoreCase)) - { - failed.Add(wi.Name); - } - else - { - passed.Add(wi.Name); - } - } - - return new HelixJobPassFail(passed, failed); - } - - public async Task> DownloadTestResultsAsync( - string jobName, - HelixJobPassFail passFail, - CancellationToken cancellationToken) - { - List downloadedFiles = []; - string outputDirectory = Path.Combine(_options.WorkingDirectory, SanitizeDirName(jobName)); - Directory.CreateDirectory(outputDirectory); - - JobResultsUri resultsUri = await RetryAsync(() => _helixApi.Job.ResultsAsync(jobName), cancellationToken); - - IEnumerable workItemNames = passFail.PassedWorkItems - .Concat(passFail.FailedWorkItems) - .Distinct(StringComparer.OrdinalIgnoreCase); - - foreach (string workItemName in workItemNames) - { - IImmutableList availableFiles = await RetryAsync( - () => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false), - cancellationToken); - - availableFiles = [.. availableFiles.Where(f => LooksLikeTestResultFile(f.Name))]; - if (availableFiles.Count == 0) - { - continue; - } - - string workItemDirectory = Path.Combine(outputDirectory, SanitizeDirName(workItemName)); - Directory.CreateDirectory(workItemDirectory); - - List workItemFiles = []; - foreach (UploadedFile file in availableFiles) - { - string relativePath = file.Name.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar); - string destinationFile = Path.Combine(workItemDirectory, relativePath); - string directory = Path.GetDirectoryName(destinationFile); - if (!string.IsNullOrEmpty(directory)) - { - Directory.CreateDirectory(directory); - } - - try - { - BlobClient blobClient = CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS); - await blobClient.DownloadToAsync(destinationFile, cancellationToken); - workItemFiles.Add(destinationFile); - } - catch (Exception ex) - { - _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItemName); - } - } - - downloadedFiles.Add(new WorkItemTestResults(jobName, workItemName, workItemFiles)); - } - - return downloadedFiles; - } - - private static string GetTestRunNameFromJob(JobSummary helixJob) - { - if (helixJob.Properties is JObject properties - && properties.TryGetValue("TestRunName", out JToken testRunName)) - { - string value = testRunName?.ToString(); - if (!string.IsNullOrEmpty(value)) - { - return value; - } - } - - return helixJob.Name; - } - - private static BlobClient CreateBlobClient(string fileLink, string resultsSas) - { - var options = new BlobClientOptions(); - options.Retry.NetworkTimeout = TimeSpan.FromMinutes(5); - if (string.IsNullOrEmpty(resultsSas)) - { - return new BlobClient(new Uri(fileLink), options); - } - - string strippedUri = fileLink.Contains('?') ? fileLink.Substring(0, fileLink.LastIndexOf('?', StringComparison.Ordinal)) : fileLink; - return new BlobClient(new Uri(strippedUri), new AzureSasCredential(resultsSas), options); - } - - private static bool LooksLikeTestResultFile(string path) - => LocalTestResultsReader.LooksLikeTestResultFile(path); - - private static string SanitizeDirName(string value) - { - foreach (char invalidChar in Path.GetInvalidFileNameChars()) - { - value = value.Replace(invalidChar, '-'); - } - - return value; - } - } - - // ----------------------------------------------------------------- - // Shared retry helper - // ----------------------------------------------------------------- - - private static async Task RetryAsync(Func> action, CancellationToken cancellationToken) - { - Exception last = null; - for (int attempt = 0; attempt < 5; attempt++) - { - cancellationToken.ThrowIfCancellationRequested(); - try - { - return await action(); - } - catch (Exception ex) when (attempt < 4) - { - last = ex; - await Task.Delay(TimeSpan.FromSeconds(Math.Pow(2, attempt + 1)), cancellationToken); - } - } - - throw last ?? new InvalidOperationException("Retry failed without capturing an exception."); + (_azdo as IDisposable)?.Dispose(); + (_helix as IDisposable)?.Dispose(); } } - - public record WorkItemTestResults(string JobName, string WorkItemName, List TestResultFiles); } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Models/WorkItemTestResults.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Models/WorkItemTestResults.cs new file mode 100644 index 00000000000..d238f3d5f21 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Models/WorkItemTestResults.cs @@ -0,0 +1,9 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Collections.Generic; + +namespace Microsoft.DotNet.Helix.JobMonitor +{ + public record WorkItemTestResults(string JobName, string WorkItemName, List TestResultFiles); +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs new file mode 100644 index 00000000000..3be0ec3e907 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs @@ -0,0 +1,158 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.Model; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Microsoft.DotNet.Helix.JobMonitor +{ + internal sealed class AzureDevOpsService : IAzureDevOpsService, IDisposable + { + private const string MonitoredJobTagPrefix = "MonitoredJob:"; + private readonly JobMonitorOptions _options; + private readonly ILogger _logger; + private readonly HttpClient _azdoClient; + + public AzureDevOpsService(JobMonitorOptions options, ILogger logger) + { + _options = options; + _logger = logger; + _azdoClient = new HttpClient(); + string encodedToken = Convert.ToBase64String(Encoding.UTF8.GetBytes("unused:" + options.SystemAccessToken)); + _azdoClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Basic", encodedToken); + _azdoClient.DefaultRequestHeaders.UserAgent.ParseAdd("dotnet-helix-job-monitor"); + } + + public async Task> GetTimelineRecordsAsync(CancellationToken cancellationToken) + { + JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/build/builds/{_options.BuildId}/timeline?api-version=7.1-preview.2", cancellationToken: cancellationToken); + return data?["records"]?.ToObject() ?? []; + } + + public async Task> GetProcessedHelixJobNamesAsync(CancellationToken cancellationToken) + { + string buildUri = Uri.EscapeDataString($"vstfs:///Build/Build/{_options.BuildId}"); + JObject data = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?buildUri={buildUri}&api-version=7.1", cancellationToken: cancellationToken); + var processed = new HashSet(StringComparer.OrdinalIgnoreCase); + + foreach (JObject run in (data?["value"] as JArray ?? []).Cast()) + { + int? runId = run.Value("id"); + string state = run.Value("state"); + if (runId == null || !string.Equals(state, "Completed", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + string helixJobName = await GetMonitoredHelixJobNameAsync(runId.Value, cancellationToken); + if (!string.IsNullOrEmpty(helixJobName)) + { + processed.Add(helixJobName); + } + } + + return processed; + } + + private async Task GetMonitoredHelixJobNameAsync(int testRunId, CancellationToken cancellationToken) + { + JObject run = await SendAsync(HttpMethod.Get, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=7.1", cancellationToken: cancellationToken); + if (run?["tags"] is not JArray tags) + { + return null; + } + + foreach (JToken tag in tags) + { + string tagName = tag?.Value("name"); + if (!string.IsNullOrEmpty(tagName) && tagName.StartsWith(MonitoredJobTagPrefix, StringComparison.OrdinalIgnoreCase)) + { + return tagName.Substring(MonitoredJobTagPrefix.Length); + } + } + + return null; + } + + public async Task CreateTestRunAsync(string name, string helixJobName, CancellationToken cancellationToken) + { + JObject result = await SendAsync(HttpMethod.Post, + $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs?api-version=5.0", + new JObject + { + ["automated"] = true, + ["build"] = new JObject { ["id"] = _options.BuildId }, + ["name"] = name, + ["state"] = "InProgress", + ["tags"] = new JArray { new JObject { ["name"] = MonitoredJobTagPrefix + helixJobName } }, + }, + cancellationToken: cancellationToken); + return result?["id"]?.ToObject() ?? 0; + } + + public async Task CompleteTestRunAsync(int testRunId, CancellationToken cancellationToken) + { + await SendAsync(new HttpMethod("PATCH"), + $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=5.0", + new JObject { ["state"] = "Completed" }, + cancellationToken: cancellationToken); + } + + public async Task UploadTestResultsAsync(int testRunId, IReadOnlyList results, CancellationToken cancellationToken) + { + var publisher = new AzureDevOpsResultPublisher( + new AzureDevOpsReportingParameters( + new Uri(_options.CollectionUri, UriKind.Absolute), + _options.TeamProject, + testRunId.ToString(CultureInfo.InvariantCulture), + _options.SystemAccessToken), + _logger); + + bool allPassed = true; + foreach (WorkItemTestResults workItem in results) + { + _logger.LogInformation("Publishing test results for work item '{WorkItemName}' in job '{JobName}'...", workItem.WorkItemName, workItem.JobName); + allPassed &= await publisher.UploadTestResultsAsync(workItem.TestResultFiles, + new { HelixJobId = workItem.JobName, HelixWorkItemName = workItem.WorkItemName }, + cancellationToken); + } + + return allPassed; + } + + private async Task SendAsync(HttpMethod method, string requestUri, JToken body = null, CancellationToken cancellationToken = default) + { + return await RetryHelper.RetryAsync(async () => + { + using var request = new HttpRequestMessage(method, requestUri); + if (body != null) + { + request.Content = new StringContent(body.ToString(Formatting.None), Encoding.UTF8, "application/json"); + } + + using HttpResponseMessage response = await _azdoClient.SendAsync(request, cancellationToken); + string content = response.Content != null ? await response.Content.ReadAsStringAsync(cancellationToken) : null; + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException($"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}"); + } + + return string.IsNullOrWhiteSpace(content) ? [] : JObject.Parse(content); + }, cancellationToken); + } + + public void Dispose() => _azdoClient.Dispose(); + } +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs new file mode 100644 index 00000000000..ec00f52505f --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs @@ -0,0 +1,182 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.Immutable; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using Azure; +using Azure.Storage.Blobs; +using Microsoft.DotNet.Helix.Client; +using Microsoft.DotNet.Helix.Client.Models; +using Microsoft.DotNet.Helix.JobMonitor.Models; +using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json.Linq; + +namespace Microsoft.DotNet.Helix.JobMonitor +{ + internal sealed class HelixService : IHelixService + { + private readonly JobMonitorOptions _options; + private readonly ILogger _logger; + private readonly IHelixApi _helixApi; + + public HelixService(JobMonitorOptions options, ILogger logger) + { + _options = options; + _logger = logger; + _helixApi = string.IsNullOrEmpty(options.HelixAccessToken) + ? ApiFactory.GetAnonymous(options.HelixBaseUri) + : ApiFactory.GetAuthenticated(options.HelixBaseUri, options.HelixAccessToken); + } + + public async Task> GetJobsAsync(CancellationToken cancellationToken) + { + // Build the Helix source filter. For PR builds, use the PR merge ref. + // For CI builds without a PR, use the branch-based source. + string source = _options.PrNumber.HasValue + ? $"pr/public/{_options.Organization}/{_options.RepositoryName}/refs/pull/{_options.PrNumber}/merge" + : $"official/public/{_options.Organization}/{_options.RepositoryName}"; + + IImmutableList jobs = await RetryHelper.RetryAsync( + async () => await _helixApi.Job.ListAsync(source: source), + cancellationToken); + + return jobs + .Where(j => ((JObject)j.Properties).TryGetValue("BuildId", out JToken buildId) && buildId?.ToString() == _options.BuildId) + .Select(j => new HelixJobInfo( + j.Name, + j.Finished != null ? "finished" : "running", + GetTestRunNameFromJob(j))) + .ToList(); + } + + public async Task GetJobPassFailAsync(string jobName, CancellationToken cancellationToken) + { + IImmutableList workItems = await RetryHelper.RetryAsync( + () => _helixApi.WorkItem.ListAsync(jobName), + cancellationToken); + + var passed = new List(); + var failed = new List(); + + foreach (WorkItemSummary wi in workItems) + { + if (wi.ExitCode != 0 || !wi.State.Equals("Finished", StringComparison.OrdinalIgnoreCase)) + { + failed.Add(wi.Name); + } + else + { + passed.Add(wi.Name); + } + } + + return new HelixJobPassFail(passed, failed); + } + + public async Task> DownloadTestResultsAsync( + string jobName, + HelixJobPassFail passFail, + CancellationToken cancellationToken) + { + List downloadedFiles = []; + string outputDirectory = Path.Combine(_options.WorkingDirectory, SanitizeDirName(jobName)); + Directory.CreateDirectory(outputDirectory); + + JobResultsUri resultsUri = await RetryHelper.RetryAsync(() => _helixApi.Job.ResultsAsync(jobName), cancellationToken); + + IEnumerable workItemNames = passFail.PassedWorkItems + .Concat(passFail.FailedWorkItems) + .Distinct(StringComparer.OrdinalIgnoreCase); + + foreach (string workItemName in workItemNames) + { + IImmutableList availableFiles = await RetryHelper.RetryAsync( + () => _helixApi.WorkItem.ListFilesAsync(workItemName, jobName, false), + cancellationToken); + + availableFiles = [.. availableFiles.Where(f => LooksLikeTestResultFile(f.Name))]; + if (availableFiles.Count == 0) + { + continue; + } + + string workItemDirectory = Path.Combine(outputDirectory, SanitizeDirName(workItemName)); + Directory.CreateDirectory(workItemDirectory); + + List workItemFiles = []; + foreach (UploadedFile file in availableFiles) + { + string relativePath = file.Name.Replace('\\', Path.DirectorySeparatorChar).Replace('/', Path.DirectorySeparatorChar); + string destinationFile = Path.Combine(workItemDirectory, relativePath); + string directory = Path.GetDirectoryName(destinationFile); + if (!string.IsNullOrEmpty(directory)) + { + Directory.CreateDirectory(directory); + } + + try + { + BlobClient blobClient = CreateBlobClient(file.Link, resultsUri.ResultsUriRSAS); + await blobClient.DownloadToAsync(destinationFile, cancellationToken); + workItemFiles.Add(destinationFile); + } + catch (Exception ex) + { + _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItemName); + } + } + + downloadedFiles.Add(new WorkItemTestResults(jobName, workItemName, workItemFiles)); + } + + return downloadedFiles; + } + + private static string GetTestRunNameFromJob(JobSummary helixJob) + { + if (helixJob.Properties is JObject properties + && properties.TryGetValue("TestRunName", out JToken testRunName)) + { + string value = testRunName?.ToString(); + if (!string.IsNullOrEmpty(value)) + { + return value; + } + } + + return helixJob.Name; + } + + private static BlobClient CreateBlobClient(string fileLink, string resultsSas) + { + var options = new BlobClientOptions(); + options.Retry.NetworkTimeout = TimeSpan.FromMinutes(5); + if (string.IsNullOrEmpty(resultsSas)) + { + return new BlobClient(new Uri(fileLink), options); + } + + string strippedUri = fileLink.Contains('?') ? fileLink.Substring(0, fileLink.LastIndexOf('?', StringComparison.Ordinal)) : fileLink; + return new BlobClient(new Uri(strippedUri), new AzureSasCredential(resultsSas), options); + } + + private static bool LooksLikeTestResultFile(string path) + => LocalTestResultsReader.LooksLikeTestResultFile(path); + + private static string SanitizeDirName(string value) + { + foreach (char invalidChar in Path.GetInvalidFileNameChars()) + { + value = value.Replace(invalidChar, '-'); + } + + return value; + } + } +} From 770ed6679797a2057f41ff55594c7984dbcb45ff Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 27 Apr 2026 15:46:24 +0200 Subject: [PATCH 62/66] Log which jobs have not finished running --- .../JobMonitor/JobMonitorRunner.cs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 4e8b39f672d..0034cca30c5 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -67,7 +67,24 @@ private async Task RunCoreAsync(CancellationToken cancellationToken) { IReadOnlySet alreadyProcessed = await _azdo.GetProcessedHelixJobNamesAsync(cancellationToken); var processedHelixJobs = new HashSet(alreadyProcessed, StringComparer.OrdinalIgnoreCase); + IReadOnlyList latestAssociatedJobs = Array.Empty(); + try + { + return await RunLoopAsync(processedHelixJobs, latestJobs => latestAssociatedJobs = latestJobs, cancellationToken); + } + catch (OperationCanceledException) + { + ReportTimeout(latestAssociatedJobs, processedHelixJobs); + return 1; + } + } + + private async Task RunLoopAsync( + HashSet processedHelixJobs, + Action> reportLatestJobs, + CancellationToken cancellationToken) + { bool anyNonMonitorJobFailures = false; int failedHelixJobCount = 0; int processedHelixJobCount = 0; @@ -80,6 +97,7 @@ private async Task RunCoreAsync(CancellationToken cancellationToken) IReadOnlyList timelineRecords = await _azdo.GetTimelineRecordsAsync(cancellationToken); IReadOnlyList associatedJobsWithBuild = await _helix.GetJobsAsync(cancellationToken); + reportLatestJobs(associatedJobsWithBuild); // Filter jobs to completed ones belonging to this build IReadOnlyCollection completedJobs = @@ -192,5 +210,31 @@ public void Dispose() (_azdo as IDisposable)?.Dispose(); (_helix as IDisposable)?.Dispose(); } + + private void ReportTimeout( + IReadOnlyList latestAssociatedJobs, + HashSet processedHelixJobs) + { + var timeout = TimeSpan.FromMinutes(_options.MaximumWaitMinutes); + var unfinishedJobs = latestAssociatedJobs + .Where(j => !j.IsCompleted || !processedHelixJobs.Contains(j.JobName)) + .OrderBy(j => j.JobName, StringComparer.OrdinalIgnoreCase) + .ToList(); + + if (unfinishedJobs.Count == 0) + { + _logger.LogCritical("Helix Job Monitor timed out after {TimeoutMinutes} minute(s) ({Timeout}). No unfinished Helix jobs were tracked at the time of timeout.", + timeout.TotalMinutes, + timeout); + return; + } + + _logger.LogError( + "Helix Job Monitor timed out after {TimeoutMinutes} minute(s) ({Timeout}). {UnfinishedCount} Helix job(s) had not finished: {UnfinishedJobs}", + timeout.TotalMinutes, + timeout, + unfinishedJobs.Count, + string.Join(", ", unfinishedJobs.Select(j => $"{j.JobName} (status: {j.Status})"))); + } } } From d7b08bf81fc73b5f4a3df5fd1d1476203044d18e Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 27 Apr 2026 15:48:46 +0200 Subject: [PATCH 63/66] Fix the CI build --- .../Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj index 50f940b40a3..13de144da67 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj @@ -6,8 +6,4 @@ enable - - - - From 1926954a5eedbf44137f756fa9678efbfc04f5e1 Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 27 Apr 2026 16:07:15 +0200 Subject: [PATCH 64/66] Add the `monitorAllStages` flag --- .../core-templates/job/helix-job-monitor.yml | 9 ++++ .../stages/helix-job-monitor.yml | 8 +++ .../JobMonitor/HelixJobMonitorUtilities.cs | 49 +++++++++++++++++++ .../JobMonitor/JobMonitorOptions.cs | 12 +++++ .../JobMonitor/JobMonitorRunner.cs | 39 ++++++++++++--- .../JobMonitor/Models/HelixJobInfo.cs | 26 +++++++++- .../JobMonitor/Services/HelixService.cs | 18 ++++++- 7 files changed, 151 insertions(+), 10 deletions(-) diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 53a6ea1c2ce..f3c7ef28555 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -75,6 +75,13 @@ parameters: type: string default: '' +# When true (default), the monitor tracks Helix jobs and pipeline jobs across every stage of the +# build. When false, the monitor only tracks jobs that belong to the same Azure DevOps stage as +# the monitor job itself (the stage name is read from $(System.StageName) at runtime). +- name: monitorAllStages + type: boolean + default: true + # Optional dependency list for the generated job. - name: dependsOn type: object @@ -201,6 +208,8 @@ jobs: --max-wait-minutes '${{ parameters.timeoutInMinutes }}' --job-monitor-name '${{ parameters.jobMonitorName }}' --attempt '$(System.JobAttempt)' + --monitor-all-stages '${{ parameters.monitorAllStages }}' + --stage-name '$(System.StageName)' ) organization='${{ parameters.organization }}' diff --git a/eng/common/core-templates/stages/helix-job-monitor.yml b/eng/common/core-templates/stages/helix-job-monitor.yml index 0c3d68ce9fb..67ab95248bd 100644 --- a/eng/common/core-templates/stages/helix-job-monitor.yml +++ b/eng/common/core-templates/stages/helix-job-monitor.yml @@ -79,6 +79,13 @@ parameters: type: string default: '' +# When true (default), the monitor tracks Helix jobs and pipeline jobs across every stage of the +# build. When false, the monitor only tracks jobs that belong to the same Azure DevOps stage as +# the monitor job itself (i.e. this stage). +- name: monitorAllStages + type: boolean + default: true + # Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool # nupkg. When set, the artifact is downloaded and the directory containing the nupkg is added # as a NuGet source for the 'dotnet tool install' command. Primarily intended for the Arcade @@ -115,3 +122,4 @@ stages: organization: ${{ parameters.organization }} repository: ${{ parameters.repository }} prNumber: ${{ parameters.prNumber }} + monitorAllStages: ${{ parameters.monitorAllStages }} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs b/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs index f380f43d94d..a28e96673b0 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/HelixJobMonitorUtilities.cs @@ -39,6 +39,55 @@ public static bool HasFailedNonMonitorJobs(IEnumerable + /// Returns the subset of that belongs to the pipeline stage + /// named , including the Stage record itself and any + /// descendant records (Phases, Jobs, Tasks). When the named Stage is not present in the + /// timeline an empty list is returned. + /// + public static IReadOnlyList FilterRecordsToStage( + IEnumerable records, + string stageName) + { + if (string.IsNullOrEmpty(stageName)) + { + return (records ?? []).ToList(); + } + + var all = (records ?? []).ToList(); + var stageRoot = all.FirstOrDefault(r => + string.Equals(r.Type, "Stage", StringComparison.OrdinalIgnoreCase) + && string.Equals(r.Name, stageName, StringComparison.OrdinalIgnoreCase)); + + if (stageRoot == null) + { + return []; + } + + // Iteratively collect all descendants of the stage record by following ParentId. + var byParent = all + .Where(r => !string.IsNullOrEmpty(r.ParentId)) + .ToLookup(r => r.ParentId, StringComparer.OrdinalIgnoreCase); + + var result = new List { stageRoot }; + var queue = new Queue(); + queue.Enqueue(stageRoot.Id); + while (queue.Count > 0) + { + string parentId = queue.Dequeue(); + foreach (AzureDevOpsTimelineRecord child in byParent[parentId]) + { + result.Add(child); + if (!string.IsNullOrEmpty(child.Id)) + { + queue.Enqueue(child.Id); + } + } + } + + return result; + } + private static IEnumerable GetRelevantJobRecords(IEnumerable records, string jobMonitorName) { return (records ?? []) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs index c68befc59e0..346b2ffacd8 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorOptions.cs @@ -55,6 +55,12 @@ public sealed class JobMonitorOptions [Option("attempt", HelpText = "Azure DevOps attempt number for the current job.")] public int? Attempt { get; set; } + [Option("monitor-all-stages", HelpText = "When true (default) the monitor tracks Helix jobs and pipeline jobs across all stages of the build. When false the monitor only tracks jobs that belong to the same stage as the monitor itself (see --stage-name).", Default = true)] + public bool MonitorAllStages { get; set; } = true; + + [Option("stage-name", HelpText = "Name of the Azure DevOps pipeline stage the monitor is running in. Used to scope monitoring when --monitor-all-stages is false. Defaults to the SYSTEM_STAGENAME environment variable.")] + public string StageName { get; set; } + public static JobMonitorOptions Parse(string[] args) { JobMonitorOptions parsed = null; @@ -97,6 +103,7 @@ private void ApplyEnvironmentDefaults() WorkingDirectory ??= System.IO.Path.Combine(System.IO.Path.GetTempPath(), "helix-job-monitor", BuildId ?? "unknown"); PrNumber ??= GetEnvironmentInt("SYSTEM_PULLREQUEST_PULLREQUESTNUMBER"); Attempt ??= GetEnvironmentInt("SYSTEM_JOBATTEMPT"); + StageName ??= Environment.GetEnvironmentVariable("SYSTEM_STAGENAME"); } private void Validate() @@ -115,6 +122,11 @@ private void Validate() { throw new InvalidOperationException("Organization must be provided either by argument or pipeline environment."); } + + if (!MonitorAllStages && string.IsNullOrWhiteSpace(StageName)) + { + throw new InvalidOperationException("--stage-name (or the SYSTEM_STAGENAME environment variable) must be set when --monitor-all-stages is false."); + } } private static string RequireValue(string value, string argumentName, string environmentName) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index 0034cca30c5..1b657edff23 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -65,9 +65,18 @@ public async Task RunAsync() private async Task RunCoreAsync(CancellationToken cancellationToken) { + if (_options.MonitorAllStages || string.IsNullOrEmpty(_options.StageName)) + { + _logger.LogInformation("Monitoring Helix jobs for the pipeline"); + } + else + { + _logger.LogInformation("Monitoring Helix jobs sent from stage '{StageName}'", _options.StageName); + } + IReadOnlySet alreadyProcessed = await _azdo.GetProcessedHelixJobNamesAsync(cancellationToken); var processedHelixJobs = new HashSet(alreadyProcessed, StringComparer.OrdinalIgnoreCase); - IReadOnlyList latestAssociatedJobs = Array.Empty(); + IReadOnlyList latestAssociatedJobs = []; try { @@ -89,7 +98,7 @@ private async Task RunLoopAsync( int failedHelixJobCount = 0; int processedHelixJobCount = 0; int allHelixJobCount = 0; - int completedJobsCount = 0; + int completedJobsCount = -1; while (true) { @@ -97,6 +106,20 @@ private async Task RunLoopAsync( IReadOnlyList timelineRecords = await _azdo.GetTimelineRecordsAsync(cancellationToken); IReadOnlyList associatedJobsWithBuild = await _helix.GetJobsAsync(cancellationToken); + + // When the monitor is scoped to a single stage, drop timeline records and Helix jobs + // that belong to other stages so they don't gate completion or contribute failures. + if (!_options.MonitorAllStages && !string.IsNullOrEmpty(_options.StageName)) + { + timelineRecords = HelixJobMonitorUtilities.FilterRecordsToStage(timelineRecords, _options.StageName); + associatedJobsWithBuild = + [ + ..associatedJobsWithBuild.Where(j => + string.IsNullOrEmpty(j.StageName) + || string.Equals(j.StageName, _options.StageName, StringComparison.OrdinalIgnoreCase)) + ]; + } + reportLatestJobs(associatedJobsWithBuild); // Filter jobs to completed ones belonging to this build @@ -205,12 +228,6 @@ private async Task ProcessCompletedJobAsync( return failedWorkItemCount == 0; } - public void Dispose() - { - (_azdo as IDisposable)?.Dispose(); - (_helix as IDisposable)?.Dispose(); - } - private void ReportTimeout( IReadOnlyList latestAssociatedJobs, HashSet processedHelixJobs) @@ -236,5 +253,11 @@ private void ReportTimeout( unfinishedJobs.Count, string.Join(", ", unfinishedJobs.Select(j => $"{j.JobName} (status: {j.Status})"))); } + + public void Dispose() + { + (_azdo as IDisposable)?.Dispose(); + (_helix as IDisposable)?.Dispose(); + } } } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs index e0e71afcf52..32cbb9c8927 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Models/HelixJobInfo.cs @@ -18,13 +18,15 @@ public HelixJobInfo(JobSummary helixJob) JobName = helixJob.Name; Status = helixJob.Finished != null ? "finished" : "running"; TestRunName = GetTestRunNameFromJob(helixJob); + StageName = GetStringPropertyFromJob(helixJob, "System.StageName"); } - public HelixJobInfo(string jobName, string status, string testRunName = null) + public HelixJobInfo(string jobName, string status, string testRunName = null, string stageName = null) { JobName = jobName ?? throw new ArgumentNullException(nameof(jobName)); Status = status ?? throw new ArgumentNullException(nameof(status)); TestRunName = testRunName; + StageName = stageName; } public string JobName { get; } @@ -37,6 +39,13 @@ public HelixJobInfo(string jobName, string status, string testRunName = null) /// public string TestRunName { get; } + /// + /// Name of the Azure DevOps pipeline stage that submitted this Helix job, taken from + /// the "System.StageName" property stamped onto the job by SendHelixJob. May be + /// null if the property is not present. + /// + public string StageName { get; } + public bool IsCompleted => Status.Equals("finished", StringComparison.OrdinalIgnoreCase) || Status.Equals("failed", StringComparison.OrdinalIgnoreCase); @@ -64,5 +73,20 @@ private static string GetTestRunNameFromJob(JobSummary helixJob) return helixJob.Name; } + + private static string GetStringPropertyFromJob(JobSummary helixJob, string propertyName) + { + if (helixJob.Properties is JObject properties + && properties.TryGetValue(propertyName, out JToken token)) + { + string value = token?.ToString(); + if (!string.IsNullOrEmpty(value)) + { + return value; + } + } + + return null; + } } } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs index 996589302dc..4eb73e892d6 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs @@ -53,7 +53,8 @@ public async Task> GetJobsAsync(CancellationToken ca .Select(j => new HelixJobInfo( j.Name, j.Finished != null ? "finished" : "running", - GetTestRunNameFromJob(j))) + GetTestRunNameFromJob(j), + GetStringPropertyFromJob(j, "System.StageName"))) ]; } @@ -127,6 +128,21 @@ private static string GetTestRunNameFromJob(JobSummary helixJob) return helixJob.Name; } + private static string GetStringPropertyFromJob(JobSummary helixJob, string propertyName) + { + if (helixJob.Properties is JObject properties + && properties.TryGetValue(propertyName, out JToken token)) + { + string value = token?.ToString(); + if (!string.IsNullOrEmpty(value)) + { + return value; + } + } + + return null; + } + private static BlobClient CreateBlobClient(string fileLink, string resultsSas) { var options = new BlobClientOptions(); From 72509e51184229461ea8f14f6fe1a4538aa2b9ec Mon Sep 17 00:00:00 2001 From: Premek Vysoky Date: Mon, 27 Apr 2026 16:54:41 +0200 Subject: [PATCH 65/66] Shorten the tag name --- .../JobMonitor/Services/AzureDevOpsService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs index 900c5baf06e..ad2caede982 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs @@ -24,7 +24,7 @@ internal sealed class AzureDevOpsService : IAzureDevOpsService, IDisposable // particular Helix job. The full tag value is "MonitoredJob:{helixJobName}" and is // attached to the test run when it is created. This lets us look up which Helix jobs // we have already processed without encoding the Helix job name into the run name. - private const string MonitoredJobTagPrefix = "MonitoredHelixJob"; + private const string MonitoredJobTagPrefix = "MonitoredJob"; private readonly JobMonitorOptions _options; private readonly ILogger _logger; From eedbf7efcbad6de3d808e1cca40dc2f174fea97c Mon Sep 17 00:00:00 2001 From: "Matt Mitchell (.NET)" Date: Mon, 27 Apr 2026 16:20:23 -0700 Subject: [PATCH 66/66] Update tests for merged runner changes Adapt tests to the updated JobMonitorRunner from premun's branch: - Timeout now returns 1 (runner catches OperationCanceledException) - ProcessCompletedJobAsync always marks jobs processed - CompleteTestRunAsync in finally block (always called) - ListWorkItemsAsync + DownloadTestResultsAsync replaces GetJobPassFailAsync - Fix FakeHelixService to set ExitCode on WorkItemSummary Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- ...tNet.Helix.AzureDevOpsTestPublisher.csproj | 4 + .../Fakes/FakeHelixService.cs | 23 ++++-- .../JobMonitorRunnerTests.cs | 81 +++++-------------- 3 files changed, 41 insertions(+), 67 deletions(-) diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj index 13de144da67..50f940b40a3 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj @@ -6,4 +6,8 @@ enable + + + + diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs index 8df39d67c5a..b6a4a51103f 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs @@ -79,14 +79,21 @@ public Task> ListWorkItemsAsync( string jobName, CancellationToken _) { - return - Task.FromResult>( - [ - ..CurrentSnapshot.PassFailByJob[jobName].PassedWorkItems.Select( - w => new WorkItemSummary($"{jobName}/{w}", jobName, w, "Passed")), - ..CurrentSnapshot.PassFailByJob[jobName].FailedWorkItems.Select( - w => new WorkItemSummary($"{jobName}/{w}", jobName, w, "Failed")) - ]); + var items = new List(); + + foreach (string w in CurrentSnapshot.PassFailByJob[jobName].PassedWorkItems) + { + var wi = new WorkItemSummary($"{jobName}/{w}", jobName, w, "Finished") { ExitCode = 0 }; + items.Add(wi); + } + + foreach (string w in CurrentSnapshot.PassFailByJob[jobName].FailedWorkItems) + { + var wi = new WorkItemSummary($"{jobName}/{w}", jobName, w, "Finished") { ExitCode = 1 }; + items.Add(wi); + } + + return Task.FromResult>(items); } private sealed record HelixSnapshot( diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs index aa3676ff937..c152d40eb55 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs @@ -68,13 +68,11 @@ public async Task MultipleJobsAcrossMultiplePolls_ProcessesEachOnce() [Fact] public async Task StageCompletesWithNoHelixJobs_ExitZero_NoTestRuns() { - var (azdo, helix, runner, _) = CreateScenario( + var (azdo, _, runner, _) = CreateScenario( timelineSnapshots: [[PipelineJob("Build Linux", "completed", "succeeded"), MonitorJob()]], helixSnapshots: [(jobs: Array.Empty(), passFail: EmptyPassFail())]); - int exitCode = await runner.RunAsync(CancellationToken.None); - - Assert.Equal(0, exitCode); + Assert.Equal(0, await runner.RunAsync(CancellationToken.None)); Assert.Empty(azdo.CreatedTestRuns); Assert.Empty(azdo.UploadedJobNames); } @@ -90,9 +88,7 @@ public async Task PipelineJobFailsBeforeHelixSubmission_ExitOne_NoTestRuns() timelineSnapshots: [[PipelineJob("Build Linux", "completed", "failed"), MonitorJob()]], helixSnapshots: [(jobs: Array.Empty(), passFail: EmptyPassFail())]); - int exitCode = await runner.RunAsync(CancellationToken.None); - - Assert.Equal(1, exitCode); + Assert.Equal(1, await runner.RunAsync(CancellationToken.None)); Assert.Empty(azdo.CreatedTestRuns); } @@ -113,9 +109,7 @@ public async Task HelixJobFails_ExitOne_ResultsStillUploaded() timelineSnapshots: [[PipelineJob("Build Linux", "completed", "succeeded"), MonitorJob()]], helixSnapshots: [(jobs: [HelixJob("job-linux", "finished")], passFail: Dict(("job-linux", PassFail(failed: ["wi-1"]))))]); - int exitCode = await runner.RunAsync(CancellationToken.None); - - Assert.Equal(1, exitCode); + Assert.Equal(1, await runner.RunAsync(CancellationToken.None)); Assert.Equal(["job-linux"], azdo.UploadedJobNames); Assert.Single(azdo.CompletedTestRunIds); } @@ -131,17 +125,6 @@ public async Task AllHelixWorkItemsFail_ExitOne_ResultsUploaded() Assert.Equal(["job-linux"], azdo.UploadedJobNames); } - [Fact] - public async Task InfrastructureFailure_HelixJobFailedNoWorkItems_ExitOne() - { - var (azdo, _, runner, _) = CreateScenario( - timelineSnapshots: [[PipelineJob("Build Linux", "completed", "succeeded"), MonitorJob()]], - helixSnapshots: [(jobs: [HelixJob("job-linux", "failed")], passFail: Dict(("job-linux", PassFail())))]); - - Assert.Equal(1, await runner.RunAsync(CancellationToken.None)); - Assert.Equal(["job-linux"], azdo.UploadedJobNames); - } - [Fact] public async Task MultipleHelixJobsAllFail_ExitOne_AllResultsUploaded() { @@ -190,7 +173,7 @@ public async Task MonitorRetry_ProcessesReplacementDelta() var helix = new FakeHelixService(); ConfigureSnapshots(azdo, helix, [[PipelineJob("Build Linux (retry)", "completed", "succeeded"), MonitorJob()]], - [(jobs: [HelixJob("job-linux-attempt1", "failed"), HelixJob("job-linux-attempt2", "finished")], passFail: Dict(("job-linux-attempt1", PassFail(failed: ["wi-2"])), ("job-linux-attempt2", PassFail(passed: ["wi-2"]))))]); + [(jobs: [HelixJob("job-linux-attempt1", "finished"), HelixJob("job-linux-attempt2", "finished")], passFail: Dict(("job-linux-attempt2", PassFail(passed: ["wi-2"]))))]); var runner = CreateRunner(azdo, helix); Assert.Equal(0, await runner.RunAsync(CancellationToken.None)); @@ -235,26 +218,6 @@ public async Task MultipleRetries_SkipsAllPriorGenerations() Assert.Equal(["job-linux-attempt3", "job-win-attempt3"], azdo.UploadedJobNames); } - [Fact] - public async Task MonitorCrashAndRestart_ProcessesRemainingDelta() - { - var azdo = new FakeAzureDevOpsService(); - var helix = new FakeHelixService(); - ConfigureSnapshots(azdo, helix, - [[PipelineJob("Build Linux", "completed", "succeeded"), PipelineJob("Build Win", "completed", "succeeded"), MonitorJob()]], - [(jobs: [HelixJob("job-linux", "finished"), HelixJob("job-win", "finished")], passFail: Dict(("job-linux", PassFail(passed: ["linux-wi"])), ("job-win", PassFail(passed: ["win-wi"]))))]); - - helix.FailDownloadForJob("job-win"); - var runner1 = CreateRunner(azdo, helix); - Assert.Equal(1, await runner1.RunAsync(CancellationToken.None)); - Assert.Equal(["job-linux"], azdo.UploadedJobNames); - - helix.ClearDownloadFailures(); - var runner2 = CreateRunner(azdo, helix); - Assert.Equal(0, await runner2.RunAsync(CancellationToken.None)); - Assert.Equal(["job-linux", "job-win"], azdo.UploadedJobNames); - } - [Fact] public async Task RetryWithFailedSubsetResubmitted_OnlyNewJobProcessed() { @@ -262,7 +225,7 @@ public async Task RetryWithFailedSubsetResubmitted_OnlyNewJobProcessed() var helix = new FakeHelixService(); ConfigureSnapshots(azdo, helix, [[PipelineJob("Build Linux (retry)", "completed", "succeeded"), MonitorJob()]], - [(jobs: [HelixJob("job-linux-attempt1", "failed"), HelixJob("job-linux-attempt2", "finished")], passFail: Dict(("job-linux-attempt1", PassFail(failed: ["wi-2"])), ("job-linux-attempt2", PassFail(passed: ["wi-2"]))))]); + [(jobs: [HelixJob("job-linux-attempt1", "finished"), HelixJob("job-linux-attempt2", "finished")], passFail: Dict(("job-linux-attempt2", PassFail(passed: ["wi-2"]))))]); var runner = CreateRunner(azdo, helix); Assert.Equal(0, await runner.RunAsync(CancellationToken.None)); @@ -274,15 +237,19 @@ public async Task RetryWithFailedSubsetResubmitted_OnlyNewJobProcessed() // ----------------------------------------------------------------------- [Fact] - public async Task MonitorTimesOut_ThrowsOperationCanceledException() + public async Task MonitorTimesOut_ReturnsOne() { - var (_, _, runner, _) = CreateScenario( - timelineSnapshots: [[PipelineJob("Build Linux", "inProgress"), MonitorJob()]], - helixSnapshots: [(jobs: [HelixJob("job-linux", "running")], passFail: EmptyPassFail())]); + // The runner catches OperationCanceledException and returns 1. + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + ConfigureSnapshots(azdo, helix, + [[PipelineJob("Build Linux", "inProgress"), MonitorJob()]], + [(jobs: [HelixJob("job-linux", "running")], passFail: EmptyPassFail())]); using var cts = new CancellationTokenSource(); cts.Cancel(); - await Assert.ThrowsAsync(() => runner.RunAsync(cts.Token)); + var runner = CreateRunner(azdo, helix); + Assert.Equal(1, await runner.RunAsync(cts.Token)); } [Fact] @@ -297,8 +264,9 @@ public async Task AllPipelineJobsFailWhileHelixStillRunning_ExitsImmediately() } [Fact] - public async Task DownloadFailureMidStream_RetryReusesTestRun_NoDuplicate() + public async Task DownloadFailure_TestRunStillCompleted() { + // The runner always completes the test run (finally block), even on download failure. var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); ConfigureSnapshots(azdo, helix, @@ -306,17 +274,12 @@ public async Task DownloadFailureMidStream_RetryReusesTestRun_NoDuplicate() [(jobs: [HelixJob("job-linux", "finished"), HelixJob("job-win", "finished")], passFail: Dict(("job-linux", PassFail(passed: ["linux-wi"])), ("job-win", PassFail(passed: ["win-wi"]))))]); helix.FailDownloadForJob("job-win"); - var runner1 = CreateRunner(azdo, helix); - Assert.Equal(1, await runner1.RunAsync(CancellationToken.None)); - Assert.Equal(["job-linux"], azdo.UploadedJobNames); - - helix.ClearDownloadFailures(); - var runner2 = CreateRunner(azdo, helix); - Assert.Equal(0, await runner2.RunAsync(CancellationToken.None)); - Assert.Equal(["job-linux", "job-win"], azdo.UploadedJobNames); + var runner = CreateRunner(azdo, helix); + Assert.Equal(1, await runner.RunAsync(CancellationToken.None)); - // Key invariant: exactly 1 test run CREATED for job-win (second call reused in-progress one) - Assert.Equal(1, azdo.CreatedTestRuns.Count(n => n.Equals("job-win", StringComparison.OrdinalIgnoreCase))); + Assert.Equal(["job-linux"], azdo.UploadedJobNames); + // Both test runs completed (finally block ensures CompleteTestRunAsync is called) + Assert.Equal(2, azdo.CompletedTestRunIds.Count); } [Fact]