From 76f36a6418058b3c861a17b69e6523c5dd6f1592 Mon Sep 17 00:00:00 2001 From: Missy Messa Date: Tue, 7 Apr 2026 15:12:03 -0700 Subject: [PATCH 1/2] Replace external PublishSymbols task with SymbolUploadHelper-based task (WI 10148/10149) Replaces the Microsoft.SymbolUploader.Build.Task dependency in PublishToSymbolServers.proj with a new PublishSymbolsUsingSymbolUploadHelper MSBuild task that uses the proven internal SymbolUploadHelper infrastructure already used by Maestro promotion publishing. Key changes: - New task PublishSymbolsUsingSymbolUploadHelper in Build.Tasks.Feed that wraps SymbolUploadHelper - Supports both PAT auth (backward compat) and DefaultIdentityTokenCredential (Entra/MI) - Updated PublishToSymbolServers.proj to use the new task and reference Build.Tasks.Feed - SymbolServerTargets now use org names (microsoft, microsoftpublicsymbols) instead of URLs Migration path: When PATs are still configured, PAT auth is used automatically. When PATs are removed, DefaultIdentityTokenCredential provides Entra auth. --- .../SdkTasks/PublishToSymbolServers.proj | 34 +-- .../PublishSymbolsUsingSymbolUploadHelper.cs | 246 ++++++++++++++++++ 2 files changed, 263 insertions(+), 17 deletions(-) create mode 100644 src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishSymbolsUsingSymbolUploadHelper.cs diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishToSymbolServers.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishToSymbolServers.proj index a3e199b3ae5..6009efe2196 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishToSymbolServers.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishToSymbolServers.proj @@ -4,14 +4,15 @@ - + + $(DotNetSymbolServerTokenMsdl) - - + + $(DotNetSymbolServerTokenSymWeb) - - + - + diff --git a/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishSymbolsUsingSymbolUploadHelper.cs b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishSymbolsUsingSymbolUploadHelper.cs new file mode 100644 index 00000000000..6f3853633ac --- /dev/null +++ b/src/Microsoft.DotNet.Build.Tasks.Feed/src/PublishSymbolsUsingSymbolUploadHelper.cs @@ -0,0 +1,246 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Collections.Frozen; +using System.Linq; +using System.Threading.Tasks; +using Azure.Core; +using Microsoft.Build.Framework; +using Microsoft.DotNet.ArcadeAzureIntegration; +using Microsoft.DotNet.Internal.SymbolHelper; +using MsBuildUtils = Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks.Feed; + +/// +/// MSBuild task that publishes symbols to a symbol server using the internal SymbolUploadHelper +/// infrastructure. Supports both PAT-based auth (legacy) and Entra/managed identity auth. +/// +public class PublishSymbolsUsingSymbolUploadHelper : MsBuildUtils.Task +{ + /// + /// The Azure DevOps organization to publish symbols to (e.g., "microsoft" or "microsoftpublicsymbols"). + /// + [Required] + public string AzdoOrg { get; set; } + + /// + /// Optional personal access token. If provided, PAT-based auth is used. + /// If empty or not set, DefaultIdentityTokenCredential (Entra/MI) is used. + /// + public string PersonalAccessToken { get; set; } + + /// + /// Optional managed identity client ID for DefaultIdentityTokenCredential. + /// + public string ManagedIdentityClientId { get; set; } + + /// + /// Symbol packages (*.symbols.nupkg) to publish. + /// + public ITaskItem[] PackagesToPublish { get; set; } + + /// + /// Individual files (PDBs, DLLs, etc.) to publish. + /// + public ITaskItem[] FilesToPublish { get; set; } + + /// + /// Files to exclude from symbol packages during publishing. + /// + public ITaskItem[] PackageExcludeFiles { get; set; } + + /// + /// Number of days before the symbol request expires. Default is 3650. + /// + public int ExpirationInDays { get; set; } = 3650; + + /// + /// Whether to enable verbose logging from the symbol client. + /// + public bool VerboseLogging { get; set; } + + /// + /// Whether to perform a dry run without actually publishing. + /// + public bool DryRun { get; set; } + + /// + /// Whether to convert portable PDBs to Windows PDBs. + /// + public bool ConvertPortablePdbsToWindowsPdbs { get; set; } + + /// + /// Whether to treat PDB conversion issues as informational messages. + /// + public bool TreatPdbConversionIssuesAsInfo { get; set; } + + /// + /// Comma-separated list of PDB conversion diagnostic IDs to treat as warnings. + /// + public string PdbConversionTreatAsWarning { get; set; } + + /// + /// Whether to publish special CLR files (DAC, DBI, SOS) under diagnostic indexes. + /// + public bool PublishSpecialClrFiles { get; set; } + + /// + /// Directory containing loose PDB/DLL files to publish. + /// If set, files from this directory are added via AddDirectory instead of individual file items. + /// + public string PDBArtifactsDirectory { get; set; } + + public override bool Execute() + { + return ExecuteAsync().GetAwaiter().GetResult(); + } + + private async Task ExecuteAsync() + { + try + { + TokenCredential credential = CreateCredential(); + TaskTracer tracer = new(Log, verbose: VerboseLogging); + + IEnumerable pdbWarnings = ParsePdbConversionWarnings(); + FrozenSet exclusions = PackageExcludeFiles?.Select(i => i.ItemSpec).ToFrozenSet() + ?? FrozenSet.Empty; + + SymbolPublisherOptions options = new( + AzdoOrg, + credential, + packageFileExcludeList: exclusions, + convertPortablePdbs: ConvertPortablePdbsToWindowsPdbs, + treatPdbConversionIssuesAsInfo: TreatPdbConversionIssuesAsInfo, + pdbConversionTreatAsWarning: pdbWarnings, + dotnetInternalPublishSpecialClrFiles: PublishSpecialClrFiles, + verboseClient: VerboseLogging, + isDryRun: DryRun); + + SymbolUploadHelper helper = DryRun + ? SymbolUploadHelperFactory.GetSymbolHelperFromLocalTool(tracer, options, ".") + : await SymbolUploadHelperFactory.GetSymbolHelperWithDownloadAsync(tracer, options); + + string requestName = $"arcade-sdk/{AzdoOrg}/{Guid.NewGuid()}"; + Log.LogMessage(MessageImportance.High, "Creating symbol request '{0}' for org '{1}'", requestName, AzdoOrg); + + int result = await helper.CreateRequest(requestName); + if (result != 0) + { + Log.LogError("Failed to create symbol request '{0}'. Exit code: {1}", requestName, result); + return false; + } + + bool succeeded = false; + try + { + // Add loose files directory if specified + if (!string.IsNullOrEmpty(PDBArtifactsDirectory)) + { + Log.LogMessage(MessageImportance.High, "Adding directory '{0}' to symbol request", PDBArtifactsDirectory); + result = await helper.AddDirectory(requestName, PDBArtifactsDirectory); + if (result != 0) + { + Log.LogError("Failed to add directory to symbol request. Exit code: {0}", result); + return false; + } + } + + // Add individual file items if specified (and no directory was given) + if (FilesToPublish?.Length > 0 && string.IsNullOrEmpty(PDBArtifactsDirectory)) + { + // SymbolUploadHelper works with directories, so we need to group files + // by their parent directory and add each directory. + var directories = FilesToPublish + .Select(f => System.IO.Path.GetDirectoryName(f.ItemSpec)) + .Where(d => !string.IsNullOrEmpty(d)) + .Distinct(StringComparer.OrdinalIgnoreCase); + + foreach (string dir in directories) + { + Log.LogMessage(MessageImportance.Normal, "Adding file directory '{0}' to symbol request", dir); + result = await helper.AddDirectory(requestName, dir); + if (result != 0) + { + Log.LogError("Failed to add directory '{0}' to symbol request. Exit code: {1}", dir, result); + return false; + } + } + } + + // Add symbol packages + if (PackagesToPublish?.Length > 0) + { + IEnumerable packagePaths = PackagesToPublish.Select(p => p.ItemSpec); + Log.LogMessage(MessageImportance.High, "Adding {0} symbol package(s) to request", PackagesToPublish.Length); + + result = await helper.AddPackagesToRequest(requestName, packagePaths); + if (result != 0) + { + Log.LogError("Failed to add packages to symbol request. Exit code: {0}", result); + return false; + } + } + + Log.LogMessage(MessageImportance.High, "Finalizing symbol request with expiration of {0} days", ExpirationInDays); + result = await helper.FinalizeRequest(requestName, (uint)ExpirationInDays); + if (result != 0) + { + Log.LogError("Failed to finalize symbol request. Exit code: {0}", result); + return false; + } + + succeeded = true; + } + finally + { + if (!succeeded) + { + Log.LogMessage(MessageImportance.High, "Symbol publishing failed. Deleting request '{0}'.", requestName); + await helper.DeleteRequest(requestName); + } + } + + Log.LogMessage(MessageImportance.High, "Successfully published symbols to '{0}'", AzdoOrg); + return true; + } + catch (Exception ex) + { + Log.LogErrorFromException(ex, showStackTrace: true); + return false; + } + } + + private TokenCredential CreateCredential() + { + if (!string.IsNullOrEmpty(PersonalAccessToken)) + { + Log.LogMessage(MessageImportance.Normal, "Using PAT-based authentication for symbol publishing"); + return new PATCredential(PersonalAccessToken); + } + + Log.LogMessage(MessageImportance.Normal, "Using Entra/managed identity authentication for symbol publishing"); + var options = new DefaultIdentityTokenCredentialOptions(); + if (!string.IsNullOrEmpty(ManagedIdentityClientId)) + { + options.ManagedIdentityClientId = ManagedIdentityClientId; + } + return new DefaultIdentityTokenCredential(options); + } + + private IEnumerable ParsePdbConversionWarnings() + { + if (string.IsNullOrEmpty(PdbConversionTreatAsWarning)) + { + return []; + } + + return PdbConversionTreatAsWarning + .Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Where(s => int.TryParse(s, out _)) + .Select(int.Parse); + } +} From c236c1961d68a0af795a792597733e06aa3183bc Mon Sep 17 00:00:00 2001 From: Missy Messa Date: Mon, 20 Apr 2026 16:03:34 -0700 Subject: [PATCH 2/2] Address review: add staging+promotion flow, document dry run behavior - Symbols now stage in dnceng org first via SymbolUploadHelper, then promote to target servers via SymbolPromotionHelper.RegisterAndPublishRequest() - Replaced per-target-org batching with single task invocation using PublishToInternalServer/PublishToPublicServer visibility flags - Renamed AzdoOrg to StagingAzdoOrg (default: dnceng) to clarify purpose - Added SymbolRequestProject parameter for promotion service registration - Promotion credential uses DefaultIdentityTokenCredential (Entra/MI) - Documented dry run behavior difference: no per-file indexability validation (now handled by 1ES symbol publishing tool) - Updated PR pipeline to use new DotNetSymbolServerTokenStaging param --- azure-pipelines-pr.yml | 3 +- .../SdkTasks/PublishToSymbolServers.proj | 33 ++--- .../PublishSymbolsUsingSymbolUploadHelper.cs | 117 +++++++++++++++--- 3 files changed, 117 insertions(+), 36 deletions(-) diff --git a/azure-pipelines-pr.yml b/azure-pipelines-pr.yml index 7ca86bef5c4..9400911b858 100644 --- a/azure-pipelines-pr.yml +++ b/azure-pipelines-pr.yml @@ -165,8 +165,7 @@ stages: inputs: filePath: eng\common\sdk-task.ps1 arguments: -task PublishToSymbolServers /p:DryRun="true" -restore -msbuildEngine dotnet - /p:DotNetSymbolServerTokenMsdl=DryRunPTA - /p:DotNetSymbolServerTokenSymWeb=DryRunPTA + /p:DotNetSymbolServerTokenStaging=DryRunPAT /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' /p:SymbolPublishingExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SymbolPublishingExclusionsFile.txt' diff --git a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishToSymbolServers.proj b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishToSymbolServers.proj index 6009efe2196..9f6e1cf7447 100644 --- a/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishToSymbolServers.proj +++ b/src/Microsoft.DotNet.Arcade.Sdk/tools/SdkTasks/PublishToSymbolServers.proj @@ -5,14 +5,23 @@ This MSBuild file is intended to be used as the body of the default publishing release pipeline. The release pipeline will use this file to publish symbols to MSDL and SymWeb using the internal SymbolUploadHelper. + + Symbols are staged in the dnceng Azure DevOps org and then promoted to the + target symbol servers (internal/public) via the SymbolRequest promotion service. + + Note: Unlike the previous PublishSymbols task (Microsoft.SymbolUploader.Build.Task), + this task does not validate that individual files are indexable during dry runs. + The 1ES symbol publishing tool now manages indexability validation. Dry runs will + still decompose packages and exercise the request lifecycle but skip per-file + indexability checks. Parameters: - PDBArtifactsDirectory : Full path to directory containing PDB files to be published and (optionally) DLLs matching these PDBs - BlobBasePath : Full path containing *.symbols.nupkg packages to be published. - - DotNetSymbolServerTokenMsdl : (Optional) PAT to access MSDL. If empty, Entra/managed identity auth is used. - - DotNetSymbolServerTokenSymWeb : (Optional) PAT to access SymWeb. If empty, Entra/managed identity auth is used. + - DotNetSymbolServerTokenStaging : (Optional) PAT to access the staging org (dnceng). If empty, Entra/managed identity auth is used. - ManagedIdentityClientId : (Optional) Client ID of the managed identity to use for Entra auth. + - SymbolRequestProject : (Required for non-dry-run) Project name for registration with the SymbolRequest promotion service. - DotNetSymbolExpirationInDays : Expiration days for published packages. Default is 3650. - SymbolPublishingExclusionsFile : Path to file containing exclusion list to be used by Symbol Uploader. - PublishSpecialClrFiles : If true, publish the DAC, DBI and SOS using the coreclr index. If false, don't do any special indexing. @@ -82,22 +91,18 @@ Text="Going to ignore this file -> %(PackageExcludeFiles.Identity)." /> - - - $(DotNetSymbolServerTokenMsdl) - - - - $(DotNetSymbolServerTokenSymWeb) - + - + -/// MSBuild task that publishes symbols to a symbol server using the internal SymbolUploadHelper -/// infrastructure. Supports both PAT-based auth (legacy) and Entra/managed identity auth. +/// MSBuild task that publishes symbols to symbol servers using the internal SymbolUploadHelper +/// infrastructure. Symbols are first staged in a temporary Azure DevOps org (default: "dnceng"), +/// then promoted to the target symbol servers (internal/public) via the SymbolRequest service. +/// Supports both PAT-based auth (legacy) and Entra/managed identity auth. +/// +/// Note: Unlike the old PublishSymbols task (Microsoft.SymbolUploader.Build.Task), this task +/// does not validate that individual files are indexable during dry runs. The 1ES symbol +/// publishing tool now manages that validation. Dry runs will still decompose packages +/// and exercise the request lifecycle but skip per-file indexability checks. /// public class PublishSymbolsUsingSymbolUploadHelper : MsBuildUtils.Task { /// - /// The Azure DevOps organization to publish symbols to (e.g., "microsoft" or "microsoftpublicsymbols"). + /// The temporary/staging Azure DevOps organization where symbols are uploaded before promotion. + /// Default is "dnceng". Symbols cannot be published directly to the target orgs + /// (microsoftpublicsymbols/microsoft); they must be staged here first. /// - [Required] - public string AzdoOrg { get; set; } + public string StagingAzdoOrg { get; set; } = "dnceng"; /// - /// Optional personal access token. If provided, PAT-based auth is used. + /// The project name used for the SymbolRequest promotion service registration. + /// Required for non-dry-run publishing. + /// + public string SymbolRequestProject { get; set; } + + /// + /// Optional personal access token for staging org auth. If provided, PAT-based auth is used. /// If empty or not set, DefaultIdentityTokenCredential (Entra/MI) is used. /// public string PersonalAccessToken { get; set; } /// /// Optional managed identity client ID for DefaultIdentityTokenCredential. + /// Used for both staging upload and promotion service auth. /// public string ManagedIdentityClientId { get; set; } + /// + /// Whether to publish symbols to the internal symbol server (SymWeb/microsoft org). + /// Default is true. + /// + public bool PublishToInternalServer { get; set; } = true; + + /// + /// Whether to publish symbols to the public symbol server (MSDL/microsoftpublicsymbols org). + /// Default is true. + /// + public bool PublishToPublicServer { get; set; } = true; + /// /// Symbol packages (*.symbols.nupkg) to publish. /// @@ -102,7 +129,14 @@ private async Task ExecuteAsync() { try { - TokenCredential credential = CreateCredential(); + if (!PublishToInternalServer && !PublishToPublicServer) + { + Log.LogMessage(MessageImportance.High, "Neither internal nor public symbol server publishing is requested. Skipping."); + return true; + } + + TokenCredential stagingCredential = CreateStagingCredential(); + TokenCredential promotionCredential = CreatePromotionCredential(); TaskTracer tracer = new(Log, verbose: VerboseLogging); IEnumerable pdbWarnings = ParsePdbConversionWarnings(); @@ -110,8 +144,8 @@ private async Task ExecuteAsync() ?? FrozenSet.Empty; SymbolPublisherOptions options = new( - AzdoOrg, - credential, + StagingAzdoOrg, + stagingCredential, packageFileExcludeList: exclusions, convertPortablePdbs: ConvertPortablePdbsToWindowsPdbs, treatPdbConversionIssuesAsInfo: TreatPdbConversionIssuesAsInfo, @@ -124,8 +158,8 @@ private async Task ExecuteAsync() ? SymbolUploadHelperFactory.GetSymbolHelperFromLocalTool(tracer, options, ".") : await SymbolUploadHelperFactory.GetSymbolHelperWithDownloadAsync(tracer, options); - string requestName = $"arcade-sdk/{AzdoOrg}/{Guid.NewGuid()}"; - Log.LogMessage(MessageImportance.High, "Creating symbol request '{0}' for org '{1}'", requestName, AzdoOrg); + string requestName = $"arcade-sdk/{StagingAzdoOrg}/{Guid.NewGuid()}"; + Log.LogMessage(MessageImportance.High, "Creating symbol request '{0}' in staging org '{1}'", requestName, StagingAzdoOrg); int result = await helper.CreateRequest(requestName); if (result != 0) @@ -134,7 +168,7 @@ private async Task ExecuteAsync() return false; } - bool succeeded = false; + bool uploadSucceeded = false; try { // Add loose files directory if specified @@ -185,7 +219,7 @@ private async Task ExecuteAsync() } } - Log.LogMessage(MessageImportance.High, "Finalizing symbol request with expiration of {0} days", ExpirationInDays); + Log.LogMessage(MessageImportance.High, "Finalizing symbol request in staging org with expiration of {0} days", ExpirationInDays); result = await helper.FinalizeRequest(requestName, (uint)ExpirationInDays); if (result != 0) { @@ -193,18 +227,50 @@ private async Task ExecuteAsync() return false; } - succeeded = true; + uploadSucceeded = true; } finally { - if (!succeeded) + if (!uploadSucceeded) { - Log.LogMessage(MessageImportance.High, "Symbol publishing failed. Deleting request '{0}'.", requestName); + Log.LogMessage(MessageImportance.High, "Symbol upload failed. Deleting request '{0}'.", requestName); await helper.DeleteRequest(requestName); } } - Log.LogMessage(MessageImportance.High, "Successfully published symbols to '{0}'", AzdoOrg); + // Promote the finalized request to the target symbol servers + SymbolPromotionHelper.Visibility visibility = PublishToPublicServer + ? SymbolPromotionHelper.Visibility.Public + : SymbolPromotionHelper.Visibility.Internal; + + if (DryRun) + { + Log.LogMessage(MessageImportance.High, + "Dry run: would register request '{0}' to project '{1}' with visibility '{2}' for {3} days.", + requestName, SymbolRequestProject, visibility, ExpirationInDays); + } + else + { + if (string.IsNullOrEmpty(SymbolRequestProject)) + { + Log.LogError("SymbolRequestProject is required for non-dry-run symbol publishing promotion."); + return false; + } + + Log.LogMessage(MessageImportance.High, + "Promoting symbol request '{0}' to project '{1}' with visibility '{2}'", + requestName, SymbolRequestProject, visibility); + + if (!await SymbolPromotionHelper.RegisterAndPublishRequest( + tracer, promotionCredential, SymbolPromotionHelper.Environment.Prod, + SymbolRequestProject, requestName, (uint)ExpirationInDays, visibility)) + { + Log.LogError("Failed to register and promote symbol request to the target symbol servers."); + return false; + } + } + + Log.LogMessage(MessageImportance.High, "Successfully published and promoted symbols (visibility: {0})", visibility); return true; } catch (Exception ex) @@ -214,15 +280,26 @@ private async Task ExecuteAsync() } } - private TokenCredential CreateCredential() + private TokenCredential CreateStagingCredential() { if (!string.IsNullOrEmpty(PersonalAccessToken)) { - Log.LogMessage(MessageImportance.Normal, "Using PAT-based authentication for symbol publishing"); + Log.LogMessage(MessageImportance.Normal, "Using PAT-based authentication for staging symbol upload"); return new PATCredential(PersonalAccessToken); } - Log.LogMessage(MessageImportance.Normal, "Using Entra/managed identity authentication for symbol publishing"); + Log.LogMessage(MessageImportance.Normal, "Using Entra/managed identity authentication for staging symbol upload"); + return CreateDefaultCredential(); + } + + private TokenCredential CreatePromotionCredential() + { + Log.LogMessage(MessageImportance.Normal, "Using Entra/managed identity authentication for symbol promotion"); + return CreateDefaultCredential(); + } + + private DefaultIdentityTokenCredential CreateDefaultCredential() + { var options = new DefaultIdentityTokenCredentialOptions(); if (!string.IsNullOrEmpty(ManagedIdentityClientId)) {