[WIP][DO NOT MERGE] Add the Helix Job Monitor

Arcade.slnx

@@ -5,7 +5,9 @@
     <Platform Name="x86" />
   </Configurations>
   <Folder Name="/Microsoft.DotNet.Helix/">
+    <Project Path="src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Microsoft.DotNet.Helix.AzureDevOpsTestPublisher.csproj" />
     <Project Path="src/Microsoft.DotNet.Helix/Client/CSharp/Microsoft.DotNet.Helix.Client.csproj" />
+    <Project Path="src/Microsoft.DotNet.Helix/JobMonitor/Microsoft.DotNet.Helix.JobMonitor.csproj" />
     <Project Path="src/Microsoft.DotNet.Helix/JobSender/Microsoft.DotNet.Helix.JobSender.csproj" />
     <Project Path="src/Microsoft.DotNet.Helix/Sdk/Microsoft.DotNet.Helix.Sdk.csproj" />
   </Folder>

Directory.Packages.props

@@ -56,6 +56,7 @@
     <PackageVersion Include="Microsoft.Extensions.Http" Version="$(MicrosoftExtensionsHttpVersion)" />
     <PackageVersion Include="Microsoft.Extensions.FileProviders.Abstractions" Version="$(MicrosoftExtensionsFileProvidersAbstractionsVersion)" />
     <PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="$(MicrosoftExtensionsFileSystemGlobbingVersion)" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="$(MicrosoftExtensionsLoggingAbstractionsVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
     <PackageVersion Include="System.Composition" Version="$(SystemCompositionVersion)" />
     <PackageVersion Include="System.IO.Packaging" Version="$(SystemIOPackagingVersion)" />

azure-pipelines-pr.yml

@@ -64,8 +64,7 @@ stages:
       - job: Windows_NT
         timeoutInMinutes: 90
         pool:
-          name: $(DncEngPublicBuildPool)
-          demands: ImageOverride -equals windows.vs2026.amd64.open
+          vmImage: windows-latest # TODO: Testing only, revert
         strategy:
           matrix:
             Build_Release:
@@ -225,6 +224,15 @@ stages:
             SYSTEM_ACCESSTOKEN: $(System.AccessToken)
             HelixAccessToken: ''
 
+  # Helix Job Monitor runs as a job within the Test stage so it can monitor
+  # Helix work items submitted by the other jobs in this stage.
+  - template: /eng/common/core-templates/job/helix-job-monitor.yml
+    parameters:
+      timeoutInMinutes: 60 # TODO: Increase this
+      # Install from the nupkg produced by the Build stage's Windows_NT Release job.
+      toolNupkgArtifactName: Artifacts_Windows_NT_Release
+      toolNupkgArtifactSubPath: packages/Release/NonShipping
+
 - stage: Test_XHarness
   displayName: Test XHarness SDK
   dependsOn: build

eng/Version.Details.props

@@ -34,6 +34,7 @@ This file should be imported by eng/Versions.props
     <MicrosoftExtensionsFileProvidersAbstractionsPackageVersion>10.0.3</MicrosoftExtensionsFileProvidersAbstractionsPackageVersion>
     <MicrosoftExtensionsFileSystemGlobbingPackageVersion>10.0.3</MicrosoftExtensionsFileSystemGlobbingPackageVersion>
     <MicrosoftExtensionsHttpPackageVersion>10.0.3</MicrosoftExtensionsHttpPackageVersion>
+    <MicrosoftExtensionsLoggingAbstractionsPackageVersion>10.0.3</MicrosoftExtensionsLoggingAbstractionsPackageVersion>
     <MicrosoftExtensionsLoggingConsolePackageVersion>10.0.3</MicrosoftExtensionsLoggingConsolePackageVersion>
     <SystemCompositionPackageVersion>10.0.3</SystemCompositionPackageVersion>
     <SystemIOPackagingPackageVersion>10.0.3</SystemIOPackagingPackageVersion>
@@ -82,6 +83,7 @@ This file should be imported by eng/Versions.props
     <MicrosoftExtensionsFileProvidersAbstractionsVersion>$(MicrosoftExtensionsFileProvidersAbstractionsPackageVersion)</MicrosoftExtensionsFileProvidersAbstractionsVersion>
     <MicrosoftExtensionsFileSystemGlobbingVersion>$(MicrosoftExtensionsFileSystemGlobbingPackageVersion)</MicrosoftExtensionsFileSystemGlobbingVersion>
     <MicrosoftExtensionsHttpVersion>$(MicrosoftExtensionsHttpPackageVersion)</MicrosoftExtensionsHttpVersion>
+    <MicrosoftExtensionsLoggingAbstractionsVersion>$(MicrosoftExtensionsLoggingAbstractionsPackageVersion)</MicrosoftExtensionsLoggingAbstractionsVersion>
     <MicrosoftExtensionsLoggingConsoleVersion>$(MicrosoftExtensionsLoggingConsolePackageVersion)</MicrosoftExtensionsLoggingConsoleVersion>
     <SystemCompositionVersion>$(SystemCompositionPackageVersion)</SystemCompositionVersion>
     <SystemIOPackagingVersion>$(SystemIOPackagingPackageVersion)</SystemIOPackagingVersion>

eng/Version.Details.xml

@@ -90,6 +90,10 @@
     </Dependency>
     <!-- Necessary for source-build. This allows the live version of the dependency
          to flow in and eliminates related prebuilts. -->
+    <Dependency Name="Microsoft.Extensions.Logging.Abstractions" Version="10.0.3">
+      <Uri>https://github.com/dotnet/runtime</Uri>
+      <Sha>dc5fd7a8dce8309e4add8fd4bd5d8718f221b15a</Sha>
+    </Dependency>
     <Dependency Name="Microsoft.Extensions.Logging.Console" Version="10.0.3">
       <Uri>https://github.com/dotnet/runtime</Uri>
       <Sha>dc5fd7a8dce8309e4add8fd4bd5d8718f221b15a</Sha>

eng/common/core-templates/job/helix-job-monitor.yml

@@ -0,0 +1,254 @@
+parameters:
+# Azure DevOps job identifier.
+- name: jobName
+  type: string
+  default: HelixJobMonitor
+
+# Pool override. When empty the template selects a default azurelinux pool based on the team project.
+- name: pool
+  type: object
+  default: {}
+
+# NuGet package id of the Helix job monitor tool.
+- name: toolPackageId
+  type: string
+  default: Microsoft.DotNet.Helix.JobMonitor
+
+# Console command exposed by the installed tool package.
+- name: toolCommand
+  type: string
+  default: dotnet-helix-job-monitor
+
+# Optional explicit tool version. Only honored when 'toolNupkgArtifactName' is set; in the
+# default code path the version is taken from the consuming repo's .config/dotnet-tools.json.
+- name: toolVersion
+  type: string
+  default: ''
+
+# Optional NuGet feed used as an additional source when installing the tool. Only honored
+# when 'toolNupkgArtifactName' is set; in the default code path the tool is restored from
+# the consuming repo's .config/dotnet-tools.json manifest and no extra feeds are consulted.
+- name: toolSource
+  type: string
+  default: ''
+
+# Base URI for the Helix service (--helix-base-uri).
+- name: helixBaseUri
+  type: string
+  default: https://helix.dot.net/
+
+# Helix API access token forwarded to the tool via the HELIX_ACCESSTOKEN environment variable.
+- name: helixAccessToken
+  type: string
+  default: ''
+
+# Polling interval in seconds (--polling-interval-seconds).
+- name: pollingIntervalSeconds
+  type: number
+  default: 30
+
+# Maximum run time of the monitor job in minutes. Also used for --max-wait-minutes.
+- name: timeoutInMinutes
+  type: number
+  default: 360
+
+# Display name reported by the tool to Azure DevOps (--job-monitor-name).
+- name: jobMonitorName
+  type: string
+  default: Helix Job Monitor
+
+# Owner segment of the source repository (e.g. 'dotnet' for 'dotnet/runtime') passed via --organization.
+# Defaults to the owner segment of BUILD_REPOSITORY_NAME when empty.
+- name: organization
+  type: string
+  default: ''
+
+# Name of the source repository (e.g. 'runtime' for 'dotnet/runtime') passed via --repository.
+# Defaults to the repo segment of BUILD_REPOSITORY_NAME when empty.
+- name: repository
+  type: string
+  default: ''
+
+# Pull request number being built (--pr-number). Defaults to SYSTEM_PULLREQUEST_PULLREQUESTNUMBER
+# when empty.
+- name: prNumber
+  type: string
+  default: ''
+
+# When true (default), the monitor tracks Helix jobs and pipeline jobs across every stage of the
+# build. When false, the monitor only tracks jobs that belong to the same Azure DevOps stage as
+# the monitor job itself (the stage name is read from $(System.StageName) at runtime).
+- name: monitorAllStages
+  type: boolean
+  default: true
+
+# Optional dependency list for the generated job.
+- name: dependsOn
+  type: object
+  default: []
+
+# Optional condition for the generated job.
+- name: condition
+  type: string
+  default: ''
+
+# Advanced: optional pipeline artifact (produced earlier in this run) that contains the tool
+# nupkg. When set, the artifact is downloaded and the tool is installed from the nupkg into
+# a local tool-path; this bypasses the repo's .config/dotnet-tools.json manifest and is
+# primarily intended for the Arcade repository itself, where the Helix job monitor tool is
+# built in the same pipeline that runs this template.
+#
+# When this parameter is empty (the default), the consuming repository must declare the tool
+# in its .config/dotnet-tools.json manifest (alongside other local .NET tools); the template
+# will check out the repo and run 'dotnet tool restore' to install the version pinned there.
+- name: toolNupkgArtifactName
+  type: string
+  default: ''
+
+# Advanced: sub-path within the downloaded artifact where the tool nupkg is located. Defaults
+# to the standard Arcade non-shipping packages location for a Release build (relative to the
+# pipeline artifact root, which is itself the build's 'artifacts' directory).
+- name: toolNupkgArtifactSubPath
+  type: string
+  default: 'packages/Release/NonShipping'
+
+jobs:
+- job: ${{ parameters.jobName }}
+  displayName: Monitor Helix Jobs
+  timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
+  ${{ if ne(length(parameters.dependsOn), 0) }}:
+    dependsOn: ${{ parameters.dependsOn }}
+  ${{ if ne(parameters.condition, '') }}:
+    condition: ${{ parameters.condition }}
+  pool:
+    ${{ if eq(variables['System.TeamProject'], 'public') }}:
+      name: $(DncEngPublicBuildPool)
+      demands: ImageOverride -equals build.azurelinux.3.amd64.open
+    ${{ else }}:
+      name: $(DncEngInternalBuildPool)
+      demands: ImageOverride -equals build.azurelinux.3.amd64
+  steps:
+  - checkout: self
+    fetchDepth: 1
+
+  - ${{ if ne(parameters.toolNupkgArtifactName, '') }}:
+    - task: DownloadPipelineArtifact@2
+      displayName: Download Helix Job Monitor artifact
+      inputs:
+        buildType: current
+        artifactName: ${{ parameters.toolNupkgArtifactName }}
+        itemPattern: '${{ parameters.toolNupkgArtifactSubPath }}/${{ parameters.toolPackageId }}.*.nupkg'
+        targetPath: $(Agent.TempDirectory)/helix-job-monitor-nupkg
+
+    - bash: |
+        set -euo pipefail
+
+        toolPath="$AGENT_TEMPDIRECTORY/helix-job-monitor-tool"
+        mkdir -p "$toolPath"
+
+        packageId='${{ parameters.toolPackageId }}'
+        toolVersion='${{ parameters.toolVersion }}'
+        nupkgArtifactSubPath='${{ parameters.toolNupkgArtifactSubPath }}'
+        nupkgDir="$AGENT_TEMPDIRECTORY/helix-job-monitor-nupkg/$nupkgArtifactSubPath"
+
+        if [ ! -d "$nupkgDir" ]; then
+          echo "Expected nupkg directory '$nupkgDir' was not produced by the artifact download." >&2
+          exit 1
+        fi
+
+        nupkg=$(find "$nupkgDir" -maxdepth 1 -type f -name "$packageId.*.nupkg" | head -n 1)
+        if [ -z "$nupkg" ]; then
+          echo "No '$packageId.*.nupkg' found in '$nupkgDir'." >&2
+          exit 1
+        fi
+
+        # Derive the version from the nupkg filename so the local package is selected
+        # deterministically instead of resolving against any other configured feed.
+        nupkgBase=$(basename "$nupkg" .nupkg)
+        derivedVersion="${nupkgBase#${packageId}.}"
+        if [ -z "$toolVersion" ]; then
+          toolVersion="$derivedVersion"
+        fi
+
+        echo "Using locally built '$packageId' version '$toolVersion' from '$nupkgDir'."
+
+        # Create a minimal NuGet.config that only references the local nupkg directory.
+        # This avoids conflicts with the repo's package source mapping which blocks --add-source.
+        toolNugetConfig="$AGENT_TEMPDIRECTORY/helix-job-monitor-nuget.config"
+        printf '<?xml version="1.0" encoding="utf-8"?>\n<configuration>\n  <packageSources>\n    <clear />\n    <add key="local-tool" value="%s" />\n  </packageSources>\n</configuration>\n' "$nupkgDir" > "$toolNugetConfig"
+
+        pushd "$(Build.SourcesDirectory)" > /dev/null
+        ./eng/common/dotnet.sh tool install \
+          --tool-path "$toolPath" "$packageId" \
+          --version "$toolVersion" \
+          --configfile "$toolNugetConfig"
+
+        # Locate the tool DLL so the run step can invoke it via ./eng/common/dotnet.sh exec.
+        toolDll=$(find "$toolPath/.store" -path '*/tools/*/any/*.deps.json' -type f | head -n 1)
+        toolDll="${toolDll%.deps.json}.dll"
+        if [ ! -f "$toolDll" ]; then
+          echo "Could not find tool DLL in '$toolPath/.store'." >&2
+          exit 1
+        fi
+
+        echo "Tool DLL: $toolDll"
+        echo "##vso[task.setvariable variable=HelixJobMonitorDll]$toolDll"
+      displayName: Install Helix Job Monitor
+
+  - ${{ else }}:
+    - bash: ./eng/common/dotnet.sh tool restore
+      displayName: Restore Helix Job Monitor
+
+  - bash: |
+      set -euo pipefail
+
+      toolArgs=(
+        --helix-base-uri           '${{ parameters.helixBaseUri }}'
+        --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}'
+        --max-wait-minutes         '${{ parameters.timeoutInMinutes }}'
+        --job-monitor-name         '${{ parameters.jobMonitorName }}'
+        --attempt                  '$(System.JobAttempt)'
+        --monitor-all-stages       '${{ parameters.monitorAllStages }}'
+        --stage-name               '$(System.StageName)'
+      )
+
+      organization='${{ parameters.organization }}'
+      repository='${{ parameters.repository }}'
+      prNumber='${{ parameters.prNumber }}'
+
+      # Fall back to Azure DevOps-provided environment variables when the caller did not
+      # supply organization / repository / pr-number explicitly. BUILD_REPOSITORY_NAME is
+      # typically 'owner/repo' for GitHub-backed builds.
+      if [ -z "$organization" ] || [ -z "$repository" ]; then
+        buildRepoName="${BUILD_REPOSITORY_NAME:-}"
+        if [ -n "$buildRepoName" ] && [[ "$buildRepoName" == */* ]]; then
+          repoOwner="${buildRepoName%%/*}"
+          repoName="${buildRepoName#*/}"
+          if [ -z "$organization" ]; then organization="$repoOwner"; fi
+          if [ -z "$repository" ];   then repository="$repoName";   fi
+        fi
+      fi
+
+      if [ -z "$prNumber" ]; then
+        prNumber="${SYSTEM_PULLREQUEST_PULLREQUESTNUMBER:-}"
+      fi
+
+      if [ -n "$organization" ]; then toolArgs+=( --organization "$organization" ); fi
+      if [ -n "$repository" ];   then toolArgs+=( --repository   "$repository" );   fi
+      if [ -n "$prNumber" ];     then toolArgs+=( --pr-number    "$prNumber" );     fi
+
+      if [ -n '${{ parameters.toolNupkgArtifactName }}' ]; then
+        # Tool was installed from a local nupkg; run the DLL via the repo-local dotnet.
+        export DOTNET_ROOT="$(Build.SourcesDirectory)/.dotnet"
+        ./eng/common/dotnet.sh exec "$(HelixJobMonitorDll)" "${toolArgs[@]}"
+      else
+        # Tool was restored from the local .config/dotnet-tools.json manifest; invoke it
+        # through the manifest from the repo root.
+        pushd "$BUILD_SOURCESDIRECTORY" > /dev/null
+        trap 'popd > /dev/null' EXIT
+        ./eng/common/dotnet.sh tool run '${{ parameters.toolCommand }}' -- "${toolArgs[@]}"
+      fi
+    displayName: Monitor Helix Jobs
+    env:
+      SYSTEM_ACCESSTOKEN: $(System.AccessToken)
+      HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }}