NOT ready: Add dynamic runtimesettings (#8341)

Build/18.0/packages.config

@@ -1,7 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
 <!-- 3rd party packages-->
-  <package id="Python" version="3.8.9" />
   <package id="Newtonsoft.Json" version="13.0.1" targetFramework="net45" />
 
 <!-- packages from private sources. See nuget.config for more info -->
@@ -83,4 +82,5 @@
   <package id="Microsoft.VisualStudio.RpcContracts" version = "17.15.25-pre" /> <!-- 17.15.26-pre -->
   <package id="Microsoft.DiaSymReader.Pdb2Pdb" version = "1.1.0-beta2-21181-01" />
   <package id="Microsoft.TestPlatform.ObjectModel" version = "17.14.1" /> <!-- 17.4.0-preview-20221003-03 -> 17.14.1 -->
+  <package id="Microsoft.DevDiv.Validation.TestPlatform.Settings.Tasks" version = "1.0.1108" />
 </packages>

Build/PreBuild.ps1

@@ -208,7 +208,7 @@ try {
     if (-not $interactive) {
         $arglist += "-NonInteractive"
     }
-    $nuget = Get-Command nuget.exe -EA 0
+    $nuget = Get-Command .\nuget.exe -EA 0
     if (-not $nuget) {
         $nuget = Get-Command .\nuget.exe
     }

Build/azuredevops/test/uitest-integration-stage.yml

@@ -0,0 +1,157 @@
+parameters:
+  - name: BinariesDirectory
+    type: string
+    default: '$(Drop)/binaries'
+  - name: CleanupProcesses
+    type: string
+    default: 'devenv, vshub, microsoft.vshub.server.httphost, python, ipy, microsoft.pythontools.analyzer, vstest.executionengine.x86, vstest.discoveryengine.x86'
+  - name: Drop
+    type: string
+    default: '$(System.ArtifactsDirectory)/$(DefinitionName)'
+  - name: TestFilterCriteria
+    type: string
+    default: 'Priority=0'
+  - name: UITestNames
+    type: string
+    default: ''
+
+steps:
+  - task: ExtractFiles@1
+    displayName: Extract files
+    condition: succeeded()
+    inputs:
+      archiveFilePatterns: '${{ parameters.Drop }}/TestData.zip'
+      destinationFolder: '${{ parameters.Drop }}/TestData'
+      cleanDestinationFolder: false
+      overwriteExistingFiles: false
+
+  - task: PowerShell@2
+    displayName: Cleanup
+    condition: succeeded()
+    continueOnError: true
+    timeoutInMinutes: 5
+    inputs:
+      targetType: inline
+      script: |
+        $names = @(
+          '${{ parameters.CleanupProcesses }}'
+            -split ',' |
+            ForEach-Object { $_.Trim() } |
+            Where-Object { -not [string]::IsNullOrWhiteSpace($_) }
+        )
+        $running = Get-Process -Name $names -ErrorAction SilentlyContinue
+        while ($running) {
+          Write-Host 'Killing:'
+          $running
+          try {
+            $running | Stop-Process -Force
+          } catch {
+          }
+          Start-Sleep -Seconds 3
+          $running = Get-Process -Name $names -ErrorAction SilentlyContinue
+        }
+      errorActionPreference: stop
+      failOnStderr: true
+
+  - task: CopyFiles@2
+    displayName: Copy TestData into binaries
+    condition: succeeded()
+    continueOnError: true
+    inputs:
+      SourceFolder: '${{ parameters.Drop }}/TestData'
+      Contents: '**'
+      TargetFolder: '${{ parameters.BinariesDirectory }}/TestData'
+      CleanTargetFolder: 'true'
+      OverWrite: 'false'
+      flattenFolders: 'false'
+      preserveTimestamp: 'false'
+      retryCount: '0'
+
+  - task: CopyFiles@2
+    displayName: Copy runsettings to binary folder
+    condition: succeeded()
+    inputs:
+      SourceFolder: '${{ parameters.BinariesDirectory }}/TestData'
+      Contents: 'ptvs.runsettings'
+      TargetFolder: '${{ parameters.BinariesDirectory }}'
+      CleanTargetFolder: 'false'
+      OverWrite: 'false'
+      flattenFolders: 'false'
+      preserveTimestamp: 'false'
+      retryCount: '0'
+
+  - task: PowerShell@2
+    displayName: Skip Verification
+    condition: succeeded()
+    inputs:
+      targetType: inline
+      script: |
+        reg import "${{ parameters.BinariesDirectory }}\TestData\EnableSkipVerification.reg"
+      errorActionPreference: stop
+      failOnStderr: false
+
+  - task: VSTest@2
+    displayName: '[MultiConfiguration] Run UI Test ${{ parameters.UITestNames }}'
+    condition: succeeded()
+    continueOnError: true
+    inputs:
+      testSelector: testAssemblies
+      testAssemblyVer2: '${{ parameters.UITestNames }}UITestsRunner.dll'
+      tcmTestRun: '$(test.RunId)'
+      searchFolder: '${{ parameters.BinariesDirectory }}'
+      resultsFolder: '$(Agent.TempDirectory)/TestResults'
+      testFiltercriteria: '${{ parameters.TestFilterCriteria }}'
+      runOnlyImpactedTests: false
+      runAllTestsAfterXBuilds: 50
+      uiTests: true
+      vstestLocationMethod: version
+      vsTestVersion: latest
+      runSettingsFile: '${{ parameters.Drop }}\\ptvs.runsettings'
+      overrideTestrunParameters: '-TargetFrameworkVersion .NETFramework,Version=v4.7.2'
+      runInParallel: false
+      runTestsInIsolation: false
+      codeCoverageEnabled: true
+      distributionBatchType: basedOnExecutionTime
+      batchingBasedOnAgentsOption: autoBatchSize
+      customBatchSizeValue: 10
+      batchingBasedOnExecutionTimeOption: customTimeBatchSize
+      customRunTimePerBatchValue: 300
+      dontDistribute: false
+      testRunTitle: '${{ parameters.UITestNames }} UI Tests'
+      publishRunAttachments: true
+      failOnMinTestsNotRun: false
+      minimumExpectedTests: 1
+      diagnosticsEnabled: true
+      collectDumpOn: onAbortOnly
+      rerunFailedTests: true
+      rerunType: basedOnTestFailurePercentage
+      rerunFailedThreshold: 50
+      rerunFailedTestCasesMaxLimit: 5
+      rerunMaxAttempts: 2
+
+  - task: PowerShell@2
+    displayName: Cleanup
+    condition: succeeded()
+    timeoutInMinutes: 5
+    inputs:
+      targetType: inline
+      script: |
+        $names = @(
+          '${{ parameters.CleanupProcesses }}'
+            -split ',' |
+            ForEach-Object { $_.Trim() } |
+            Where-Object { -not [string]::IsNullOrWhiteSpace($_) }
+        )
+        $running = Get-Process -Name $names -ErrorAction SilentlyContinue
+        while ($running) {
+          Write-Host 'Killing:'
+          $running
+          try {
+            $running | Stop-Process -Force
+          } catch {
+          }
+          Start-Sleep -Seconds 3
+          $running = Get-Process -Name $names -ErrorAction SilentlyContinue
+        }
+      errorActionPreference: stop
+      failOnStderr: true

Build/debugpy-version.txt

@@ -1 +1 @@
-1.8.17
+1.8.18

Build/nuget.config

@@ -3,9 +3,10 @@
     <clear />
     <!-- This is the old private azure feed, leaving it commented here for reference -->
     <!--<add key="PTVS_Packages" value="https://devdiv.pkgs.visualstudio.com/DevDiv/_packaging/PTVS_Packages/nuget/v3/index.json" />-->
-    
+
     <!-- This is the public feed that we should use for both npm and nuget -->
-    <add key="msft_consumption" value="https://pkgs.dev.azure.com/azure-public/vside/_packaging/msft_consumption/nuget/v3/index.json" />
+    <add key="msft_consumption" value="https://pkgs.dev.azure.com/azure-public/vside/_packaging/msft_consumption/nuget/v3/index.json" /> 
+	  <!-- <add key="VS" value="https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json" /> -->
     <!-- <add key="vs-impl" value="https://pkgs.dev.azure.com/azure-public/vside/_packaging/vs-impl%40Local/nuget/v3/index.json" /> -->
     <!-- <add key="vssdk" value="https://pkgs.dev.azure.com/azure-public/vside/_packaging/vssdk/nuget/v3/index.json" /> -->
 

Build/nuget.testdata.config

@@ -0,0 +1,24 @@
+<configuration>
+  <packageSources>
+    <clear />
+    <!-- This is the old private azure feed, leaving it commented here for reference -->
+    <!--<add key="PTVS_Packages" value="https://devdiv.pkgs.visualstudio.com/DevDiv/_packaging/PTVS_Packages/nuget/v3/index.json" />-->
+
+    <!-- This is the public feed that we should use for both npm and nuget -->
+    <!-- <add key="msft_consumption" value="https://pkgs.dev.azure.com/azure-public/vside/_packaging/msft_consumption/nuget/v3/index.json" /> -->
+	<add key="VS" value="https://devdiv.pkgs.visualstudio.com/_packaging/VS/nuget/v3/index.json" /> 
+    <!-- <add key="vs-impl" value="https://pkgs.dev.azure.com/azure-public/vside/_packaging/vs-impl%40Local/nuget/v3/index.json" /> -->
+    <!-- <add key="vssdk" value="https://pkgs.dev.azure.com/azure-public/vside/_packaging/vssdk/nuget/v3/index.json" /> -->
+
+  </packageSources>
+  <trustedSigners>
+    <repository name="nuget" serviceIndex="https://api.nuget.org/v3/index.json">
+      <owners>Microsoft;xunit;manuel.roemer;sharwell;jamesnk;aarnott;MarcoRossignoli;Thecentury;AndreyAkinshin;0xd4d;ericnewton76;jkeech;steve.dower</owners>
+      <certificate fingerprint="0e5f38f57dc1bcc806d8494f4f90fbcedd988b46760709cbeec6f4219aa6157d" hashAlgorithm="SHA256" allowUntrustedRoot="false" />
+    </repository>
+    <author name="Microsoft">
+      <certificate fingerprint="aa12da22a49bce7d5c1ae64cc1f3d892f150da76140f210abd2cbffca2c18a27" hashAlgorithm="SHA256" allowUntrustedRoot="false" />
+      <certificate fingerprint="3f9001ea83c560d712c24cf213c3d312cb3bff51ee89435d3430bd06b5d0eece" hashAlgorithm="SHA256" allowUntrustedRoot="false" />
+    </author>
+  </trustedSigners>
+</configuration>

Build/packages.testdata.config

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="Microsoft.DevDiv.Validation.TestPlatform.Settings.Tasks" version="1.0.1108" />
+</packages>

Build/templates/build.yml

@@ -2,6 +2,9 @@ parameters:
 - name: ptvsPackageVersion
   type: string
   default: $(Build.BuildNumber)
+- name: skipSigning
+  type: boolean
+  default: false
 
 steps:
 
@@ -57,4 +60,5 @@ steps:
                           /p:BuildNumber=${{ parameters.ptvsPackageVersion }} 
                           /p:BUILD_BINARIESDIRECTORY=$(Build.BinariesDirectory)
                           /p:BUILD_STAGINGDIRECTORY=$(Build.StagingDirectory) 
+                          /p:SkipSigning=${{ parameters.skipSigning }}
                           /bl:$(Build.SourcesDirectory)\logs\BuildInstaller.binlog'

Build/templates/publish_test_data.yml

@@ -1,5 +1,43 @@
 steps:
 
+  - task: NuGetToolInstaller@1
+    displayName: Install NuGet
+
+  - task: NuGetCommand@2
+    displayName: Restore runsettings build tasks
+    inputs:
+      command: restore
+      feedsToUse: config
+      restoreSolution: '$(Build.SourcesDirectory)/Build/packages.testdata.config'
+      restoreDirectory: '$(Build.SourcesDirectory)/packages'
+      nugetConfigPath: '$(Build.SourcesDirectory)/Build/nuget.testdata.config'
+
+  # Generate a runsettings file that points at the correct Test Store drop.
+  - task: MSBuild@1
+    displayName: 'Build PTVS.runsettingsproj'
+    inputs:
+      solution: '$(Build.SourcesDirectory)/Python/Tests/PTVS.runsettingsproj'
+      msbuildArguments: >
+        /p:PackagesPath=$(Build.SourcesDirectory)\packages
+        /p:TestStoreDropName=$(TestStoreDropName)
+
+  - task: PowerShell@2
+    displayName: 'Update default.runsettings from generated ptvs.runsettings'
+    inputs:
+      targetType: 'inline'
+      script: |
+        $ErrorActionPreference = 'Stop'
+
+        $source = "$(Build.SourcesDirectory)\Python\Tests\bin\RunSettings\ptvs.runsettings"
+        $dest = "$(Build.SourcesDirectory)\Python\Tests\TestData\ptvs.runsettings"
+
+        if (-not (Test-Path -LiteralPath $source)) {
+          throw "Expected generated runsettings file not found: $source"
+        }
+
+        Copy-Item -LiteralPath $source -Destination $dest -Force
+
+
   # Compress just the test data
   # Used in this release task to run tests: https://devdiv.visualstudio.com/DevDiv/_taskgroup/a7b18468-aab5-4be7-b718-02f85b0828c4
   - task: ArchiveFiles@2
@@ -20,6 +58,13 @@ steps:
       Contents: 'TestData.zip'
       TargetFolder: '$(Build.ArtifactStagingDirectory)/TestDataFolder'
 
+  - task: CopyFiles@2
+    displayName: 'Copy ptvs.runsettings to subfolder'
+    inputs:
+      SourceFolder: '$(Build.SourcesDirectory)/Python/Tests/TestData'
+      Contents: 'ptvs.runsettings'
+      TargetFolder: '$(Build.ArtifactStagingDirectory)/TestDataFolder'
+
 
 # The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish build artifact: TestData' in the templateContext section.
 # The Task 'PublishBuildArtifacts@1' has been converted to an output named 'Publish build artifact: RunSettings' in the templateContext section.

Python/PythonTools.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
-# Visual Studio Version 17
-VisualStudioVersion = 17.0.31211.46
+# Visual Studio Version 18
+VisualStudioVersion = 18.3.11227.31 main
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Product", "Product", "{C0C7AD6F-5E39-4D3F-A6B6-FD30713B0464}"
 EndProject
@@ -16,6 +16,9 @@ EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestUtilities.UI", "..\Common\Tests\Utilities.UI\TestUtilities.UI.csproj", "{E8150EBC-6B62-40BF-BF91-1DC60149B530}"
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{3D21C7AC-6F9D-448C-8356-44EAFB2D14FC}"
+	ProjectSection(SolutionItems) = preProject
+		Tests\PTVS.runsettingsproj = Tests\PTVS.runsettingsproj
+	EndProjectSection
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Attacher", "Product\Attacher\Attacher.csproj", "{666A22BB-0CB5-4C08-A20F-E17450DA6573}"
 EndProject