diff --git a/azure-pipelines-PR.yml b/azure-pipelines-PR.yml index bf59708bdb1..bcc6926cd7e 100644 --- a/azure-pipelines-PR.yml +++ b/azure-pipelines-PR.yml @@ -14,7 +14,7 @@ trigger: - docs/* - .vscode/* - .devcontainer/* - - tests/scripts/ + - tests/scripts/* - attributions.md - CODE_OF_CONDUCT.md - DEVGUIDE.md @@ -256,8 +256,7 @@ stages: ArtifactName: 'Windows Release WindowsLangVersionPreview process dumps' ArtifactType: Container parallel: true - - - job: WindowsNoRealsig_testCoreclr + - job: WindowsNoRealsig_Build pool: # The PR build definition sets this variable: # WindowsMachineQueueName=Windows.vs2022.amd64.open @@ -270,13 +269,19 @@ stages: - checkout: self clean: true - - script: eng\CIBuild.cmd -compressallmetadata -buildnorealsig -testCoreclr -configuration Release + - script: eng\CIBuild.cmd -compressallmetadata -buildnorealsig -configuration Release env: DOTNET_DbgEnableMiniDump: 1 DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true displayName: Build + + - task: PublishPipelineArtifact@1 + displayName: 'Publish Pipeline Artifact' + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts' + artifact: WindowsNoRealsig_Build - task: PublishBuildArtifacts@1 displayName: Publish Build BinLog @@ -293,11 +298,12 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows Release WindowsNoRealsig_testCoreclr process dumps' + ArtifactName: 'Windows Release WindowsNoRealsig_Build process dumps' ArtifactType: Container parallel: true - - job: WindowsNoRealsig_testDesktop + - job: WindowsNoRealsig_testCoreclr + dependsOn: WindowsNoRealsig_Build pool: # The PR build definition sets this variable: # WindowsMachineQueueName=Windows.vs2022.amd64.open @@ -309,31 +315,93 @@ stages: steps: - checkout: self clean: true + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Build Artifact' + inputs: + artifact: WindowsNoRealsig_Build + path: '$(Build.SourcesDirectory)/artifacts' + + - script: | + echo "Listing downloaded pipeline artifact" + dir "$(Build.SourcesDirectory)/artifacts" + displayName: 'List Pipeline Workspace' - - script: eng\CIBuild.cmd -compressallmetadata -buildnorealsig -testDesktop -configuration Release + - pwsh: eng\Build.ps1 -ci -restore -testCoreclr -configuration Release env: DOTNET_DbgEnableMiniDump: 1 DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true displayName: Build + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'XUnit' + testRunTitle: WindowsNoRealsig_testCoreclr + mergeTestResults: true + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' + continueOnError: true - task: PublishBuildArtifacts@1 - displayName: Publish Build BinLog - condition: always() + displayName: Publish Dumps + condition: failed() continueOnError: true inputs: - PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows Release build binlogs' + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' + ArtifactName: 'Windows Release WindowsNoRealsig_testCoreclr process dumps' ArtifactType: Container parallel: true + + - job: WindowsNoRealsig_testDesktop + dependsOn: WindowsNoRealsig_Build + pool: + # The PR build definition sets this variable: + # WindowsMachineQueueName=Windows.vs2022.amd64.open + # and there is an alternate build definition that sets this to a queue that is always scouting the + # next preview of Visual Studio. + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 120 + strategy: + parallel: 4 + steps: + - checkout: self + clean: true + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Build Artifact' + inputs: + artifact: WindowsNoRealsig_Build + path: '$(Build.SourcesDirectory)/artifacts' + + - pwsh: eng\Build.ps1 -ci -restore -configuration Release -testDesktop -testBatch $(System.JobPositionInPhase) + env: + DOTNET_DbgEnableMiniDump: 1 + DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp + NativeToolsOnMachine: true + displayName: Build + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'XUnit' + testRunTitle: WindowsNoRealsig_testDesktop batch $(System.JobPositionInPhase) + mergeTestResults: true + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' + continueOnError: true + - task: PublishBuildArtifacts@1 displayName: Publish Dumps condition: failed() continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' - ArtifactName: 'Windows Release WindowsNoRealsig_testDesktop process dumps' + ArtifactName: 'Windows Release WindowsNoRealsig_testDesktop process dumps $(System.JobPositionInPhase)' ArtifactType: Container parallel: true @@ -413,11 +481,8 @@ stages: ArtifactType: Container parallel: true - # Windows With Compressed Metadata - - job: WindowsCompressedMetadata + - job: WindowsCompressedMetadata_vs variables: - - name: XUNIT_LOGS - value: $(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration) - name: __VSNeverShowWhatsNew value: 1 pool: @@ -429,83 +494,199 @@ stages: demands: ImageOverride -equals $(WindowsMachineQueueName) timeoutInMinutes: 120 strategy: - maxParallel: 5 + maxParallel: 2 matrix: - desktop_release: - _configuration: Release - _testKind: testDesktop - coreclr_release: - _configuration: Release - _testKind: testCoreclr fsharpqa_release: _configuration: Release _testKind: testFSharpQA vs_release: _configuration: Release _testKind: testVs - transparent_compiler_release: - _configuration: Release - _testKind: testCoreclr - - ${{ if eq(variables['Build.Reason'], 'Flaky, disabled, was PullRequest') }}: - inttests_release: - _configuration: Release - _testKind: testIntegration + # inttests_release: + # _configuration: Release + # _testKind: testIntegration steps: - checkout: self - clean: true - + clean: true - powershell: eng\SetupVSHive.ps1 displayName: Setup VS Hive - condition: or(eq(variables['_testKind'], 'testVs'), eq(variables['_testKind'], 'testIntegration')) - - # yes, this is miserable, but - https://github.com/dotnet/arcade/issues/13239 - - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) + - script: eng\CIBuild.cmd -compressallmetadata -configuration Release -$(_testKind) env: DOTNET_DbgEnableMiniDump: 1 DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. - DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\$(_configuration)\$(Build.BuildId)-%e-%p-%t.dmp + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true - displayName: Build / Test - condition: and( ne(variables['_testKind'], 'testIntegration'), ne(variables['System.JobName'], 'transparent_compiler_release') ) + displayName: Build + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'XUnit' + testRunTitle: WindowsCompressedMetadata $(_testKind) + mergeTestResults: true + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)' + continueOnError: true + condition: ne(variables['_testKind'], 'testFSharpQA') + - task: PublishBuildArtifacts@1 + displayName: Publish Tests BinLog + condition: always() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/$(_configuration)\Build.VisualFSharp.sln.binlog' + ArtifactName: 'Windows $(_configuration) $(_testKind) test binlogs' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\$(_configuration)' + ArtifactName: 'Windows $(_configuration) $(_testKind) process dumps' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration)' + ArtifactName: 'Windows $(_configuration) $(_testKind) test logs' + publishLocation: Container + continueOnError: true + condition: always() + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents Windows $(_testKind)' + publishLocation: Container + continueOnError: true + condition: failed() + + # Windows With Compressed Metadata + - job: WindowsCompressedMetadata_Build + variables: + - name: __VSNeverShowWhatsNew + value: 1 + pool: + # The PR build definition sets this variable: + # WindowsMachineQueueName=Windows.vs2022.amd64.open + # and there is an alternate build definition that sets this to a queue that is always scouting the + # next preview of Visual Studio. + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 120 + + steps: + - checkout: self + clean: true - - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) + - script: eng\CIBuild.cmd -compressallmetadata -configuration Release env: DOTNET_DbgEnableMiniDump: 1 DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. - DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\$(_configuration)\$(Build.BuildId)-%e-%p-%t.dmp - TEST_TRANSPARENT_COMPILER: 1 + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true - displayName: Build / Test Transparent Compiler - condition: and( eq(variables['System.JobName'], 'transparent_compiler_release'), ne(variables['_testKind'], 'testIntegration') ) + displayName: Build + - task: PublishPipelineArtifact@1 + displayName: 'Publish Pipeline Artifact' + inputs: + targetPath: '$(Build.SourcesDirectory)/artifacts' + artifact: 'WindowsCompressedMetadata_Build' - - script: eng\CIBuild.cmd -compressallmetadata -configuration $(_configuration) -$(_testKind) + - task: PublishBuildArtifacts@1 + displayName: Publish Tests BinLog + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/Release\Build.VisualFSharp.sln.binlog' + ArtifactName: 'WindowsCompressedMetadata test binlogs' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' + ArtifactName: 'WindowsCompressedMetadata process dumps' + ArtifactType: Container + parallel: true + - script: dotnet build $(Build.SourcesDirectory)/eng/DumpPackageRoot/DumpPackageRoot.csproj + displayName: Dump NuGet cache contents + condition: failed() + - task: PublishBuildArtifacts@1 + displayName: Publish NuGet cache contents + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' + ArtifactName: 'NuGetPackageContents WindowsCompressedMetadata' + publishLocation: Container + continueOnError: true + condition: failed() + + - job: WindowsCompressedMetadata_Test + dependsOn: WindowsCompressedMetadata_Build + variables: + - name: XUNIT_LOGS + value: $(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration) + - name: __VSNeverShowWhatsNew + value: 1 + pool: + # The PR build definition sets this variable: + # WindowsMachineQueueName=Windows.vs2022.amd64.open + # and there is an alternate build definition that sets this to a queue that is always scouting the + # next preview of Visual Studio. + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 120 + strategy: + maxParallel: 4 + matrix: + coreclr_release: + _configuration: Release + _testKind: testCoreclr + transparentCompilerPart: "" + transparent_compiler_release: + _configuration: Release + _testKind: testCoreclr + TEST_TRANSPARENT_COMPILER: 1 #Pipeline variables map to env vars. + transparentCompilerPart: " TransparentCompiler" + steps: + - checkout: self + clean: true + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Build Artifact' + inputs: + artifact: WindowsCompressedMetadata_Build + path: '$(Build.SourcesDirectory)/artifacts' + + - pwsh: ./eng/Build.ps1 -ci -restore -configuration $(_configuration) -$(_testKind) + workingDirectory: $(Build.SourcesDirectory) env: DOTNET_DbgEnableMiniDump: 1 DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. - DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\$(_configuration)\$(Build.BuildId)-%e-%p-%t.dmp + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp NativeToolsOnMachine: true - displayName: Build / Integration Test - continueOnError: true - condition: eq(variables['_testKind'], 'testIntegration') + displayName: Test - task: PublishTestResults@2 displayName: Publish Test Results inputs: testResultsFormat: 'XUnit' - testRunTitle: WindowsCompressedMetadata $(_testKind) + testRunTitle: 'WindowsCompressedMetadata $(_testKind)$(transparentCompilerPart)' mergeTestResults: true testResultsFiles: '*.xml' searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_configuration)' continueOnError: true - condition: ne(variables['_testKind'], 'testFSharpQA') - task: PublishBuildArtifacts@1 displayName: Publish Tests BinLog condition: always() continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log/$(_configuration)\Build.VisualFSharp.sln.binlog' - ArtifactName: 'Windows $(_configuration) $(_testKind) test binlogs' + ArtifactName: 'Windows $(_configuration) $(_testKind)$(transparentCompilerPart) test binlogs' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 @@ -514,14 +695,14 @@ stages: continueOnError: true inputs: PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\$(_configuration)' - ArtifactName: 'Windows $(_configuration) $(_testKind) process dumps' + ArtifactName: 'Windows $(_configuration) $(_testKind)$(transparentCompilerPart) process dumps' ArtifactType: Container parallel: true - task: PublishBuildArtifacts@1 displayName: Publish Test Logs inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\$(_configuration)' - ArtifactName: 'Windows $(_configuration) $(_testKind) test logs' + ArtifactName: 'Windows $(_configuration) $(_testKind)$(transparentCompilerPart) test logs' publishLocation: Container continueOnError: true condition: always() @@ -532,10 +713,73 @@ stages: displayName: Publish NuGet cache contents inputs: PathtoPublish: '$(Build.SourcesDirectory)\artifacts\NugetPackageRootContents' - ArtifactName: 'NuGetPackageContents Windows $(_testKind)' + ArtifactName: 'NuGetPackageContents Windows $(_testKind)$(transparentCompilerPart)' publishLocation: Container continueOnError: true condition: failed() + + # Windows With Compressed Metadata Desktop + - job: WindowsCompressedMetadata_testDesktop + dependsOn: WindowsCompressedMetadata_Build + variables: + - name: XUNIT_LOGS + value: $(Build.SourcesDirectory)\artifacts\TestResults\Release + - name: __VSNeverShowWhatsNew + value: 1 + pool: + # The PR build definition sets this variable: + # WindowsMachineQueueName=Windows.vs2022.amd64.open + # and there is an alternate build definition that sets this to a queue that is always scouting the + # next preview of Visual Studio. + name: $(DncEngPublicBuildPool) + demands: ImageOverride -equals $(WindowsMachineQueueName) + timeoutInMinutes: 120 + strategy: + parallel: 4 + steps: + - checkout: self + clean: true + - task: DownloadPipelineArtifact@2 + displayName: 'Download Build Artifact' + inputs: + artifact: WindowsCompressedMetadata_Build + path: '$(Build.SourcesDirectory)/artifacts' + + - pwsh: ./eng/Build.ps1 -ci -restore -configuration Release -testDesktop -testBatch $(System.JobPositionInPhase) + workingDirectory: $(Build.SourcesDirectory) + env: + DOTNET_DbgEnableMiniDump: 1 + DOTNET_DbgMiniDumpType: 3 # Triage dump, 1 for mini, 2 for Heap, 3 for triage, 4 for full. Don't use 4 unless you know what you're doing. + DOTNET_DbgMiniDumpName: $(Build.SourcesDirectory)\artifacts\log\Release\$(Build.BuildId)-%e-%p-%t.dmp + NativeToolsOnMachine: true + displayName: Test + + - task: PublishTestResults@2 + displayName: Publish Test Results + inputs: + testResultsFormat: 'XUnit' + testRunTitle: WindowsCompressedMetadata testDesktop batch $(System.JobPositionInPhase) + mergeTestResults: true + testResultsFiles: '*.xml' + searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/Release' + continueOnError: true + - task: PublishBuildArtifacts@1 + displayName: Publish Dumps + condition: failed() + continueOnError: true + inputs: + PathToPublish: '$(Build.SourcesDirectory)\artifacts\log\Release' + ArtifactName: 'Windows testDesktop process dumps batch $(System.JobPositionInPhase)' + ArtifactType: Container + parallel: true + - task: PublishBuildArtifacts@1 + displayName: Publish Test Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)\artifacts\TestResults\Release' + ArtifactName: 'Windows testDesktop test logs batch $(System.JobPositionInPhase)' + publishLocation: Container + continueOnError: true + condition: always() # Mock official build - job: MockOfficial diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 146372ecde6..fe2fe5731bd 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -73,6 +73,7 @@ param ( [switch]$compressAllMetadata, [switch]$buildnorealsig = $true, [switch]$verifypackageshipstatus = $false, + [string]$testBatch = "", [parameter(ValueFromRemainingArguments = $true)][string[]]$properties) Set-StrictMode -version 2.0 @@ -356,16 +357,24 @@ function VerifyAssemblyVersionsAndSymbols() { } } -function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string]$testadapterpath, [boolean] $asBackgroundJob = $false, [string] $settings = "") { +function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [string] $settings = "") { + $dotnetPath = InitializeDotNetCli $dotnetExe = Join-Path $dotnetPath "dotnet.exe" $projectName = [System.IO.Path]::GetFileNameWithoutExtension($testProject) + + $testBatchSuffix = "" + if ($testBatch) { + $testBatchSuffix = "_batch$testBatch" + } + # {assembly} and {framework} will expand respectively. See https://github.com/spekt/testlogger/wiki/Logger-Configuration#logfilepath # This is useful to deconflict log filenames when there are many test assemblies, e.g. when testing a whole solution. - $testLogPath = "$ArtifactsDir\TestResults\$configuration\{assembly}_{framework}.xml" - $testBinLogPath = "$LogDir\${projectName}_$targetFramework.binlog" - $args = "test $testProject -c $configuration -f $targetFramework -v n --test-adapter-path $testadapterpath --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath" - $args += " --blame --blame-hang-timeout 5minutes --results-directory $ArtifactsDir\TestResults\$configuration -p:vstestusemsbuildoutput=true" + $testLogPath = "$ArtifactsDir\TestResults\$configuration\{assembly}_{framework}$testBatchSuffix.xml" + + $testBinLogPath = "$LogDir\${projectName}_$targetFramework$testBatch.binlog" + $args = "test $testProject -c $configuration -f $targetFramework --logger ""xunit;LogFilePath=$testLogPath"" /bl:$testBinLogPath" + $args += " --blame-hang-timeout 5minutes --results-directory $ArtifactsDir\TestResults\$configuration" if (-not $noVisualStudio -or $norestore) { $args += " --no-restore" @@ -376,22 +385,12 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str } $args += " $settings" - - if ($asBackgroundJob) { - Write-Host - Write-Host("Starting on the background: $args") - Write-Host("------------------------------------") - Start-Job -ScriptBlock { - $argArray = $using:args -Split " " - & $using:dotnetExe $argArray - if ($LASTEXITCODE -ne 0) { - throw "Command failed to execute with exit code $($LASTEXITCODE): $using:dotnetExe $using:args" - } - } - } else { - Write-Host("$args") - Exec-Console $dotnetExe $args + if ($testBatch) { + $args += " --filter batch=$testBatch" } + + Write-Host("$args") + Exec-Console $dotnetExe $args } function Prepare-TempDir() { @@ -595,34 +594,12 @@ try { $script:BuildCategory = "Test" $script:BuildMessage = "Failure running tests" - function Receive($job) { - while($job.HasMoreData) { - Receive-Job $job | Write-Host - Start-Sleep -Seconds 1 - } - Receive-Job $job -Wait -ErrorAction Stop - } - if ($testCoreClr) { - $cpuLimit = if ($ci) { "-m:1 -- xUnit.MaxParallelThreads=1" } else { "" } - TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" -settings $cpuLimit + TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:coreclrTargetFramework } - if ($testDesktop -and $ci) { - $bgJob = TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" -asBackgroundJob $true - - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Test.Utilities\FSharp.Test.Utilities.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Test.Utilities\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Build.UnitTests\FSharp.Build.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Build.UnitTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\" - - Receive -job $bgJob - } - - if ($testDesktop -and -not $ci ) { - TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" + if ($testDesktop) { + TestUsingMSBuild -testProject "$RepoRoot\FSharp.sln" -targetFramework $script:desktopTargetFramework } if ($testFSharpQA) { @@ -653,50 +630,49 @@ try { } if ($testFSharpCore) { - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Core.UnitTests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Core.UnitTests\FSharp.Core.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testCompiler) { - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testCompilerComponentTests) { - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.ComponentTests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.ComponentTests\FSharp.Compiler.ComponentTests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testCompilerService) { - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Service.Tests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Service.Tests\FSharp.Compiler.Service.Tests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testCambridge) { - TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharpSuite.Tests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\fsharp\FSharpSuite.Tests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testScripting) { - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\" - TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Compiler.Private.Scripting.UnitTests\" + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:coreclrTargetFramework + TestUsingMSBuild -testProject "$RepoRoot\tests\FSharp.Compiler.Private.Scripting.UnitTests\FSharp.Compiler.Private.Scripting.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testEditor -and -not $noVisualStudio) { - TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.Tests\FSharp.Editor.Tests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Editor.Tests\FSharp.Editor.Tests.fsproj" + TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.Tests\FSharp.Editor.Tests.fsproj" -targetFramework $script:desktopTargetFramework } if ($testVs -and -not $noVisualStudio) { - TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\UnitTests\VisualFSharp.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\VisualFSharp.UnitTests\" + TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\UnitTests\VisualFSharp.UnitTests.fsproj" -targetFramework $script:desktopTargetFramework } - if ($testIntegration) { - TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework -testadapterpath "$ArtifactsDir\bin\FSharp.Editor.IntegrationTests\" + TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework } if ($testAOT) { diff --git a/eng/build.sh b/eng/build.sh index 3c9404c8161..07c3889f33d 100755 --- a/eng/build.sh +++ b/eng/build.sh @@ -76,6 +76,7 @@ prepare_machine=false source_build=false product_build=false buildnorealsig=true +testbatch="" properties="" docker=false @@ -104,6 +105,11 @@ while [[ $# > 0 ]]; do args="$args $1" shift ;; + --testbatch) + testbatch=$2 + args="$args $1" + shift + ;; --verbosity|-v) verbosity=$2 args="$args $1" @@ -224,8 +230,16 @@ function Test() { projectname=$(basename -- "$testproject") projectname="${projectname%.*}" - testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework.xml" - args="test \"$testproject\" --no-restore --no-build -c $configuration -f $targetframework --test-adapter-path . --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration -p:vstestusemsbuildoutput=false" + testbatchsuffix="" + if [[ "$testbatch" != "" ]]; then + testbatchsuffix="_batch$testbatch" + fi + testlogpath="$artifacts_dir/TestResults/$configuration/${projectname}_$targetframework$testbatchsuffix.xml" + args="test \"$testproject\" --no-build -c $configuration -f $targetframework --logger \"xunit;LogFilePath=$testlogpath\" --blame-hang-timeout 5minutes --results-directory $artifacts_dir/TestResults/$configuration" + + if [[ "$testbatch" != "" ]]; then + args="$args --filter batch=$testbatch" + fi "$DOTNET_INSTALL_DIR/dotnet" $args || exit $? } diff --git a/tests/FSharp.Test.Utilities/XunitHelpers.fs b/tests/FSharp.Test.Utilities/XunitHelpers.fs index 50deb42ea68..6f57efef4a4 100644 --- a/tests/FSharp.Test.Utilities/XunitHelpers.fs +++ b/tests/FSharp.Test.Utilities/XunitHelpers.fs @@ -91,6 +91,23 @@ module TestCaseCustomizations = else testCase.TestMethod + let sha = Security.Cryptography.SHA256.Create() + + // We add extra trait to each test, of the form "batch=n" where n is between 1 and 4. + // It can be used to filter on in multi-agent testing in CI + // with dotnet test filter switch, for example "--filter batch=1" + // That way each agent can run test for a batch of tests. + let NumberOfBatchesInMultiAgentTesting = 4u + + let addBatchTrait (testCase: ITestCase) = + // Get a batch number stable between multiple test runs. + // UniqueID is ideal here, it does not change across many compilations of the same code + // and it will split theories with member data into many batches. + let data = Text.Encoding.UTF8.GetBytes testCase.UniqueID + let hashCode = BitConverter.ToUInt32(sha.ComputeHash(data), 0) + let batch = hashCode % NumberOfBatchesInMultiAgentTesting + 1u + testCase.Traits.Add("batch", ResizeArray [ string batch ]) + type CustomTestCase = inherit XunitTestCase // xUinit demands this constructor for deserialization. @@ -110,6 +127,7 @@ type CustomTestCase = override testCase.Initialize () = base.Initialize() testCase.TestMethod <- TestCaseCustomizations.rewriteTestMethod testCase + TestCaseCustomizations.addBatchTrait testCase type CustomTheoryTestCase = inherit XunitTheoryTestCase @@ -128,6 +146,7 @@ type CustomTheoryTestCase = override testCase.Initialize () = base.Initialize() testCase.TestMethod <- TestCaseCustomizations.rewriteTestMethod testCase + TestCaseCustomizations.addBatchTrait testCase #endif