Merge pull request #3740 from sharwell/split-jobs

Run tests in parallel jobs

.nuget/packages.config

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="Codecov" version="1.9.0" />
+  <package id="CodecovUploader" version="0.7.1" />
   <package id="OpenCover" version="4.6.519" />
-  <package id="ReportGenerator" version="2.3.5.0" targetFramework="net452" />
+  <package id="ReportGenerator" version="5.2.0" />
   <package id="xunit.runner.console" version="2.4.1" />
 </packages>

azure-pipelines.yml

@@ -1,78 +1,16 @@
-pool:
+pool:
   name: Azure Pipelines
   vmImage: windows-latest
   demands:
   - msbuild
   - visualstudio
   - vstest
 
-jobs:
-- job: Build
-  variables:
-    BuildSolution: StyleCopAnalyzers.sln
-    BuildPlatform: Any CPU
-  strategy:
-    matrix:
-      Debug:
-        BuildConfiguration: Debug
-        _debugArg: '-Debug'
-      Release:
-        BuildConfiguration: Release
-        _debugArg: ''
-  steps:
-  - powershell: .\init.ps1 -NoRestore
-    displayName: Install .NET Core SDK
+stages:
+- template: build/build-and-test.yml
+  parameters:
+    BuildConfiguration: Debug
 
-  - task: NuGetToolInstaller@0
-    displayName: 'Use NuGet 5.3.1'
-    inputs:
-      versionSpec: 5.3.1
-
-  - task: NuGetCommand@2
-    displayName: 'NuGet restore'
-    inputs:
-      restoreSolution: '$(BuildSolution)'
-      feedsToUse: 'config'
-      nugetConfigPath: 'NuGet.config'
-
-  - task: VSBuild@1
-    displayName: 'Build solution StyleCopAnalyzers.sln'
-    inputs:
-      solution: '$(BuildSolution)'
-      platform: '$(BuildPlatform)'
-      configuration: '$(BuildConfiguration)'
-      msbuildArgs: '/bl:$(Build.SourcesDirectory)/msbuild.binlog'
-
-  - task: PowerShell@2
-    displayName: Run Tests
-    inputs:
-      workingDirectory: '$(Build.SourcesDirectory)/build'
-      filePath: build/opencover-report.ps1
-      arguments: '$(_debugArg) -NoBuild -NoReport -Azure'
-
-  - task: PublishTestResults@2
-    displayName: Publish test results
-    condition: always()
-    inputs:
-      testResultsFormat: xUnit
-      testResultsFiles: 'build/*.xml'
-      mergeTestResults: true
-      testRunTitle: '$(BuildConfiguration)'
-
-  - task: PowerShell@2
-    displayName: Upload coverage reports to codecov.io
-    condition: eq(variables['BuildConfiguration'], 'Debug')
-    inputs:
-      workingDirectory: '$(Build.SourcesDirectory)/build'
-      targetType: inline
-      script: |
-        $packageConfig = [xml](Get-Content ..\.nuget\packages.config)
-        $codecov_version = $packageConfig.SelectSingleNode('/packages/package[@id="Codecov"]').version
-        $codecov = "..\packages\Codecov.$codecov_version\tools\codecov.exe"
-        &$codecov -f '..\build\OpenCover.Reports\OpenCover.StyleCopAnalyzers.xml' --required
-
-  - task: PublishBuildArtifacts@1
-    displayName: Publish build logs
-    inputs:
-      pathtoPublish: msbuild.binlog
-    condition: failed()
+- template: build/build-and-test.yml
+  parameters:
+    BuildConfiguration: Release

build/build-and-test.yml

@@ -0,0 +1,280 @@
+parameters:
+- name: BuildConfiguration
+  displayName: Build Configuration
+  type: string
+  default: Debug
+  values: [ 'Debug', 'Release' ]
+- name: BuildSolution
+  displayName: Solution
+  type: string
+  default: StyleCopAnalyzers.sln
+- name: BuildPlatform
+  displayName: Platform
+  type: string
+  default: Any CPU
+
+stages:
+- stage: Build_${{ parameters.BuildConfiguration }}
+  displayName: Build ${{ parameters.BuildConfiguration }}
+  dependsOn: []
+  jobs:
+  - job: Build
+    steps:
+    - powershell: .\init.ps1 -NoRestore
+      displayName: Install .NET Core SDK
+
+    - task: NuGetToolInstaller@0
+      displayName: 'Use NuGet 5.3.1'
+      inputs:
+        versionSpec: 5.3.1
+
+    - task: NuGetCommand@2
+      displayName: 'NuGet restore'
+      inputs:
+        restoreSolution: '${{ parameters.BuildSolution }}'
+        feedsToUse: 'config'
+        nugetConfigPath: 'NuGet.config'
+
+    - task: VSBuild@1
+      displayName: 'Build solution ${{ parameters.BuildSolution }}'
+      inputs:
+        solution: '${{ parameters.BuildSolution }}'
+        platform: '${{ parameters.BuildPlatform }}'
+        configuration: '${{ parameters.BuildConfiguration }}'
+        maximumCpuCount: false # AnnotatorBuildTask doesn't support parallel builds yet
+        msbuildArgs: '/v:minimal /bl:$(Build.SourcesDirectory)/msbuild.binlog'
+
+#    - task: PowerShell@2
+#      displayName: Upload coverage reports to codecov.io
+#      condition: eq(variables['BuildConfiguration'], 'Debug')
+#      inputs:
+#        workingDirectory: '$(Build.SourcesDirectory)/build'
+#        targetType: inline
+#        script: |
+#          $packageConfig = [xml](Get-Content ..\.nuget\packages.config)
+#          $codecov_version = $packageConfig.SelectSingleNode('/packages/package[@id="Codecov"]').version
+#          $codecov = "..\packages\Codecov.$codecov_version\tools\codecov.exe"
+#          &$codecov -f '..\build\OpenCover.Reports\OpenCover.StyleCopAnalyzers.xml' --required
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish build logs
+      inputs:
+        targetPath: msbuild.binlog
+        artifact: Build logs ${{ parameters.BuildConfiguration }}
+      condition: failed()
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish solution packages
+      inputs:
+        targetPath: $(Build.SourcesDirectory)/packages
+        artifact: slnPackages-${{ parameters.BuildConfiguration }}
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish build output (Test C# 6)
+      inputs:
+        targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test/bin
+        artifact: buildTest-cs6-${{ parameters.BuildConfiguration }}
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish build output (Test C# 7)
+      inputs:
+        targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp7/bin
+        artifact: buildTest-cs7-${{ parameters.BuildConfiguration }}
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish build output (Test C# 8)
+      inputs:
+        targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp8/bin
+        artifact: buildTest-cs8-${{ parameters.BuildConfiguration }}
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish build output (Test C# 9)
+      inputs:
+        targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp9/bin
+        artifact: buildTest-cs9-${{ parameters.BuildConfiguration }}
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish build output (Test C# 10)
+      inputs:
+        targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp10/bin
+        artifact: buildTest-cs10-${{ parameters.BuildConfiguration }}
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish build output (Test C# 11)
+      inputs:
+        targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp11/bin
+        artifact: buildTest-cs11-${{ parameters.BuildConfiguration }}
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish build output (Test C# 12)
+      inputs:
+        targetPath: $(Build.SourcesDirectory)/StyleCop.Analyzers/StyleCop.Analyzers.Test.CSharp12/bin
+        artifact: buildTest-cs12-${{ parameters.BuildConfiguration }}
+
+- stage: Test_CSharp_6_${{ parameters.BuildConfiguration }}
+  displayName: Test C# 6 ${{ parameters.BuildConfiguration }}
+  dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ]
+  jobs:
+  - template: test.yml
+    parameters:
+      BuildConfiguration: ${{ parameters.BuildConfiguration }}
+      BuildSolution: ${{ parameters.BuildSolution }}
+      BuildPlatform: ${{ parameters.BuildPlatform }}
+      LangVersion: '6'
+      FrameworkVersion: 'net452'
+
+- stage: Test_CSharp_7_${{ parameters.BuildConfiguration }}
+  displayName: Test C# 7 ${{ parameters.BuildConfiguration }}
+  dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ]
+  jobs:
+  - template: test.yml
+    parameters:
+      BuildConfiguration: ${{ parameters.BuildConfiguration }}
+      BuildSolution: ${{ parameters.BuildSolution }}
+      BuildPlatform: ${{ parameters.BuildPlatform }}
+      LangVersion: '7'
+      FrameworkVersion: 'net46'
+
+- stage: Test_CSharp_8_${{ parameters.BuildConfiguration }}
+  displayName: Test C# 8 ${{ parameters.BuildConfiguration }}
+  dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ]
+  jobs:
+  - template: test.yml
+    parameters:
+      BuildConfiguration: ${{ parameters.BuildConfiguration }}
+      BuildSolution: ${{ parameters.BuildSolution }}
+      BuildPlatform: ${{ parameters.BuildPlatform }}
+      LangVersion: '8'
+      FrameworkVersion: 'net472'
+
+- stage: Test_CSharp_9_${{ parameters.BuildConfiguration }}
+  displayName: Test C# 9 ${{ parameters.BuildConfiguration }}
+  dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ]
+  jobs:
+  - template: test.yml
+    parameters:
+      BuildConfiguration: ${{ parameters.BuildConfiguration }}
+      BuildSolution: ${{ parameters.BuildSolution }}
+      BuildPlatform: ${{ parameters.BuildPlatform }}
+      LangVersion: '9'
+      FrameworkVersion: 'net472'
+
+- stage: Test_CSharp_10_${{ parameters.BuildConfiguration }}
+  displayName: Test C# 10 ${{ parameters.BuildConfiguration }}
+  dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ]
+  jobs:
+  - template: test.yml
+    parameters:
+      BuildConfiguration: ${{ parameters.BuildConfiguration }}
+      BuildSolution: ${{ parameters.BuildSolution }}
+      BuildPlatform: ${{ parameters.BuildPlatform }}
+      LangVersion: '10'
+      FrameworkVersion: 'net472'
+
+- stage: Test_CSharp_11_${{ parameters.BuildConfiguration }}
+  displayName: Test C# 11 ${{ parameters.BuildConfiguration }}
+  dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ]
+  jobs:
+  - template: test.yml
+    parameters:
+      BuildConfiguration: ${{ parameters.BuildConfiguration }}
+      BuildSolution: ${{ parameters.BuildSolution }}
+      BuildPlatform: ${{ parameters.BuildPlatform }}
+      LangVersion: '11'
+      FrameworkVersion: 'net472'
+
+- stage: Test_CSharp_12_${{ parameters.BuildConfiguration }}
+  displayName: Test C# 12 ${{ parameters.BuildConfiguration }}
+  dependsOn: [ 'Build_${{ parameters.BuildConfiguration }}' ]
+  jobs:
+  - template: test.yml
+    parameters:
+      BuildConfiguration: ${{ parameters.BuildConfiguration }}
+      BuildSolution: ${{ parameters.BuildSolution }}
+      BuildPlatform: ${{ parameters.BuildPlatform }}
+      LangVersion: '12'
+      FrameworkVersion: 'net472'
+
+- stage: Publish_Code_Coverage_${{ parameters.BuildConfiguration }}
+  displayName: Publish Code Coverage
+  condition: eq('${{ parameters.BuildConfiguration }}', 'Debug')
+  dependsOn:
+  - Test_CSharp_6_${{ parameters.BuildConfiguration }}
+  - Test_CSharp_7_${{ parameters.BuildConfiguration }}
+  - Test_CSharp_8_${{ parameters.BuildConfiguration }}
+  - Test_CSharp_9_${{ parameters.BuildConfiguration }}
+  - Test_CSharp_10_${{ parameters.BuildConfiguration }}
+  - Test_CSharp_11_${{ parameters.BuildConfiguration }}
+  - Test_CSharp_12_${{ parameters.BuildConfiguration }}
+  jobs:
+  - job: WrapUp
+    steps:
+    - checkout: self
+      fetchDepth: 0 # avoid shallow clone so nbgv can do its work.
+      clean: true
+    - task: DownloadPipelineArtifact@2
+      displayName: 🔻 Download solution packages
+      continueOnError: true
+      inputs:
+        buildType: current
+        artifactName: slnPackages-${{ parameters.BuildConfiguration }}
+        targetPath: $(Build.SourcesDirectory)/packages
+    - download: current
+      artifact: coverageResults-cs6
+      displayName: 🔻 Download C# 6 code coverage results
+      continueOnError: true
+    - download: current
+      artifact: coverageResults-cs7
+      displayName: 🔻 Download C# 7 code coverage results
+      continueOnError: true
+    - download: current
+      artifact: coverageResults-cs8
+      displayName: 🔻 Download C# 8 code coverage results
+      continueOnError: true
+    - download: current
+      artifact: coverageResults-cs9
+      displayName: 🔻 Download C# 9 code coverage results
+      continueOnError: true
+    - download: current
+      artifact: coverageResults-cs10
+      displayName: 🔻 Download C# 10 code coverage results
+      continueOnError: true
+    - download: current
+      artifact: coverageResults-cs11
+      displayName: 🔻 Download C# 11 code coverage results
+      continueOnError: true
+    - download: current
+      artifact: coverageResults-cs12
+      displayName: 🔻 Download C# 12 code coverage results
+      continueOnError: true
+    - task: PowerShell@2
+      displayName: ⚙ Merge coverage
+      timeoutInMinutes: 20
+      inputs:
+        workingDirectory: $(Build.SourcesDirectory)
+        targetType: inline
+        script: |
+          $packageConfig = [xml](Get-Content .\.nuget\packages.config)
+          $packages_folder = '.\packages'
+          $reportgenerator_version = $packageConfig.SelectSingleNode('/packages/package[@id="ReportGenerator"]').version
+          $report_generator = "$packages_folder\ReportGenerator.$reportgenerator_version\tools\net47\ReportGenerator.exe"
+          &$report_generator -targetdir:$(Pipeline.Workspace)/coverageResults-final -reporttypes:Cobertura "-reports:$(Pipeline.Workspace)/coverageResults-*/OpenCover.*.xml"
+
+    - task: PowerShell@2
+      displayName: Public code coverage to codecov.io
+      timeoutInMinutes: 20
+      inputs:
+        workingDirectory: $(Build.SourcesDirectory)
+        targetType: inline
+        script: |
+          $packageConfig = [xml](Get-Content .\.nuget\packages.config)
+          $packages_folder = '.\packages'
+          $codecov_version = $packageConfig.SelectSingleNode('/packages/package[@id="CodecovUploader"]').version
+          $codecov = "$packages_folder\CodecovUploader.$codecov_version\tools\codecov.exe"
+          &$codecov -f '$(Pipeline.Workspace)/coverageResults-final/Cobertura.xml' --required
+
+    - task: PublishPipelineArtifact@1
+      displayName: Publish merged coverage
+      inputs:
+        targetPath: $(Pipeline.Workspace)/coverageResults-final
+        artifact: coverageResults-final$(System.JobAttempt)-${{ parameters.BuildConfiguration }}