diff --git a/azure-pipelines-official.yml b/azure-pipelines-official.yml index eb612a012b..4133d3d096 100644 --- a/azure-pipelines-official.yml +++ b/azure-pipelines-official.yml @@ -127,7 +127,7 @@ extends: enabled: true pool: name: $(DncEngInternalBuildPool) - image: windows.vs2022preview.amd64 + image: windows.vs2026preview.scout.amd64 os: windows settings: networkIsolationPolicy: Permissive,CFSClean @@ -153,7 +153,7 @@ extends: timeoutInMinutes: 120 pool: name: $(DncEngInternalBuildPool) - image: windows.vs2022preview.amd64 + image: windows.vs2026preview.scout.amd64 os: windows steps: # This steps help to understand versions of .NET runtime installed on the machine, @@ -187,6 +187,8 @@ extends: -performanceTest name: Test displayName: Test + env: + DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet # This step is only helpful for diagnosing some issues with vstest/test host that would not appear # through the console or trx @@ -257,7 +259,7 @@ extends: - ${{ pool.os }} pool: name: $(DncEngInternalBuildPool) - image: windows.vs2022preview.amd64 + image: windows.vs2026preview.scout.amd64 os: windows steps: # The template job needs a log, otherwise it writes a warning. We can disable log uploading only for diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 541b35ce34..b10cc7ce90 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -81,7 +81,7 @@ stages: timeoutInMinutes: 120 pool: name: NetCore-Public - demands: ImageOverride -equals windows.vs2022preview.amd64.open + demands: ImageOverride -equals windows.vs2026preview.scout.amd64.open strategy: matrix: Release: @@ -115,6 +115,8 @@ stages: /bl:$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)/Test.binlog name: Test displayName: Test + env: + DOTNET_ROOT: $(Build.SourcesDirectory)/.dotnet # This step is only helpful for diagnosing some issues with vstest/test host that would not appear # through the console or trx diff --git a/build.cmd b/build.cmd index 4afad04714..10a90a684c 100644 --- a/build.cmd +++ b/build.cmd @@ -1,3 +1,3 @@ @echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore -build %*" +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\Build.ps1""" -restore -build %*" exit /b %ErrorLevel% diff --git a/eng/Version.Details.props b/eng/Version.Details.props index 1aac108722..a8ded7bfbb 100644 --- a/eng/Version.Details.props +++ b/eng/Version.Details.props @@ -14,7 +14,7 @@ This file should be imported by eng/Versions.props 2.0.0 - 10.0.0-beta.25408.3 + 11.0.0-beta.25571.8 1.1.0-beta2-19575-01 1.1.0-beta2-19575-01 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6966816e09..f1f366416b 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -22,9 +22,9 @@ - + https://github.com/dotnet/arcade - d27184f7cb92b4abb4b20e91ecb5c43bc43de65b + a8b9e73f56e84dcb4c7ed0f23f79b308b4f83a01 https://github.com/dotnet/symreader-converter diff --git a/eng/build.ps1 b/eng/build.ps1 new file mode 100644 index 0000000000..06cecc7a53 --- /dev/null +++ b/eng/build.ps1 @@ -0,0 +1,45 @@ +[CmdletBinding(PositionalBinding=$false)] +Param( + [string][Alias('c')]$configuration = "Debug", + [string]$platform = $null, + [string] $projects, + [string][Alias('v')]$verbosity = "minimal", + [string] $msbuildEngine = $null, + [bool] $warnAsError = $true, + [bool] $nodeReuse = $true, + [switch] $buildCheck = $false, + [switch][Alias('r')]$restore, + [switch] $deployDeps, + [switch][Alias('b')]$build, + [switch] $rebuild, + [switch] $deploy, + [switch][Alias('t')]$test, + [switch] $integrationTest, + [switch] $performanceTest, + [switch] $sign, + [switch] $pack, + [switch] $publish, + [switch] $clean, + [switch][Alias('pb')]$productBuild, + [switch]$fromVMR, + [switch][Alias('bl')]$binaryLog, + [switch][Alias('nobl')]$excludeCIBinarylog, + [switch] $ci, + [switch] $prepareMachine, + [string] $runtimeSourceFeed = '', + [string] $runtimeSourceFeedKey = '', + [switch] $excludePrereleaseVS, + [switch] $nativeToolsOnMachine, + [switch] $help, + [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties +) + +# Workaround for DOTNET_HOST_PATH not being set by older MSBuild +if (-not $env:DOTNET_HOST_PATH) { + $env:DOTNET_HOST_PATH = [System.IO.Path]::GetFullPath((Join-Path (Join-Path (Join-Path $PSScriptRoot '..') '.dotnet') 'dotnet')) + if (-not (Test-Path $env:DOTNET_HOST_PATH)) { + $env:DOTNET_HOST_PATH = "$($env:DOTNET_HOST_PATH).exe" + } +} + +& $PSScriptRoot/common/build.ps1 @PSBoundParameters \ No newline at end of file diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 5db4ad71ee..fc8d618014 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -7,11 +7,11 @@ # See example call for this script below. # # - task: PowerShell@2 -# displayName: Setup Private Feeds Credentials +# displayName: Setup internal Feeds Credentials # condition: eq(variables['Agent.OS'], 'Windows_NT') # inputs: -# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 -# arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token +# filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 +# arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token # env: # Token: $(dn-bot-dnceng-artifact-feeds-rw) # @@ -34,19 +34,28 @@ Set-StrictMode -Version 2.0 . $PSScriptRoot\tools.ps1 +# Adds or enables the package source with the given name +function AddOrEnablePackageSource($sources, $disabledPackageSources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { + if ($disabledPackageSources -eq $null -or -not (EnableInternalPackageSource -DisabledPackageSources $disabledPackageSources -Creds $creds -PackageSourceName $SourceName)) { + AddPackageSource -Sources $sources -SourceName $SourceName -SourceEndPoint $SourceEndPoint -Creds $creds -Username $userName -pwd $Password + } +} + # Add source entry to PackageSources function AddPackageSource($sources, $SourceName, $SourceEndPoint, $creds, $Username, $pwd) { $packageSource = $sources.SelectSingleNode("add[@key='$SourceName']") if ($packageSource -eq $null) { + Write-Host "Adding package source $SourceName" + $packageSource = $doc.CreateElement("add") $packageSource.SetAttribute("key", $SourceName) $packageSource.SetAttribute("value", $SourceEndPoint) $sources.AppendChild($packageSource) | Out-Null } else { - Write-Host "Package source $SourceName already present." + Write-Host "Package source $SourceName already present and enabled." } AddCredential -Creds $creds -Source $SourceName -Username $Username -pwd $pwd @@ -59,6 +68,8 @@ function AddCredential($creds, $source, $username, $pwd) { return; } + Write-Host "Inserting credential for feed: " $source + # Looks for credential configuration for the given SourceName. Create it if none is found. $sourceElement = $creds.SelectSingleNode($Source) if ($sourceElement -eq $null) @@ -91,24 +102,27 @@ function AddCredential($creds, $source, $username, $pwd) { $passwordElement.SetAttribute("value", $pwd) } -function InsertMaestroPrivateFeedCredentials($Sources, $Creds, $Username, $pwd) { - $maestroPrivateSources = $Sources.SelectNodes("add[contains(@key,'darc-int')]") - - Write-Host "Inserting credentials for $($maestroPrivateSources.Count) Maestro's private feeds." - - ForEach ($PackageSource in $maestroPrivateSources) { - Write-Host "`tInserting credential for Maestro's feed:" $PackageSource.Key - AddCredential -Creds $creds -Source $PackageSource.Key -Username $Username -pwd $pwd +# Enable all darc-int package sources. +function EnableMaestroInternalPackageSources($DisabledPackageSources, $Creds) { + $maestroInternalSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") + ForEach ($DisabledPackageSource in $maestroInternalSources) { + EnableInternalPackageSource -DisabledPackageSources $DisabledPackageSources -Creds $Creds -PackageSourceName $DisabledPackageSource.key } } -function EnablePrivatePackageSources($DisabledPackageSources) { - $maestroPrivateSources = $DisabledPackageSources.SelectNodes("add[contains(@key,'darc-int')]") - ForEach ($DisabledPackageSource in $maestroPrivateSources) { - Write-Host "`tEnsuring private source '$($DisabledPackageSource.key)' is enabled by deleting it from disabledPackageSource" +# Enables an internal package source by name, if found. Returns true if the package source was found and enabled, false otherwise. +function EnableInternalPackageSource($DisabledPackageSources, $Creds, $PackageSourceName) { + $DisabledPackageSource = $DisabledPackageSources.SelectSingleNode("add[@key='$PackageSourceName']") + if ($DisabledPackageSource) { + Write-Host "Enabling internal source '$($DisabledPackageSource.key)'." + # Due to https://github.com/NuGet/Home/issues/10291, we must actually remove the disabled entries $DisabledPackageSources.RemoveChild($DisabledPackageSource) + + AddCredential -Creds $creds -Source $DisabledPackageSource.Key -Username $userName -pwd $Password + return $true } + return $false } if (!(Test-Path $ConfigFile -PathType Leaf)) { @@ -121,15 +135,17 @@ $doc = New-Object System.Xml.XmlDocument $filename = (Get-Item $ConfigFile).FullName $doc.Load($filename) -# Get reference to or create one if none exist already +# Get reference to - fail if none exist $sources = $doc.DocumentElement.SelectSingleNode("packageSources") if ($sources -eq $null) { - $sources = $doc.CreateElement("packageSources") - $doc.DocumentElement.AppendChild($sources) | Out-Null + Write-PipelineTelemetryError -Category 'Build' -Message "Eng/common/SetupNugetSources.ps1 returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile" + ExitWithExitCode 1 } $creds = $null +$feedSuffix = "v3/index.json" if ($Password) { + $feedSuffix = "v2" # Looks for a node. Create it if none is found. $creds = $doc.DocumentElement.SelectSingleNode("packageSourceCredentials") if ($creds -eq $null) { @@ -138,33 +154,22 @@ if ($Password) { } } +$userName = "dn-bot" + # Check for disabledPackageSources; we'll enable any darc-int ones we find there $disabledSources = $doc.DocumentElement.SelectSingleNode("disabledPackageSources") if ($disabledSources -ne $null) { Write-Host "Checking for any darc-int disabled package sources in the disabledPackageSources node" - EnablePrivatePackageSources -DisabledPackageSources $disabledSources -} - -$userName = "dn-bot" - -# Insert credential nodes for Maestro's private feeds -InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -pwd $Password - -# 3.1 uses a different feed url format so it's handled differently here -$dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") -if ($dotnet31Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password + EnableMaestroInternalPackageSources -DisabledPackageSources $disabledSources -Creds $creds } - -$dotnetVersions = @('5','6','7','8','9') +$dotnetVersions = @('5','6','7','8','9','10') foreach ($dotnetVersion in $dotnetVersions) { $feedPrefix = "dotnet" + $dotnetVersion; $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") if ($dotnetSource -ne $null) { - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -pwd $Password - AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -pwd $Password + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password + AddOrEnablePackageSource -Sources $sources -DisabledPackageSources $disabledSources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/$feedSuffix" -Creds $creds -Username $userName -pwd $Password } } diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index 4604b61b03..b97cc53637 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -11,8 +11,8 @@ # - task: Bash@3 # displayName: Setup Internal Feeds # inputs: -# filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh -# arguments: $(Build.SourcesDirectory)/NuGet.config +# filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.sh +# arguments: $(System.DefaultWorkingDirectory)/NuGet.config # condition: ne(variables['Agent.OS'], 'Windows_NT') # - task: NuGetAuthenticate@1 # @@ -52,78 +52,124 @@ if [[ `uname -s` == "Darwin" ]]; then TB='' fi -# Ensure there is a ... section. -grep -i "" $ConfigFile -if [ "$?" != "0" ]; then - echo "Adding ... section." - ConfigNodeHeader="" - PackageSourcesTemplate="${TB}${NL}${TB}" +# Enables an internal package source by name, if found. Returns 0 if found and enabled, 1 if not found. +EnableInternalPackageSource() { + local PackageSourceName="$1" + + # Check if disabledPackageSources section exists + grep -i "" "$ConfigFile" > /dev/null + if [ "$?" != "0" ]; then + return 1 # No disabled sources section + fi + + # Check if this source name is disabled + grep -i " /dev/null + if [ "$?" == "0" ]; then + echo "Enabling internal source '$PackageSourceName'." + # Remove the disabled entry (including any surrounding comments or whitespace on the same line) + sed -i.bak "//d" "$ConfigFile" + + # Add the source name to PackageSources for credential handling + PackageSources+=("$PackageSourceName") + return 0 # Found and enabled + fi + + return 1 # Not found in disabled sources +} + +# Add source entry to PackageSources +AddPackageSource() { + local SourceName="$1" + local SourceEndPoint="$2" + + # Check if source already exists + grep -i " /dev/null + if [ "$?" == "0" ]; then + echo "Package source $SourceName already present and enabled." + PackageSources+=("$SourceName") + return + fi + + echo "Adding package source $SourceName" + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" "$ConfigFile" + PackageSources+=("$SourceName") +} + +# Adds or enables the package source with the given name +AddOrEnablePackageSource() { + local SourceName="$1" + local SourceEndPoint="$2" + + # Try to enable if disabled, if not found then add new source + EnableInternalPackageSource "$SourceName" + if [ "$?" != "0" ]; then + AddPackageSource "$SourceName" "$SourceEndPoint" + fi +} - sed -i.bak "s|$ConfigNodeHeader|$ConfigNodeHeader${NL}$PackageSourcesTemplate|" $ConfigFile -fi +# Enable all darc-int package sources +EnableMaestroInternalPackageSources() { + # Check if disabledPackageSources section exists + grep -i "" "$ConfigFile" > /dev/null + if [ "$?" != "0" ]; then + return # No disabled sources section + fi + + # Find all darc-int disabled sources + local DisabledDarcIntSources=() + DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' "$ConfigFile" | tr -d '"') + + for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do + if [[ $DisabledSourceName == darc-int* ]]; then + EnableInternalPackageSource "$DisabledSourceName" + fi + done +} -# Ensure there is a ... section. -grep -i "" $ConfigFile +# Ensure there is a ... section. +grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding ... section." - - PackageSourcesNodeFooter="" - PackageSourceCredentialsTemplate="${TB}${NL}${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile + Write-PipelineTelemetryError -Category 'Build' "Error: Eng/common/SetupNugetSources.sh returned a non-zero exit code. NuGet config file must contain a packageSources section: $ConfigFile" + ExitWithExitCode 1 fi PackageSources=() -# Ensure dotnet3.1-internal and dotnet3.1-internal-transport are in the packageSources if the public dotnet3.1 feeds are present -grep -i "... section. + grep -i "" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + echo "Adding ... section." - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet3.1-internal') - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding dotnet3.1-internal-transport to the packageSources." PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" + PackageSourceCredentialsTemplate="${TB}${NL}${TB}" - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourcesNodeFooter${NL}$PackageSourceCredentialsTemplate|" $ConfigFile fi - PackageSources+=('dotnet3.1-internal-transport') fi -DotNetVersions=('5' '6' '7' '8' '9') +# Check for disabledPackageSources; we'll enable any darc-int ones we find there +grep -i "" $ConfigFile > /dev/null +if [ "$?" == "0" ]; then + echo "Checking for any darc-int disabled package sources in the disabledPackageSources node" + EnableMaestroInternalPackageSources +fi + +DotNetVersions=('5' '6' '7' '8' '9' '10') for DotNetVersion in ${DotNetVersions[@]} ; do FeedPrefix="dotnet${DotNetVersion}"; - grep -i " /dev/null if [ "$?" == "0" ]; then - grep -i "" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal") - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding $FeedPrefix-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=("$FeedPrefix-internal-transport") + AddOrEnablePackageSource "$FeedPrefix-internal" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal/nuget/$FeedSuffix" + AddOrEnablePackageSource "$FeedPrefix-internal-transport" "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$FeedPrefix-internal-transport/nuget/$FeedSuffix" fi done @@ -139,29 +185,12 @@ if [ "$CredToken" ]; then # Check if there is no existing credential for this FeedName grep -i "<$FeedName>" $ConfigFile if [ "$?" != "0" ]; then - echo "Adding credentials for $FeedName." + echo " Inserting credential for feed: $FeedName" PackageSourceCredentialsNodeFooter="" - NewCredential="${TB}${TB}<$FeedName>${NL}${NL}${NL}" + NewCredential="${TB}${TB}<$FeedName>${NL}${TB}${NL}${TB}${TB}${NL}${TB}${TB}" sed -i.bak "s|$PackageSourceCredentialsNodeFooter|$NewCredential${NL}$PackageSourceCredentialsNodeFooter|" $ConfigFile fi done fi - -# Re-enable any entries in disabledPackageSources where the feed name contains darc-int -grep -i "" $ConfigFile -if [ "$?" == "0" ]; then - DisabledDarcIntSources=() - echo "Re-enabling any disabled \"darc-int\" package sources in $ConfigFile" - DisabledDarcIntSources+=$(grep -oh '"darc-int-[^"]*" value="true"' $ConfigFile | tr -d '"') - for DisabledSourceName in ${DisabledDarcIntSources[@]} ; do - if [[ $DisabledSourceName == darc-int* ]] - then - OldDisableValue="" - NewDisableValue="" - sed -i.bak "s|$OldDisableValue|$NewDisableValue|" $ConfigFile - echo "Neutralized disablePackageSources entry for '$DisabledSourceName'" - fi - done -fi diff --git a/eng/common/build.sh b/eng/common/build.sh index 9767bb411a..ec3e80d189 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -92,7 +92,7 @@ runtime_source_feed='' runtime_source_feed_key='' properties=() -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "${1/#--/-}" | tr "[:upper:]" "[:lower:]")" case "$opt" in -help|-h) diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index d901325154..cb4ccc023a 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -19,6 +19,8 @@ parameters: # publishing defaults artifacts: '' enableMicrobuild: false + enablePreviewMicrobuild: false + microbuildPluginVersion: 'latest' enableMicrobuildForMacAndLinux: false microbuildUseESRP: true enablePublishBuildArtifacts: false @@ -128,6 +130,8 @@ jobs: - template: /eng/common/core-templates/steps/install-microbuild.yml parameters: enableMicrobuild: ${{ parameters.enableMicrobuild }} + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }} enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }} microbuildUseESRP: ${{ parameters.microbuildUseESRP }} continueOnError: ${{ parameters.continueOnError }} @@ -153,6 +157,8 @@ jobs: - template: /eng/common/core-templates/steps/cleanup-microbuild.yml parameters: enableMicrobuild: ${{ parameters.enableMicrobuild }} + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildPluginVersion: ${{ parameters.microbuildPluginVersion }} enableMicrobuildForMacAndLinux: ${{ parameters.enableMicrobuildForMacAndLinux }} continueOnError: ${{ parameters.continueOnError }} @@ -163,7 +169,7 @@ jobs: inputs: testResultsFormat: 'xUnit' testResultsFiles: '*.xml' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults/$(_BuildConfig)' testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-xunit mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true @@ -174,7 +180,7 @@ jobs: inputs: testResultsFormat: 'VSTest' testResultsFiles: '*.trx' - searchFolder: '$(Build.SourcesDirectory)/artifacts/TestResults/$(_BuildConfig)' + searchFolder: '$(System.DefaultWorkingDirectory)/artifacts/TestResults/$(_BuildConfig)' testRunTitle: ${{ coalesce(parameters.testRunTitle, parameters.name, '$(System.JobName)') }}-trx mergeTestResults: ${{ parameters.mergeTestResults }} continueOnError: true @@ -218,7 +224,7 @@ jobs: - task: CopyFiles@2 displayName: Gather buildconfiguration for build retry inputs: - SourceFolder: '$(Build.SourcesDirectory)/eng/common/BuildConfiguration' + SourceFolder: '$(System.DefaultWorkingDirectory)/eng/common/BuildConfiguration' Contents: '**' TargetFolder: '$(Build.ArtifactStagingDirectory)/eng/common/BuildConfiguration' continueOnError: true diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index 8bf7d23355..c5788829a8 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -8,7 +8,7 @@ parameters: CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - SourcesDirectory: $(Build.SourcesDirectory) + SourcesDirectory: $(System.DefaultWorkingDirectory) CreatePr: true AutoCompletePr: false ReusePr: true @@ -68,7 +68,7 @@ jobs: - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - task: Powershell@2 inputs: - filePath: $(Build.SourcesDirectory)/eng/common/generate-locproject.ps1 + filePath: $(System.DefaultWorkingDirectory)/eng/common/generate-locproject.ps1 arguments: $(_GenerateLocProjectArguments) displayName: Generate LocProject.json condition: ${{ parameters.condition }} @@ -103,7 +103,7 @@ jobs: - task: CopyFiles@2 displayName: Copy LocProject.json inputs: - SourceFolder: '$(Build.SourcesDirectory)/eng/Localize/' + SourceFolder: '$(System.DefaultWorkingDirectory)/eng/Localize/' Contents: 'LocProject.json' TargetFolder: '$(Build.ArtifactStagingDirectory)/loc' condition: ${{ parameters.condition }} diff --git a/eng/common/core-templates/job/publish-build-assets.yml b/eng/common/core-templates/job/publish-build-assets.yml index d5303229c9..8b5c635fe8 100644 --- a/eng/common/core-templates/job/publish-build-assets.yml +++ b/eng/common/core-templates/job/publish-build-assets.yml @@ -38,6 +38,10 @@ parameters: # Optional: A minimatch pattern for the asset manifests to publish to BAR assetManifestsPattern: '*/manifests/**/*.xml' + repositoryAlias: self + + officialBuildId: '' + jobs: - job: Asset_Registry_Publish @@ -60,6 +64,11 @@ jobs: value: false # unconditional - needed for logs publishing (redactor tool version) - template: /eng/common/core-templates/post-build/common-variables.yml + - name: OfficialBuildId + ${{ if ne(parameters.officialBuildId, '') }}: + value: ${{ parameters.officialBuildId }} + ${{ else }}: + value: $(Build.BuildNumber) pool: # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com) @@ -78,12 +87,12 @@ jobs: - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - checkout: self + - checkout: ${{ parameters.repositoryAlias }} fetchDepth: 3 clean: true - - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: - - ${{ if eq(parameters.publishingVersion, 3) }}: + - ${{ if eq(parameters.isAssetlessBuild, 'false') }}: + - ${{ if eq(parameters.publishingVersion, 3) }}: - task: DownloadPipelineArtifact@2 displayName: Download Asset Manifests inputs: @@ -108,24 +117,35 @@ jobs: flattenFolders: true condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: NuGetAuthenticate@1 + # Populate internal runtime variables. + - template: /eng/common/templates/steps/enable-internal-sources.yml + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + - task: AzureCLI@2 displayName: Publish Build Assets inputs: azureSubscription: "Darc: Maestro Production" scriptType: ps scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/sdk-task.ps1 + scriptPath: $(System.DefaultWorkingDirectory)/eng/common/sdk-task.ps1 arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet /p:ManifestsPath='$(Build.StagingDirectory)/AssetManifests' /p:IsAssetlessBuild=${{ parameters.isAssetlessBuild }} /p:MaestroApiEndpoint=https://maestro.dot.net - /p:OfficialBuildId=$(Build.BuildNumber) + /p:OfficialBuildId=$(OfficialBuildId) + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' + condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} - + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -137,7 +157,7 @@ jobs: Add-Content -Path $filePath -Value "$(DefaultChannels)" Add-Content -Path $filePath -Value $(IsStableBuild) - $symbolExclusionfile = "$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt" + $symbolExclusionfile = "$(System.DefaultWorkingDirectory)/eng/SymbolPublishingExclusionsFile.txt" if (Test-Path -Path $symbolExclusionfile) { Write-Host "SymbolExclusionFile exists" @@ -153,7 +173,7 @@ jobs: artifactName: AssetManifests displayName: 'Publish Merged Manifest' retryCountOnTaskFailure: 10 # for any logs being locked - sbomEnabled: false # we don't need SBOM for logs + sbomEnabled: false # we don't need SBOM for logs - template: /eng/common/core-templates/steps/publish-build-artifacts.yml parameters: @@ -170,6 +190,11 @@ jobs: BARBuildId: ${{ parameters.BARBuildId }} PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} is1ESPipeline: ${{ parameters.is1ESPipeline }} + + # Darc is targeting 8.0, so make sure it's installed + - task: UseDotNet@2 + inputs: + version: 8.0.x - task: AzureCLI@2 displayName: Publish Using Darc @@ -177,7 +202,7 @@ jobs: azureSubscription: "Darc: Maestro Production" scriptType: ps scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 + scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 arguments: > -BuildId $(BARBuildId) -PublishingInfraVersion 3 @@ -186,9 +211,11 @@ jobs: -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' -SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}' + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - template: /eng/common/core-templates/steps/publish-logs.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} - JobLabel: 'Publish_Artifacts_Logs' + JobLabel: 'Publish_Artifacts_Logs' diff --git a/eng/common/core-templates/job/source-build.yml b/eng/common/core-templates/job/source-build.yml index d805d5faeb..9d820f9742 100644 --- a/eng/common/core-templates/job/source-build.yml +++ b/eng/common/core-templates/job/source-build.yml @@ -60,10 +60,10 @@ jobs: pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore-Svc-Public' ), False, 'NetCore-Public')] - demands: ImageOverride -equals build.ubuntu.2004.amd64 + demands: ImageOverride -equals build.ubuntu.2204.amd64 ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $[replace(replace(eq(contains(coalesce(variables['System.PullRequest.TargetBranch'], variables['Build.SourceBranch'], 'refs/heads/main'), 'release'), 'true'), True, 'NetCore1ESPool-Svc-Internal'), False, 'NetCore1ESPool-Internal')] - image: 1es-mariner-2 + image: 1es-azurelinux-3 os: linux ${{ else }}: pool: diff --git a/eng/common/core-templates/jobs/codeql-build.yml b/eng/common/core-templates/jobs/codeql-build.yml index 693b00b370..dbc14ac580 100644 --- a/eng/common/core-templates/jobs/codeql-build.yml +++ b/eng/common/core-templates/jobs/codeql-build.yml @@ -24,7 +24,7 @@ jobs: - name: DefaultGuardianVersion value: 0.109.0 - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + value: $(System.DefaultWorkingDirectory)\eng\common\sdl\packages.config - name: GuardianVersion value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} diff --git a/eng/common/core-templates/jobs/jobs.yml b/eng/common/core-templates/jobs/jobs.yml index 2f992b2c6e..01ada74766 100644 --- a/eng/common/core-templates/jobs/jobs.yml +++ b/eng/common/core-templates/jobs/jobs.yml @@ -43,6 +43,8 @@ parameters: artifacts: {} is1ESPipeline: '' + repositoryAlias: self + officialBuildId: '' # Internal resources (telemetry, microbuild) can only be accessed from non-public projects, # and some (Microbuild) should only be applied to non-PR cases for internal builds. @@ -114,3 +116,5 @@ jobs: enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} signingValidationAdditionalParameters: ${{ parameters.signingValidationAdditionalParameters }} + repositoryAlias: ${{ parameters.repositoryAlias }} + officialBuildId: ${{ parameters.officialBuildId }} diff --git a/eng/common/core-templates/post-build/post-build.yml b/eng/common/core-templates/post-build/post-build.yml index a151fd811e..6f0929c039 100644 --- a/eng/common/core-templates/post-build/post-build.yml +++ b/eng/common/core-templates/post-build/post-build.yml @@ -1,106 +1,106 @@ parameters: - # Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. - # Publishing V1 is no longer supported - # Publishing V2 is no longer supported - # Publishing V3 is the default - - name: publishingInfraVersion - displayName: Which version of publishing should be used to promote the build definition? - type: number - default: 3 - values: - - 3 - - - name: BARBuildId - displayName: BAR Build Id - type: number - default: 0 - - - name: PromoteToChannelIds - displayName: Channel to promote BARBuildId to - type: string - default: '' - - - name: enableSourceLinkValidation - displayName: Enable SourceLink validation - type: boolean - default: false - - - name: enableSigningValidation - displayName: Enable signing validation - type: boolean - default: true - - - name: enableSymbolValidation - displayName: Enable symbol validation - type: boolean - default: false - - - name: enableNugetValidation - displayName: Enable NuGet validation - type: boolean - default: true - - - name: publishInstallersAndChecksums - displayName: Publish installers and checksums - type: boolean - default: true - - - name: requireDefaultChannels - displayName: Fail the build if there are no default channel(s) registrations for the current build - type: boolean - default: false - - - name: SDLValidationParameters - type: object - default: - enable: false - publishGdn: false - continueOnError: false - params: '' - artifactNames: '' - downloadArtifacts: true - - - name: isAssetlessBuild - type: boolean - displayName: Is Assetless Build - default: false - - # These parameters let the user customize the call to sdk-task.ps1 for publishing - # symbols & general artifacts as well as for signing validation - - name: symbolPublishingAdditionalParameters - displayName: Symbol publishing additional parameters - type: string - default: '' - - - name: artifactsPublishingAdditionalParameters - displayName: Artifact publishing additional parameters - type: string - default: '' - - - name: signingValidationAdditionalParameters - displayName: Signing validation additional parameters - type: string - default: '' - - # Which stages should finish execution before post-build stages start - - name: validateDependsOn - type: object - default: - - build - - - name: publishDependsOn - type: object - default: - - Validate - - # Optional: Call asset publishing rather than running in a separate stage - - name: publishAssetsImmediately - type: boolean - default: false - - - name: is1ESPipeline - type: boolean - default: false +# Which publishing infra should be used. THIS SHOULD MATCH THE VERSION ON THE BUILD MANIFEST. +# Publishing V1 is no longer supported +# Publishing V2 is no longer supported +# Publishing V3 is the default +- name: publishingInfraVersion + displayName: Which version of publishing should be used to promote the build definition? + type: number + default: 3 + values: + - 3 + +- name: BARBuildId + displayName: BAR Build Id + type: number + default: 0 + +- name: PromoteToChannelIds + displayName: Channel to promote BARBuildId to + type: string + default: '' + +- name: enableSourceLinkValidation + displayName: Enable SourceLink validation + type: boolean + default: false + +- name: enableSigningValidation + displayName: Enable signing validation + type: boolean + default: true + +- name: enableSymbolValidation + displayName: Enable symbol validation + type: boolean + default: false + +- name: enableNugetValidation + displayName: Enable NuGet validation + type: boolean + default: true + +- name: publishInstallersAndChecksums + displayName: Publish installers and checksums + type: boolean + default: true + +- name: requireDefaultChannels + displayName: Fail the build if there are no default channel(s) registrations for the current build + type: boolean + default: false + +- name: SDLValidationParameters + type: object + default: + enable: false + publishGdn: false + continueOnError: false + params: '' + artifactNames: '' + downloadArtifacts: true + +- name: isAssetlessBuild + type: boolean + displayName: Is Assetless Build + default: false + +# These parameters let the user customize the call to sdk-task.ps1 for publishing +# symbols & general artifacts as well as for signing validation +- name: symbolPublishingAdditionalParameters + displayName: Symbol publishing additional parameters + type: string + default: '' + +- name: artifactsPublishingAdditionalParameters + displayName: Artifact publishing additional parameters + type: string + default: '' + +- name: signingValidationAdditionalParameters + displayName: Signing validation additional parameters + type: string + default: '' + +# Which stages should finish execution before post-build stages start +- name: validateDependsOn + type: object + default: + - build + +- name: publishDependsOn + type: object + default: + - Validate + +# Optional: Call asset publishing rather than running in a separate stage +- name: publishAssetsImmediately + type: boolean + default: false + +- name: is1ESPipeline + type: boolean + default: false stages: - ${{ if or(eq( parameters.enableNugetValidation, 'true'), eq(parameters.enableSigningValidation, 'true'), eq(parameters.enableSourceLinkValidation, 'true'), eq(parameters.SDLValidationParameters.enable, 'true')) }}: @@ -108,10 +108,10 @@ stages: dependsOn: ${{ parameters.validateDependsOn }} displayName: Validate Build Assets variables: - - template: /eng/common/core-templates/post-build/common-variables.yml - - template: /eng/common/core-templates/variables/pool-providers.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} jobs: - job: displayName: NuGet Validation @@ -134,28 +134,28 @@ stages: demands: ImageOverride -equals windows.vs2022.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/nuget-validation.ps1 - arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/nuget-validation.ps1 + arguments: -PackagesPath $(Build.ArtifactStagingDirectory)/PackageArtifacts/ - job: displayName: Signing Validation @@ -169,54 +169,54 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) image: 1es-windows-2022 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 + demands: ImageOverride -equals windows.vs2022.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Package Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: PackageArtifacts - checkDownloadedFiles: true - - # This is necessary whenever we want to publish/restore to an AzDO private feed - # Since sdk-task.ps1 tries to restore packages we need to do this authentication here - # otherwise it'll complain about accessing a private feed. - - task: NuGetAuthenticate@1 - displayName: 'Authenticate to AzDO Feeds' - - # Signing validation will optionally work with the buildmanifest file which is downloaded from - # Azure DevOps above. - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: eng\common\sdk-task.ps1 - arguments: -task SigningValidation -restore -msbuildEngine vs - /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' - /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - ${{ parameters.signingValidationAdditionalParameters }} - - - template: /eng/common/core-templates/steps/publish-logs.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} - StageLabel: 'Validation' - JobLabel: 'Signing' - BinlogToolVersion: $(BinlogToolVersion) + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: PackageArtifacts + checkDownloadedFiles: true + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@1 + displayName: 'Authenticate to AzDO Feeds' + + # Signing validation will optionally work with the buildmanifest file which is downloaded from + # Azure DevOps above. + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task SigningValidation -restore -msbuildEngine vs + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' + /p:SignCheckExclusionsFile='$(System.DefaultWorkingDirectory)/eng/SignCheckExclusionsFile.txt' + ${{ parameters.signingValidationAdditionalParameters }} + + - template: /eng/common/core-templates/steps/publish-logs.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} + StageLabel: 'Validation' + JobLabel: 'Signing' + BinlogToolVersion: $(BinlogToolVersion) - job: displayName: SourceLink Validation @@ -230,41 +230,41 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: name: $(DncEngInternalBuildPool) image: 1es-windows-2022 os: windows ${{ else }}: name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals windows.vs2022.amd64 + demands: ImageOverride -equals windows.vs2022.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - task: DownloadBuildArtifacts@0 - displayName: Download Blob Artifacts - inputs: - buildType: specific - buildVersionToDownload: specific - project: $(AzDOProjectName) - pipeline: $(AzDOPipelineId) - buildId: $(AzDOBuildId) - artifactName: BlobArtifacts - checkDownloadedFiles: true - - - task: PowerShell@2 - displayName: Validate - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/sourcelink-validation.ps1 - arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ - -ExtractPath $(Agent.BuildDirectory)/Extract/ - -GHRepoName $(Build.Repository.Name) - -GHCommit $(Build.SourceVersion) - -SourcelinkCliVersion $(SourceLinkCLIVersion) - continueOnError: true + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: specific + buildVersionToDownload: specific + project: $(AzDOProjectName) + pipeline: $(AzDOPipelineId) + buildId: $(AzDOBuildId) + artifactName: BlobArtifacts + checkDownloadedFiles: true + + - task: PowerShell@2 + displayName: Validate + inputs: + filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/sourcelink-validation.ps1 + arguments: -InputPath $(Build.ArtifactStagingDirectory)/BlobArtifacts/ + -ExtractPath $(Agent.BuildDirectory)/Extract/ + -GHRepoName $(Build.Repository.Name) + -GHCommit $(Build.SourceVersion) + -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true - ${{ if ne(parameters.publishAssetsImmediately, 'true') }}: - stage: publish_using_darc @@ -274,10 +274,10 @@ stages: dependsOn: ${{ parameters.validateDependsOn }} displayName: Publish using Darc variables: - - template: /eng/common/core-templates/post-build/common-variables.yml - - template: /eng/common/core-templates/variables/pool-providers.yml - parameters: - is1ESPipeline: ${{ parameters.is1ESPipeline }} + - template: /eng/common/core-templates/post-build/common-variables.yml + - template: /eng/common/core-templates/variables/pool-providers.yml + parameters: + is1ESPipeline: ${{ parameters.is1ESPipeline }} jobs: - job: displayName: Publish Using Darc @@ -291,30 +291,41 @@ stages: os: windows # If it's not devdiv, it's dnceng ${{ else }}: - ${{ if eq(parameters.is1ESPipeline, true) }}: + ${{ if eq(parameters.is1ESPipeline, true) }}: name: NetCore1ESPool-Publishing-Internal image: windows.vs2019.amd64 os: windows ${{ else }}: name: NetCore1ESPool-Publishing-Internal - demands: ImageOverride -equals windows.vs2019.amd64 + demands: ImageOverride -equals windows.vs2019.amd64 steps: - - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml - parameters: - BARBuildId: ${{ parameters.BARBuildId }} - PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} - is1ESPipeline: ${{ parameters.is1ESPipeline }} - - - task: NuGetAuthenticate@1 - - - task: AzureCLI@2 - displayName: Publish Using Darc - inputs: - azureSubscription: "Darc: Maestro Production" - scriptType: ps - scriptLocation: scriptPath - scriptPath: $(Build.SourcesDirectory)/eng/common/post-build/publish-using-darc.ps1 - arguments: > + - template: /eng/common/core-templates/post-build/setup-maestro-vars.yml + parameters: + BARBuildId: ${{ parameters.BARBuildId }} + PromoteToChannelIds: ${{ parameters.PromoteToChannelIds }} + is1ESPipeline: ${{ parameters.is1ESPipeline }} + + - task: NuGetAuthenticate@1 + + # Populate internal runtime variables. + - template: /eng/common/templates/steps/enable-internal-sources.yml + parameters: + legacyCredential: $(dn-bot-dnceng-artifact-feeds-rw) + + - template: /eng/common/templates/steps/enable-internal-runtimes.yml + + - task: UseDotNet@2 + inputs: + version: 8.0.x + + - task: AzureCLI@2 + displayName: Publish Using Darc + inputs: + azureSubscription: "Darc: Maestro Production" + scriptType: ps + scriptLocation: scriptPath + scriptPath: $(System.DefaultWorkingDirectory)/eng/common/post-build/publish-using-darc.ps1 + arguments: > -BuildId $(BARBuildId) -PublishingInfraVersion ${{ parameters.publishingInfraVersion }} -AzdoToken '$(System.AccessToken)' @@ -323,3 +334,5 @@ stages: -ArtifactsPublishingAdditionalParameters '${{ parameters.artifactsPublishingAdditionalParameters }}' -SymbolPublishingAdditionalParameters '${{ parameters.symbolPublishingAdditionalParameters }}' -SkipAssetsPublishing '${{ parameters.isAssetlessBuild }}' + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' diff --git a/eng/common/core-templates/post-build/setup-maestro-vars.yml b/eng/common/core-templates/post-build/setup-maestro-vars.yml index f7602980db..a7abd58c4b 100644 --- a/eng/common/core-templates/post-build/setup-maestro-vars.yml +++ b/eng/common/core-templates/post-build/setup-maestro-vars.yml @@ -36,7 +36,7 @@ steps: $AzureDevOpsBuildId = $Env:Build_BuildId } else { - . $(Build.SourcesDirectory)\eng\common\tools.ps1 + . $(System.DefaultWorkingDirectory)\eng\common\tools.ps1 $darc = Get-Darc $buildInfo = & $darc get-build ` --id ${{ parameters.BARBuildId }} ` diff --git a/eng/common/core-templates/steps/enable-internal-sources.yml b/eng/common/core-templates/steps/enable-internal-sources.yml index 64f881bffc..4085512b69 100644 --- a/eng/common/core-templates/steps/enable-internal-sources.yml +++ b/eng/common/core-templates/steps/enable-internal-sources.yml @@ -17,8 +17,8 @@ steps: - task: PowerShell@2 displayName: Setup Internal Feeds inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token + filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $Env:Token env: Token: ${{ parameters.legacyCredential }} # If running on dnceng (internal project), just use the default behavior for NuGetAuthenticate. @@ -29,8 +29,8 @@ steps: - task: PowerShell@2 displayName: Setup Internal Feeds inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config + filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config - ${{ else }}: - template: /eng/common/templates/steps/get-federated-access-token.yml parameters: @@ -39,8 +39,8 @@ steps: - task: PowerShell@2 displayName: Setup Internal Feeds inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1 - arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token) + filePath: $(System.DefaultWorkingDirectory)/eng/common/SetupNugetSources.ps1 + arguments: -ConfigFile $(System.DefaultWorkingDirectory)/NuGet.config -Password $(dnceng-artifacts-feeds-read-access-token) # This is required in certain scenarios to install the ADO credential provider. # It installed by default in some msbuild invocations (e.g. VS msbuild), but needs to be installed for others # (e.g. dotnet msbuild). diff --git a/eng/common/core-templates/steps/generate-sbom.yml b/eng/common/core-templates/steps/generate-sbom.yml index 44a9636cdf..003f7eae0f 100644 --- a/eng/common/core-templates/steps/generate-sbom.yml +++ b/eng/common/core-templates/steps/generate-sbom.yml @@ -5,8 +5,8 @@ # IgnoreDirectories - Directories to ignore for SBOM generation. This will be passed through to the CG component detector. parameters: - PackageVersion: 10.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + PackageVersion: 11.0.0 + BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' PackageName: '.NET' ManifestDirPath: $(Build.ArtifactStagingDirectory)/sbom IgnoreDirectories: '' diff --git a/eng/common/core-templates/steps/install-microbuild-impl.yml b/eng/common/core-templates/steps/install-microbuild-impl.yml new file mode 100644 index 0000000000..b9e0143ee9 --- /dev/null +++ b/eng/common/core-templates/steps/install-microbuild-impl.yml @@ -0,0 +1,34 @@ +parameters: + - name: microbuildTaskInputs + type: object + default: {} + + - name: microbuildEnv + type: object + default: {} + + - name: enablePreviewMicrobuild + type: boolean + default: false + + - name: condition + type: string + + - name: continueOnError + type: boolean + +steps: +- ${{ if eq(parameters.enablePreviewMicrobuild, 'true') }}: + - task: MicroBuildSigningPluginPreview@4 + displayName: Install Preview MicroBuild plugin + inputs: ${{ parameters.microbuildTaskInputs }} + env: ${{ parameters.microbuildEnv }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} +- ${{ else }}: + - task: MicroBuildSigningPlugin@4 + displayName: Install MicroBuild plugin + inputs: ${{ parameters.microbuildTaskInputs }} + env: ${{ parameters.microbuildEnv }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} diff --git a/eng/common/core-templates/steps/install-microbuild.yml b/eng/common/core-templates/steps/install-microbuild.yml index da30e67bc3..4f4b56ed2a 100644 --- a/eng/common/core-templates/steps/install-microbuild.yml +++ b/eng/common/core-templates/steps/install-microbuild.yml @@ -4,6 +4,8 @@ parameters: # Enable install tasks for MicroBuild on Mac and Linux # Will be ignored if 'enableMicrobuild' is false or 'Agent.Os' is 'Windows_NT' enableMicrobuildForMacAndLinux: false + # Enable preview version of MB signing plugin + enablePreviewMicrobuild: false # Determines whether the ESRP service connection information should be passed to the signing plugin. # This overlaps with _SignType to some degree. We only need the service connection for real signing. # It's important that the service connection not be passed to the MicroBuildSigningPlugin task in this place. @@ -11,8 +13,10 @@ parameters: # Unfortunately, _SignType can't be used to exclude the use of the service connection in non-real sign scenarios. The # variable is not available in template expression. _SignType has a very large proliferation across .NET, so replacing it is tough. microbuildUseESRP: true - # Location of the MicroBuild output folder - microBuildOutputFolder: '$(Build.SourcesDirectory)' + # Microbuild installation directory + microBuildOutputFolder: $(Agent.TempDirectory)/MicroBuild + # Microbuild version + microbuildPluginVersion: 'latest' continueOnError: false @@ -25,8 +29,27 @@ steps: inputs: packageType: sdk version: 8.0.x - installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet - workingDirectory: ${{ parameters.microBuildOutputFolder }} + installationPath: ${{ parameters.microBuildOutputFolder }}/.dotnet-microbuild + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) + + - script: | + set -euo pipefail + + # UseDotNet@2 prepends the dotnet executable path to the PATH variable, so we can call dotnet directly + version=$(dotnet --version) + cat << 'EOF' > ${{ parameters.microBuildOutputFolder }}/global.json + { + "sdk": { + "version": "$version", + "paths": [ + "${{ parameters.microBuildOutputFolder }}/.dotnet-microbuild" + ], + "errorMessage": "The .NET SDK version $version is required to install the MicroBuild signing plugin." + } + } + EOF + displayName: 'Add global.json to MicroBuild Installation path' + workingDirectory: ${{ parameters.microBuildOutputFolder }} condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) - script: | @@ -46,35 +69,50 @@ steps: displayName: 'Validate ESRP usage (Non-Windows)' condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT')) - - task: MicroBuildSigningPlugin@4 - displayName: Install MicroBuild plugin - inputs: - signType: $(_SignType) - zipSources: false - feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json - ${{ if eq(parameters.microbuildUseESRP, true) }}: - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, 'true') }}: - azureSubscription: 'MicroBuild Signing Task (DevDiv)' - useEsrpCli: true - ${{ elseif eq(variables['System.TeamProject'], 'DevDiv') }}: - ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea - ${{ else }}: - ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca - env: - TeamName: $(_TeamName) - MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} - SYSTEM_ACCESSTOKEN: $(System.AccessToken) - continueOnError: ${{ parameters.continueOnError }} - condition: and( - succeeded(), - or( - and( - eq(variables['Agent.Os'], 'Windows_NT'), - in(variables['_SignType'], 'real', 'test') - ), - and( - ${{ eq(parameters.enableMicrobuildForMacAndLinux, true) }}, - ne(variables['Agent.Os'], 'Windows_NT'), - eq(variables['_SignType'], 'real') - ) - )) + # Two different MB install steps. This is due to not being able to use the agent OS during + # YAML expansion, and Windows vs. Linux/Mac uses different service connections. However, + # we can avoid including the MB install step if not enabled at all. This avoids a bunch of + # extra pipeline authorizations, since most pipelines do not sign on non-Windows. + - template: /eng/common/core-templates/steps/install-microbuild-impl.yml@self + parameters: + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildTaskInputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + version: ${{ parameters.microbuildPluginVersion }} + ${{ if eq(parameters.microbuildUseESRP, true) }}: + ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + ConnectedPMEServiceName: 6cc74545-d7b9-4050-9dfa-ebefcc8961ea + ${{ else }}: + ConnectedPMEServiceName: 248d384a-b39b-46e3-8ad5-c2c210d5e7ca + microbuildEnv: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['Agent.Os'], 'Windows_NT'), in(variables['_SignType'], 'real', 'test')) + + - ${{ if eq(parameters.enableMicrobuildForMacAndLinux, true) }}: + - template: /eng/common/core-templates/steps/install-microbuild-impl.yml@self + parameters: + enablePreviewMicrobuild: ${{ parameters.enablePreviewMicrobuild }} + microbuildTaskInputs: + signType: $(_SignType) + zipSources: false + feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json + version: ${{ parameters.microbuildPluginVersion }} + workingDirectory: ${{ parameters.microBuildOutputFolder }} + ${{ if eq(parameters.microbuildUseESRP, true) }}: + ConnectedServiceName: 'MicroBuild Signing Task (DevDiv)' + ${{ if eq(variables['System.TeamProject'], 'DevDiv') }}: + ConnectedPMEServiceName: beb8cb23-b303-4c95-ab26-9e44bc958d39 + ${{ else }}: + ConnectedPMEServiceName: c24de2a5-cc7a-493d-95e4-8e5ff5cad2bc + microbuildEnv: + TeamName: $(_TeamName) + MicroBuildOutputFolderOverride: ${{ parameters.microBuildOutputFolder }} + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), ne(variables['Agent.Os'], 'Windows_NT'), eq(variables['_SignType'], 'real')) diff --git a/eng/common/core-templates/steps/publish-logs.yml b/eng/common/core-templates/steps/publish-logs.yml index de24d0087c..5a927b4c7b 100644 --- a/eng/common/core-templates/steps/publish-logs.yml +++ b/eng/common/core-templates/steps/publish-logs.yml @@ -12,22 +12,24 @@ steps: inputs: targetType: inline script: | - New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ - Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + New-Item -ItemType Directory $(System.DefaultWorkingDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(System.DefaultWorkingDirectory)/artifacts/log/Debug/* $(System.DefaultWorkingDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ continueOnError: true condition: always() - task: PowerShell@2 displayName: Redact Logs inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1 + filePath: $(System.DefaultWorkingDirectory)/eng/common/post-build/redact-logs.ps1 # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml - # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + # Sensitive data can as well be added to $(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' # If the file exists - sensitive data for redaction will be sourced from it # (single entry per line, lines starting with '# ' are considered comments and skipped) - arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' - -BinlogToolVersion ${{parameters.BinlogToolVersion}} - -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt' + arguments: -InputPath '$(System.DefaultWorkingDirectory)/PostBuildLogs' + -BinlogToolVersion '${{parameters.BinlogToolVersion}}' + -TokensFilePath '$(System.DefaultWorkingDirectory)/eng/BinlogSecretsRedactionFile.txt' + -runtimeSourceFeed https://ci.dot.net/internal + -runtimeSourceFeedKey '$(dotnetbuilds-internal-container-read-token-base64)' '$(publishing-dnceng-devdiv-code-r-build-re)' '$(MaestroAccessToken)' '$(dn-bot-all-orgs-artifact-feeds-rw)' @@ -44,7 +46,7 @@ steps: - task: CopyFiles@2 displayName: Gather post build logs inputs: - SourceFolder: '$(Build.SourcesDirectory)/PostBuildLogs' + SourceFolder: '$(System.DefaultWorkingDirectory)/PostBuildLogs' Contents: '**' TargetFolder: '$(Build.ArtifactStagingDirectory)/PostBuildLogs' condition: always() diff --git a/eng/common/core-templates/steps/source-index-stage1-publish.yml b/eng/common/core-templates/steps/source-index-stage1-publish.yml index c2917c1efc..ac019e2d03 100644 --- a/eng/common/core-templates/steps/source-index-stage1-publish.yml +++ b/eng/common/core-templates/steps/source-index-stage1-publish.yml @@ -1,6 +1,6 @@ parameters: - sourceIndexUploadPackageVersion: 2.0.0-20250425.2 - sourceIndexProcessBinlogPackageVersion: 1.0.1-20250515.1 + sourceIndexUploadPackageVersion: 2.0.0-20250906.1 + sourceIndexProcessBinlogPackageVersion: 1.0.1-20250906.1 sourceIndexPackageSource: https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json binlogPath: artifacts/log/Debug/Build.binlog @@ -14,13 +14,13 @@ steps: workingDirectory: $(Agent.TempDirectory) - script: | - $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools - $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --add-source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install BinLogToSln --version ${{parameters.sourceIndexProcessBinlogPackageVersion}} --source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools + $(Agent.TempDirectory)/dotnet/dotnet tool install UploadIndexStage1 --version ${{parameters.sourceIndexUploadPackageVersion}} --source ${{parameters.SourceIndexPackageSource}} --tool-path $(Agent.TempDirectory)/.source-index/tools displayName: "Source Index: Download netsourceindex Tools" # Set working directory to temp directory so 'dotnet' doesn't try to use global.json and use the repo's sdk. workingDirectory: $(Agent.TempDirectory) -- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(Build.SourcesDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output +- script: $(Agent.TempDirectory)/.source-index/tools/BinLogToSln -i ${{parameters.BinlogPath}} -r $(System.DefaultWorkingDirectory) -n $(Build.Repository.Name) -o .source-index/stage1output displayName: "Source Index: Process Binlog into indexable sln" - ${{ if and(ne(parameters.runAsPublic, 'true'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index e889f439b8..9f5ad6b763 100644 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -5,7 +5,7 @@ darcVersion='' versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2020-02-20' verbosity='minimal' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --darcversion) diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 7b9d97e3bd..61f302bb67 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -18,7 +18,7 @@ architecture='' runtime='dotnet' runtimeSourceFeed='' runtimeSourceFeedKey='' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in -version|-v) diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh index 2ef6823567..f6d24871c1 100644 --- a/eng/common/dotnet.sh +++ b/eng/common/dotnet.sh @@ -19,7 +19,7 @@ source $scriptroot/tools.sh InitializeDotNetCli true # install # Invoke acquired SDK with args if they are provided -if [[ $# > 0 ]]; then +if [[ $# -gt 0 ]]; then __dotnetDir=${_InitializeDotNetCli} dotnetPath=${__dotnetDir}/dotnet ${dotnetPath} "$@" diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index 524aaa57f2..fa1cdc2b30 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -33,15 +33,27 @@ $jsonTemplateFiles | ForEach-Object { $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern +$wxlFilesV3 = @() +$wxlFilesV5 = @() $wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them if (-not $wxlFiles) { $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files if ($wxlEnFiles) { - $wxlFiles = @() - $wxlEnFiles | ForEach-Object { - $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" - $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru - } + $wxlFiles = @() + $wxlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $content = Get-Content $_.FullName -Raw + + # Split files on schema to select different parser settings in the generated project. + if ($content -like "*http://wixtoolset.org/schemas/v4/wxl*") + { + $wxlFilesV5 += Copy-Item $_.FullName -Destination $destinationFile -PassThru + } + elseif ($content -like "*http://schemas.microsoft.com/wix/2006/localization*") + { + $wxlFilesV3 += Copy-Item $_.FullName -Destination $destinationFile -PassThru + } + } } } @@ -114,7 +126,32 @@ $locJson = @{ CloneLanguageSet = "WiX_CloneLanguages" LssFiles = @( "wxl_loc.lss" ) LocItems = @( - $wxlFiles | ForEach-Object { + $wxlFilesV3 | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) + { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + } + } + ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "WiX_CloneLanguages" + LssFiles = @( "P210WxlSchemaV4.lss" ) + LocItems = @( + $wxlFilesV5 | ForEach-Object { $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" $continue = $true foreach ($exclusion in $exclusions.Exclusions) { diff --git a/eng/common/internal-feed-operations.sh b/eng/common/internal-feed-operations.sh index 9378223ba0..6299e7effd 100644 --- a/eng/common/internal-feed-operations.sh +++ b/eng/common/internal-feed-operations.sh @@ -100,7 +100,7 @@ operation='' authToken='' repoName='' -while [[ $# > 0 ]]; do +while [[ $# -gt 0 ]]; do opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case "$opt" in --operation) diff --git a/eng/common/native/install-dependencies.sh b/eng/common/native/install-dependencies.sh index 477a44f335..64b87d0bcc 100644 --- a/eng/common/native/install-dependencies.sh +++ b/eng/common/native/install-dependencies.sh @@ -27,9 +27,11 @@ case "$os" in libssl-dev libkrb5-dev pigz cpio localedef -i en_US -c -f UTF-8 -A /usr/share/locale/locale.alias en_US.UTF-8 - elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ]; then + elif [ "$ID" = "fedora" ] || [ "$ID" = "rhel" ] || [ "$ID" = "azurelinux" ] || [ "$ID" = "centos"]; then pkg_mgr="$(command -v tdnf 2>/dev/null || command -v dnf)" $pkg_mgr install -y cmake llvm lld lldb clang python curl libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio + elif [ "$ID" = "amzn" ]; then + dnf install -y cmake llvm lld lldb clang python libicu-devel openssl-devel krb5-devel lttng-ust-devel pigz cpio elif [ "$ID" = "alpine" ]; then apk add build-base cmake bash curl clang llvm-dev lld lldb krb5-dev lttng-ust-dev icu-dev openssl-dev pigz cpio else diff --git a/eng/common/post-build/publish-using-darc.ps1 b/eng/common/post-build/publish-using-darc.ps1 index 1eda208a3b..48e55598bd 100644 --- a/eng/common/post-build/publish-using-darc.ps1 +++ b/eng/common/post-build/publish-using-darc.ps1 @@ -7,7 +7,9 @@ param( [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters, [Parameter(Mandatory=$false)][string] $RequireDefaultChannels, - [Parameter(Mandatory=$false)][string] $SkipAssetsPublishing + [Parameter(Mandatory=$false)][string] $SkipAssetsPublishing, + [Parameter(Mandatory=$false)][string] $runtimeSourceFeed, + [Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey ) try { diff --git a/eng/common/post-build/redact-logs.ps1 b/eng/common/post-build/redact-logs.ps1 index b7fc195915..fc0218a013 100644 --- a/eng/common/post-build/redact-logs.ps1 +++ b/eng/common/post-build/redact-logs.ps1 @@ -7,7 +7,9 @@ param( # File with strings to redact - separated by newlines. # For comments start the line with '# ' - such lines are ignored [Parameter(Mandatory=$false)][string] $TokensFilePath, - [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact + [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact, + [Parameter(Mandatory=$false)][string] $runtimeSourceFeed, + [Parameter(Mandatory=$false)][string] $runtimeSourceFeedKey ) try { diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index a9d2a2d269..9ae443f1c3 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -7,13 +7,16 @@ Param( [switch] $restore, [switch] $prepareMachine, [switch][Alias('nobl')]$excludeCIBinaryLog, + [switch]$noWarnAsError, [switch] $help, + [string] $runtimeSourceFeed = '', + [string] $runtimeSourceFeedKey = '', [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) $ci = $true $binaryLog = if ($excludeCIBinaryLog) { $false } else { $true } -$warnAsError = $true +$warnAsError = if ($noWarnAsError) { $false } else { $true } . $PSScriptRoot\tools.ps1 @@ -67,7 +70,7 @@ try { $GlobalJson.tools | Add-Member -Name "vs" -Value (ConvertFrom-Json "{ `"version`": `"16.5`" }") -MemberType NoteProperty } if( -not ($GlobalJson.tools.PSObject.Properties.Name -match "xcopy-msbuild" )) { - $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.13.0" -MemberType NoteProperty + $GlobalJson.tools | Add-Member -Name "xcopy-msbuild" -Value "17.14.16" -MemberType NoteProperty } if ($GlobalJson.tools."xcopy-msbuild".Trim() -ine "none") { $xcopyMSBuildToolsFolder = InitializeXCopyMSBuild $GlobalJson.tools."xcopy-msbuild" -install $true diff --git a/eng/common/sdk-task.sh b/eng/common/sdk-task.sh index 2f83adc026..3270f83fa9 100644 --- a/eng/common/sdk-task.sh +++ b/eng/common/sdk-task.sh @@ -10,6 +10,7 @@ show_usage() { echo "Advanced settings:" echo " --excludeCIBinarylog Don't output binary log (short: -nobl)" + echo " --noWarnAsError Do not warn as error" echo "" echo "Command line arguments not listed above are passed thru to msbuild." } @@ -52,6 +53,7 @@ exclude_ci_binary_log=false restore=false help=false properties='' +warnAsError=true while (($# > 0)); do lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" @@ -73,6 +75,10 @@ while (($# > 0)); do exclude_ci_binary_log=true shift 1 ;; + --noWarnAsError) + warnAsError=false + shift 1 + ;; --help) help=true shift 1 @@ -85,7 +91,6 @@ while (($# > 0)); do done ci=true -warnAsError=true if $help; then show_usage diff --git a/eng/common/template-guidance.md b/eng/common/template-guidance.md index 98bbc1ded0..4bf4cf41bd 100644 --- a/eng/common/template-guidance.md +++ b/eng/common/template-guidance.md @@ -50,7 +50,7 @@ extends: - task: CopyFiles@2 displayName: Gather build output inputs: - SourceFolder: '$(Build.SourcesDirectory)/artifacts/marvel' + SourceFolder: '$(System.DefaultWorkingDirectory)/artifacts/marvel' Contents: '**' TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/marvel' ``` diff --git a/eng/common/templates-official/job/job.yml b/eng/common/templates-official/job/job.yml index a8a9432874..92a0664f56 100644 --- a/eng/common/templates-official/job/job.yml +++ b/eng/common/templates-official/job/job.yml @@ -3,7 +3,7 @@ parameters: enableSbom: true runAsPublic: false PackageVersion: 9.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' jobs: - template: /eng/common/core-templates/job/job.yml diff --git a/eng/common/templates-official/variables/sdl-variables.yml b/eng/common/templates-official/variables/sdl-variables.yml index dbdd66d4a4..f1311bbb1b 100644 --- a/eng/common/templates-official/variables/sdl-variables.yml +++ b/eng/common/templates-official/variables/sdl-variables.yml @@ -4,4 +4,4 @@ variables: - name: DefaultGuardianVersion value: 0.109.0 - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file + value: $(System.DefaultWorkingDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 7cbf668c22..238fa0818f 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -6,7 +6,7 @@ parameters: enableSbom: true runAsPublic: false PackageVersion: 9.0.0 - BuildDropPath: '$(Build.SourcesDirectory)/artifacts' + BuildDropPath: '$(System.DefaultWorkingDirectory)/artifacts' jobs: - template: /eng/common/core-templates/job/job.yml @@ -77,7 +77,7 @@ jobs: parameters: is1ESPipeline: false args: - targetPath: '$(Build.SourcesDirectory)\eng\common\BuildConfiguration' + targetPath: '$(System.DefaultWorkingDirectory)\eng\common\BuildConfiguration' artifactName: 'BuildConfiguration' displayName: 'Publish build retry configuration' continueOnError: true diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index d4cfd9ccd8..4bc50bd568 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -394,8 +394,8 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = # If the version of msbuild is going to be xcopied, # use this version. Version matches a package here: - # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.13.0 - $defaultXCopyMSBuildVersion = '17.13.0' + # https://dev.azure.com/dnceng/public/_artifacts/feed/dotnet-eng/NuGet/Microsoft.DotNet.Arcade.MSBuild.Xcopy/versions/17.14.16 + $defaultXCopyMSBuildVersion = '17.14.16' if (!$vsRequirements) { if (Get-Member -InputObject $GlobalJson.tools -Name 'vs') { @@ -544,7 +544,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (Get-Member -InputObject $GlobalJson.tools -Name 'vswhere') { $vswhereVersion = $GlobalJson.tools.vswhere } else { - $vswhereVersion = '2.5.2' + # keep this in sync with the VSWhereVersion in DefaultVersions.props + $vswhereVersion = '3.1.7' } $vsWhereDir = Join-Path $ToolsDir "vswhere\$vswhereVersion" @@ -552,7 +553,8 @@ function LocateVisualStudio([object]$vsRequirements = $null){ if (!(Test-Path $vsWhereExe)) { Create-Directory $vsWhereDir - Write-Host 'Downloading vswhere' + Write-Host "Downloading vswhere $vswhereVersion" + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Retry({ Invoke-WebRequest "https://netcorenativeassets.blob.core.windows.net/resource-packages/external/windows/vswhere/$vswhereVersion/vswhere.exe" -OutFile $vswhereExe }) diff --git a/global.json b/global.json index 5ebef2be03..b8b04bb6c1 100644 --- a/global.json +++ b/global.json @@ -1,6 +1,6 @@ { "sdk": { - "version": "10.0.100-preview.7.25372.107", + "version": "10.0.200-preview.0.25568.101", "paths": [ ".dotnet", "$host$" @@ -34,9 +34,9 @@ "vs": { "version": "17.8.0" }, - "dotnet": "10.0.100-preview.7.25372.107" + "dotnet": "10.0.200-preview.0.25568.101" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "10.0.0-beta.25408.3" + "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.25571.8" } } diff --git a/restore.cmd b/restore.cmd index 77649cdb4d..ba6af7d4f3 100644 --- a/restore.cmd +++ b/restore.cmd @@ -1,2 +1,2 @@ @echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -restore %*" +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\Build.ps1""" -restore %*" diff --git a/test.cmd b/test.cmd index db9e8e061d..c879b9d2c3 100644 --- a/test.cmd +++ b/test.cmd @@ -1,3 +1,3 @@ @echo off -powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\common\Build.ps1""" -test %*" +powershell -ExecutionPolicy ByPass -NoProfile -command "& """%~dp0eng\Build.ps1""" -test %*" exit /b %ErrorLevel% diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/Build.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/Build.cs index 8dd581b00f..67e66a577d 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/Build.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/Build.cs @@ -83,7 +83,7 @@ private static void BuildTestAssetsCompatibility() var nugetCache = Path.GetFullPath(Path.Combine(Root, ".packages")); var generated = Path.GetFullPath(Path.Combine(Root, "artifacts", "tmp", "GeneratedTestAssets")); - var generatedSln = Path.Combine(generated, "CompatibilityTestAssets.sln"); + var generatedSln = Path.Combine(generated, "CompatibilityTestAssets.slnx"); var dependenciesPath = Path.Combine(Root, "eng", "Versions.props"); var dependenciesXml = XDocument.Load(dependenciesPath); diff --git a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs index 86f0dd308c..621ca677eb 100644 --- a/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs +++ b/test/Microsoft.TestPlatform.Acceptance.IntegrationTests/DotnetTestTests.cs @@ -26,7 +26,7 @@ public void RunDotnetTestWithCsproj(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var projectPath = GetIsolatedTestAsset("SimpleTestProject.csproj"); - InvokeDotnetTest($@"{projectPath} -p:VSTestUseMSBuildOutput=false --logger:""Console;Verbosity=normal"" /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} -tl:off /p:VSTestNoLogo=false --logger:""Console;Verbosity=normal"" /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion}", workingDirectory: Path.GetDirectoryName(projectPath)); // ensure our dev version is used StdOutputContains(GetFinalVersion(IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion)); @@ -60,8 +60,10 @@ public void RunDotnetTestWithCsprojPassInlineSettings(RunnerInfo runnerInfo) SetTestEnvironment(_testEnvironment, runnerInfo); var projectPath = GetIsolatedTestAsset("ParametrizedTestProject.csproj"); - InvokeDotnetTest($@"{projectPath} --logger:""Console;Verbosity=normal"" -p:VSTestUseMSBuildOutput=false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion} -- TestRunParameters.Parameter(name =\""weburl\"", value=\""http://localhost//def\"")", workingDirectory: Path.GetDirectoryName(projectPath)); + InvokeDotnetTest($@"{projectPath} --logger:""Console;Verbosity=normal"" -tl:off /p:VSTestNoLogo=false /p:PackageVersion={IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion} -- TestRunParameters.Parameter(name =\""weburl\"", value=\""http://localhost//def\"")", workingDirectory: Path.GetDirectoryName(projectPath)); + // ensure our dev version is used + StdOutputContains(GetFinalVersion(IntegrationTestEnvironment.LatestLocallyBuiltNugetVersion)); ValidateSummaryStatus(1, 0, 0); ExitCodeEquals(0); }