From 34b61952a9c613a643b74a73caffc54b0499c395 Mon Sep 17 00:00:00 2001 From: Andriy Svyryd Date: Thu, 6 Aug 2026 14:40:23 -0700 Subject: [PATCH] Add a script to only retry failed AzDo jobs --- azure-pipelines-internal-tests.yml | 78 +++++++------- azure-pipelines-public.yml | 76 +++++++------- eng/aggregate-azdo-tests.ps1 | 159 +++++++++++++++++++++++++++++ 3 files changed, 232 insertions(+), 81 deletions(-) create mode 100644 eng/aggregate-azdo-tests.ps1 diff --git a/azure-pipelines-internal-tests.yml b/azure-pipelines-internal-tests.yml index da4325a67e7..a1301f96691 100644 --- a/azure-pipelines-internal-tests.yml +++ b/azure-pipelines-internal-tests.yml @@ -1,3 +1,8 @@ +parameters: + - name: jobs + type: object + default: [] + variables: - name: _BuildConfig value: Release @@ -103,11 +108,12 @@ extends: enablePublishBuildAssets: false enablePublishTestResults: true publishAssetsImmediately: true - isAssetlessBuild: true + isAssetlessBuild: ${{ eq(length(parameters.jobs), 0) }} enableTelemetry: true helixRepo: dotnet/efcore jobs: - job: Windows + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Windows')) }} pool: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals 1es-windows-2022 @@ -160,6 +166,7 @@ extends: - job: Windows_SqlServer displayName: 'Windows SQL Server' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Windows_SqlServer')) }} pool: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals 1es-windows-2022 @@ -195,6 +202,7 @@ extends: continueOnError: true - job: macOS + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'macOS')) }} pool: name: Azure Pipelines image: macOS-15 @@ -230,6 +238,7 @@ extends: condition: always() continueOnError: true - job: Linux + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Linux')) }} timeoutInMinutes: 180 pool: name: $(DncEngInternalBuildPool) @@ -269,6 +278,7 @@ extends: - job: Helix_Windows displayName: 'Helix Windows' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows')) }} timeoutInMinutes: 180 pool: name: $(DncEngInternalBuildPool) @@ -311,6 +321,7 @@ extends: - job: Helix_Windows_SqlServer displayName: 'Helix Windows SQL Server' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_SqlServer')) }} timeoutInMinutes: 180 pool: name: $(DncEngInternalBuildPool) @@ -353,6 +364,7 @@ extends: - job: Helix_Windows_Arm64 displayName: 'Helix Windows ARM64' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_Arm64')) }} timeoutInMinutes: 180 pool: name: $(DncEngInternalBuildPool) @@ -395,6 +407,7 @@ extends: - job: Helix_Windows_Cosmos displayName: 'Helix Windows Cosmos' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_Cosmos')) }} timeoutInMinutes: 180 pool: name: $(DncEngInternalBuildPool) @@ -437,6 +450,7 @@ extends: - job: Helix_macOS_x64 displayName: 'Helix macOS x64' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_macOS_x64')) }} timeoutInMinutes: 180 pool: name: $(DncEngInternalBuildPool) @@ -479,6 +493,7 @@ extends: - job: Helix_macOS_ARM64 displayName: 'Helix macOS ARM64' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_macOS_ARM64')) }} timeoutInMinutes: 180 pool: name: $(DncEngInternalBuildPool) @@ -521,6 +536,7 @@ extends: - job: Helix_Ubuntu_SqlServer displayName: 'Helix Ubuntu SQL Server' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu_SqlServer')) }} timeoutInMinutes: 180 pool: name: $(DncEngInternalBuildPool) @@ -564,6 +580,7 @@ extends: - job: Helix_Ubuntu_Cosmos displayName: 'Helix Ubuntu Cosmos' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu_Cosmos')) }} timeoutInMinutes: 180 pool: name: $(DncEngInternalBuildPool) @@ -606,6 +623,7 @@ extends: - job: Helix_Ubuntu displayName: 'Helix Ubuntu' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu')) }} timeoutInMinutes: 180 pool: name: $(DncEngInternalBuildPool) @@ -652,6 +670,7 @@ extends: jobs: - job: Validate_Job_Results displayName: Job Results + timeoutInMinutes: 200 pool: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals 1es-windows-2022 @@ -672,46 +691,23 @@ extends: HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] steps: - pwsh: | - $groupResults = @{ - Windows = @("$(WindowsResult)", "$(HelixWindowsResult)") - Linux = @("$(LinuxResult)", "$(HelixUbuntuResult)") - MacOS = @("$(MacOsResult)", "$(HelixMacOsX64Result)") - Arm64 = @("$(HelixWindowsArm64Result)", "$(HelixMacOsArm64Result)") - Cosmos = @("$(HelixWindowsCosmosResult)", "$(HelixUbuntuCosmosResult)") - SqlServer = @("$(WindowsSqlServerResult)", "$(HelixWindowsSqlServerResult)", "$(HelixUbuntuSqlServerResult)") - } - - $failedGroups = @() - - foreach ($groupName in $groupResults.Keys) - { - $results = $groupResults[$groupName] - Write-Host "$groupName results: $($results -join ', ')" - - # Ignore jobs that were skipped (e.g. condition-gated) - $ran = $results | Where-Object { $_ -ne 'Skipped' } - if (-not $ran) - { - Write-Host " -> all jobs skipped, treating group as successful" - continue - } - - # Only 'Succeeded' counts as success. 'SucceededWithIssues' is treated as failure - # because the jobs run with continueOnError: true, which maps task failures to warnings. - $hasSuccess = $ran | Where-Object { $_ -eq 'Succeeded' } - if ($hasSuccess) - { - continue - } - - $failedGroups += $groupName - } - - if ($failedGroups.Count -gt 0) - { - Write-Error "No jobs succeeded for group(s): $($failedGroups -join ', ')." - exit 1 + $jobResults = @{ + Windows = "$(WindowsResult)" + Helix_Windows = "$(HelixWindowsResult)" + Windows_SqlServer = "$(WindowsSqlServerResult)" + Helix_Windows_SqlServer = "$(HelixWindowsSqlServerResult)" + Helix_Windows_Cosmos = "$(HelixWindowsCosmosResult)" + Linux = "$(LinuxResult)" + Helix_Ubuntu = "$(HelixUbuntuResult)" + Helix_Ubuntu_SqlServer = "$(HelixUbuntuSqlServerResult)" + macOS = "$(MacOsResult)" + Helix_macOS_x64 = "$(HelixMacOsX64Result)" + Helix_macOS_ARM64 = "$(HelixMacOsArm64Result)" + Helix_Windows_Arm64 = "$(HelixWindowsArm64Result)" + Helix_Ubuntu_Cosmos = "$(HelixUbuntuCosmosResult)" } - Write-Host 'Group validation passed.' + & eng/aggregate-azdo-tests.ps1 -JobResults $jobResults displayName: Evaluate grouped job outcomes + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/azure-pipelines-public.yml b/azure-pipelines-public.yml index fd68cf6c873..5bcb97d211a 100644 --- a/azure-pipelines-public.yml +++ b/azure-pipelines-public.yml @@ -1,3 +1,8 @@ +parameters: + - name: jobs + type: object + default: [] + variables: - name: _BuildConfig value: Release @@ -52,6 +57,7 @@ stages: helixRepo: dotnet/efcore jobs: - job: Windows + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Windows')) }} enablePublishTestResults: true timeoutInMinutes: 90 pool: @@ -113,6 +119,7 @@ stages: - job: Windows_SqlServer displayName: 'Windows SQL Server' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Windows_SqlServer')) }} enablePublishTestResults: true timeoutInMinutes: 120 pool: @@ -156,6 +163,7 @@ stages: parallel: true - job: macOS + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'macOS')) }} enablePublishTestResults: true pool: vmImage: macOS-15 @@ -198,6 +206,7 @@ stages: parallel: true - job: Linux + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Linux')) }} timeoutInMinutes: 120 enablePublishTestResults: true pool: @@ -241,6 +250,7 @@ stages: - job: Helix_Windows displayName: 'Helix Windows' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows')) }} timeoutInMinutes: 180 pool: name: $(DncEngPublicBuildPool) @@ -271,6 +281,7 @@ stages: - job: Helix_Windows_SqlServer displayName: 'Helix Windows SQL Server' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_SqlServer')) }} timeoutInMinutes: 180 pool: name: $(DncEngPublicBuildPool) @@ -301,6 +312,7 @@ stages: - job: Helix_Windows_Arm64 displayName: 'Helix Windows ARM64' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_Arm64')) }} timeoutInMinutes: 180 pool: name: $(DncEngPublicBuildPool) @@ -331,6 +343,7 @@ stages: - job: Helix_Windows_Cosmos displayName: 'Helix Windows Cosmos' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_Cosmos')) }} timeoutInMinutes: 180 pool: name: $(DncEngPublicBuildPool) @@ -361,6 +374,7 @@ stages: - job: Helix_macOS_x64 displayName: 'Helix macOS x64' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_macOS_x64')) }} timeoutInMinutes: 180 pool: name: $(DncEngPublicBuildPool) @@ -391,6 +405,7 @@ stages: - job: Helix_macOS_ARM64 displayName: 'Helix macOS ARM64' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_macOS_ARM64')) }} timeoutInMinutes: 180 pool: name: $(DncEngPublicBuildPool) @@ -421,6 +436,7 @@ stages: - job: Helix_Ubuntu_SqlServer displayName: 'Helix Ubuntu SQL Server' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu_SqlServer')) }} timeoutInMinutes: 180 pool: name: $(DncEngPublicBuildPool) @@ -452,6 +468,7 @@ stages: - job: Helix_Ubuntu_Cosmos displayName: 'Helix Ubuntu Cosmos' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu_Cosmos')) }} timeoutInMinutes: 180 pool: name: $(DncEngPublicBuildPool) @@ -482,6 +499,7 @@ stages: - job: Helix_Ubuntu displayName: 'Helix Ubuntu' + condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu')) }} timeoutInMinutes: 180 pool: name: $(DncEngPublicBuildPool) @@ -516,6 +534,7 @@ stages: jobs: - job: Validate_Job_Results displayName: Job Results + timeoutInMinutes: 200 pool: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals 1es-windows-2022-open @@ -535,46 +554,23 @@ stages: HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ] steps: - pwsh: | - $groupResults = @{ - Windows = @("$(WindowsResult)", "$(HelixWindowsResult)") - Linux = @("$(LinuxResult)", "$(HelixUbuntuResult)") - MacOS = @("$(MacOsResult)", "$(HelixMacOsX64Result)") - Arm64 = @("$(HelixWindowsArm64Result)", "$(HelixMacOsArm64Result)") - Cosmos = @("$(HelixWindowsCosmosResult)", "$(HelixUbuntuCosmosResult)") - SqlServer = @("$(WindowsSqlServerResult)", "$(HelixWindowsSqlServerResult)", "$(HelixUbuntuSqlServerResult)") - } - - $failedGroups = @() - - foreach ($groupName in $groupResults.Keys) - { - $results = $groupResults[$groupName] - Write-Host "$groupName results: $($results -join ', ')" - - # Ignore jobs that were skipped (e.g. condition-gated) - $ran = $results | Where-Object { $_ -ne 'Skipped' } - if (-not $ran) - { - Write-Host " -> all jobs skipped, treating group as successful" - continue - } - - # Only 'Succeeded' counts as success. 'SucceededWithIssues' is treated as failure - # because the jobs run with continueOnError: true, which maps task failures to warnings. - $hasSuccess = $ran | Where-Object { $_ -eq 'Succeeded' } - if ($hasSuccess) - { - continue - } - - $failedGroups += $groupName - } - - if ($failedGroups.Count -gt 0) - { - Write-Error "No jobs succeeded for group(s): $($failedGroups -join ', ')." - exit 1 + $jobResults = @{ + Windows = "$(WindowsResult)" + Helix_Windows = "$(HelixWindowsResult)" + Windows_SqlServer = "$(WindowsSqlServerResult)" + Helix_Windows_SqlServer = "$(HelixWindowsSqlServerResult)" + Helix_Windows_Cosmos = "$(HelixWindowsCosmosResult)" + Linux = "$(LinuxResult)" + Helix_Ubuntu = "$(HelixUbuntuResult)" + Helix_Ubuntu_SqlServer = "$(HelixUbuntuSqlServerResult)" + macOS = "$(MacOsResult)" + Helix_macOS_x64 = "$(HelixMacOsX64Result)" + Helix_macOS_ARM64 = "$(HelixMacOsArm64Result)" + Helix_Windows_Arm64 = "$(HelixWindowsArm64Result)" + Helix_Ubuntu_Cosmos = "$(HelixUbuntuCosmosResult)" } - Write-Host 'Group validation passed.' + & eng/aggregate-azdo-tests.ps1 -JobResults $jobResults displayName: Evaluate grouped job outcomes + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) diff --git a/eng/aggregate-azdo-tests.ps1 b/eng/aggregate-azdo-tests.ps1 new file mode 100644 index 00000000000..e93e9a5d2dd --- /dev/null +++ b/eng/aggregate-azdo-tests.ps1 @@ -0,0 +1,159 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [hashtable]$JobResults +) + +# Aggregate redundant test jobs into logical groups, and on validation retries run only the jobs needed by failed groups. +# Azure DevOps timeline records expose display names, while the pipeline and group definitions use job IDs. +$jobDisplayNames = @{ + Windows = 'Windows' + Windows_SqlServer = 'Windows SQL Server' + macOS = 'macOS' + Linux = 'Linux' + Helix_Windows = 'Helix Windows' + Helix_Windows_SqlServer = 'Helix Windows SQL Server' + Helix_Windows_Arm64 = 'Helix Windows ARM64' + Helix_Windows_Cosmos = 'Helix Windows Cosmos' + Helix_macOS_x64 = 'Helix macOS x64' + Helix_macOS_ARM64 = 'Helix macOS ARM64' + Helix_Ubuntu_SqlServer = 'Helix Ubuntu SQL Server' + Helix_Ubuntu_Cosmos = 'Helix Ubuntu Cosmos' + Helix_Ubuntu = 'Helix Ubuntu' +} + +# A group succeeds when at least one of its jobs succeeds. Jobs may participate in multiple groups. +$groupJobs = @{ + Windows = @('Windows', 'Helix_Windows') + Linux = @('Linux', 'Helix_Ubuntu') + MacOS = @('macOS', 'Helix_macOS_x64') + Arm64 = @('Helix_Windows_Arm64', 'Helix_macOS_ARM64') + Cosmos = @('Helix_Windows_Cosmos', 'Helix_Ubuntu_Cosmos') + SqlServer = @('Windows_SqlServer', 'Helix_Windows_SqlServer', 'Helix_Ubuntu_SqlServer') +} + +function Get-JobRecord($timeline, [string]$jobName) +{ + $displayName = $jobDisplayNames[$jobName] + if ([string]::IsNullOrEmpty($displayName)) + { + throw "Unknown jobName '$jobName'." + } + + @($timeline.records + | Where-Object { $_.type -eq 'Job' -and $_.name -eq $displayName } + | Sort-Object attempt -Descending)[0] +} + +function Get-FailedGroups([hashtable]$resultsByJob) +{ + $failed = @() + + foreach ($groupName in $groupJobs.Keys) + { + foreach ($jobName in $groupJobs[$groupName]) + { + if (-not $resultsByJob.ContainsKey($jobName)) + { + throw "Missing result for job '$jobName' in group '$groupName'." + } + } + + $results = @($groupJobs[$groupName] | ForEach-Object { $resultsByJob[$_] }) + $ran = @($results | Where-Object { $_ -ne 'Skipped' }) + + if ($ran.Count -gt 0 -and $ran -notcontains 'Succeeded') + { + $failed += $groupName + } + } + + return $failed +} + +$jobAttempt = 1 +[void][int]::TryParse($env:SYSTEM_JOBATTEMPT, [ref]$jobAttempt) +$stageAttempt = 1 +[void][int]::TryParse($env:SYSTEM_STAGEATTEMPT, [ref]$stageAttempt) +$failedGroups = @(Get-FailedGroups $JobResults) + +# Retrying validation queues a child build containing the distinct jobs needed by all failed groups. +if (($jobAttempt -gt 1 -or $stageAttempt -gt 1) -and $failedGroups.Count -gt 0) +{ + if ([string]::IsNullOrEmpty($env:SYSTEM_ACCESSTOKEN)) + { + throw 'SYSTEM_ACCESSTOKEN is required to retry build jobs.' + } + + $jobsToRetry = @($failedGroups | ForEach-Object { $groupJobs[$_] } | Select-Object -Unique) + $project = [Uri]::EscapeDataString($env:SYSTEM_TEAMPROJECT) + $buildsUri = "$($env:SYSTEM_COLLECTIONURI.TrimEnd('/'))/$project/_apis/build/builds" + $headers = @{ Authorization = "Bearer $env:SYSTEM_ACCESSTOKEN" } + $queueBody = @{ + definition = @{ id = [int]$env:BUILD_DEFINITIONID } + sourceBranch = $env:BUILD_SOURCEBRANCH + sourceVersion = $env:BUILD_SOURCEVERSION + templateParameters = @{ jobs = $jobsToRetry } + } | ConvertTo-Json -Depth 4 + + Write-Host "Retrying jobs: $($jobsToRetry -join ', ')" + $retryBuild = Invoke-RestMethod -Uri "$buildsUri`?api-version=7.1" -Method Post -Headers $headers -ContentType 'application/json' -Body $queueBody + if ($null -eq $retryBuild.id) + { + throw 'The retry build response did not contain a build ID.' + } + + $deadline = [DateTime]::UtcNow.AddMinutes(190) + # Wait for the child build so its final job results can replace the original failed results. + do + { + Start-Sleep -Seconds 60 + $retryBuild = Invoke-RestMethod -Uri "$buildsUri/$($retryBuild.id)?api-version=7.1" -Headers $headers + } + while ($retryBuild.status -ne 'completed' -and [DateTime]::UtcNow -lt $deadline) + + if ($retryBuild.status -ne 'completed') + { + throw "Timed out waiting for retry build $($retryBuild.id)." + } + + $timeline = Invoke-RestMethod -Uri "$buildsUri/$($retryBuild.id)/timeline?api-version=7.1" -Headers $headers + # Merge only retried jobs; results from groups that already passed remain unchanged. + foreach ($jobName in $jobsToRetry) + { + $record = Get-JobRecord $timeline $jobName + if ($null -eq $record) + { + throw "Could not find timeline record for retried job '$jobName'." + } + + $JobResults[$jobName] = $record.result + } + + $failedGroups = @(Get-FailedGroups $JobResults) +} + +foreach ($groupName in $groupJobs.Keys) +{ + $results = @($groupJobs[$groupName] | ForEach-Object { $JobResults[$_] }) + Write-Host "$groupName results: $($results -join ', ')" + + $ran = @($results | Where-Object { $_ -ne 'Skipped' }) + if ($ran.Count -eq 0) + { + Write-Host ' -> all jobs skipped, treating group as successful' + continue + } + + if ($ran -contains 'Succeeded') + { + continue + } +} + +if ($failedGroups.Count -gt 0) +{ + throw "No jobs succeeded for group(s): $($failedGroups -join ', ')." +} + +Write-Host 'Group validation passed.' \ No newline at end of file