Skip to content

Add a script to only retry failed AzDo jobs #38761

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AndriySvyryd wants to merge 1 commit into
base: main
Choose a base branch
from
+232 −81
Open

Add a script to only retry failed AzDo jobs #38761

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 37 additions & 41 deletions azure-pipelines-internal-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
parameters:
- name: jobs
type: object
default: []

variables:
- name: _BuildConfig
value: Release
Expand Down Expand Up @@ -103,11 +108,12 @@ extends:
enablePublishBuildAssets: false
enablePublishTestResults: true
publishAssetsImmediately: true
isAssetlessBuild: true
isAssetlessBuild: ${{ eq(length(parameters.jobs), 0) }}
enableTelemetry: true
helixRepo: dotnet/efcore
jobs:
- job: Windows
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Windows')) }}
pool:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals 1es-windows-2022
Expand Down Expand Up @@ -160,6 +166,7 @@ extends:

- job: Windows_SqlServer
displayName: 'Windows SQL Server'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Windows_SqlServer')) }}
pool:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals 1es-windows-2022
Expand Down Expand Up @@ -195,6 +202,7 @@ extends:
continueOnError: true

- job: macOS
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'macOS')) }}
pool:
name: Azure Pipelines
image: macOS-15
Expand Down Expand Up @@ -230,6 +238,7 @@ extends:
condition: always()
continueOnError: true
- job: Linux
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Linux')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngInternalBuildPool)
Expand Down Expand Up @@ -269,6 +278,7 @@ extends:

- job: Helix_Windows
displayName: 'Helix Windows'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngInternalBuildPool)
Expand Down Expand Up @@ -311,6 +321,7 @@ extends:

- job: Helix_Windows_SqlServer
displayName: 'Helix Windows SQL Server'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_SqlServer')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngInternalBuildPool)
Expand Down Expand Up @@ -353,6 +364,7 @@ extends:

- job: Helix_Windows_Arm64
displayName: 'Helix Windows ARM64'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_Arm64')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngInternalBuildPool)
Expand Down Expand Up @@ -395,6 +407,7 @@ extends:

- job: Helix_Windows_Cosmos
displayName: 'Helix Windows Cosmos'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_Cosmos')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngInternalBuildPool)
Expand Down Expand Up @@ -437,6 +450,7 @@ extends:

- job: Helix_macOS_x64
displayName: 'Helix macOS x64'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_macOS_x64')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngInternalBuildPool)
Expand Down Expand Up @@ -479,6 +493,7 @@ extends:

- job: Helix_macOS_ARM64
displayName: 'Helix macOS ARM64'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_macOS_ARM64')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngInternalBuildPool)
Expand Down Expand Up @@ -521,6 +536,7 @@ extends:

- job: Helix_Ubuntu_SqlServer
displayName: 'Helix Ubuntu SQL Server'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu_SqlServer')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngInternalBuildPool)
Expand Down Expand Up @@ -564,6 +580,7 @@ extends:

- job: Helix_Ubuntu_Cosmos
displayName: 'Helix Ubuntu Cosmos'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu_Cosmos')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngInternalBuildPool)
Expand Down Expand Up @@ -606,6 +623,7 @@ extends:

- job: Helix_Ubuntu
displayName: 'Helix Ubuntu'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngInternalBuildPool)
Expand Down Expand Up @@ -652,6 +670,7 @@ extends:
jobs:
- job: Validate_Job_Results
displayName: Job Results
timeoutInMinutes: 200
pool:
name: $(DncEngInternalBuildPool)
demands: ImageOverride -equals 1es-windows-2022
Expand All @@ -672,46 +691,23 @@ extends:
HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ]
steps:
- pwsh: |
$groupResults = @{
Windows = @("$(WindowsResult)", "$(HelixWindowsResult)")
Linux = @("$(LinuxResult)", "$(HelixUbuntuResult)")
MacOS = @("$(MacOsResult)", "$(HelixMacOsX64Result)")
Arm64 = @("$(HelixWindowsArm64Result)", "$(HelixMacOsArm64Result)")
Cosmos = @("$(HelixWindowsCosmosResult)", "$(HelixUbuntuCosmosResult)")
SqlServer = @("$(WindowsSqlServerResult)", "$(HelixWindowsSqlServerResult)", "$(HelixUbuntuSqlServerResult)")
}

$failedGroups = @()

foreach ($groupName in $groupResults.Keys)
{
$results = $groupResults[$groupName]
Write-Host "$groupName results: $($results -join ', ')"

# Ignore jobs that were skipped (e.g. condition-gated)
$ran = $results | Where-Object { $_ -ne 'Skipped' }
if (-not $ran)
{
Write-Host " -> all jobs skipped, treating group as successful"
continue
}

# Only 'Succeeded' counts as success. 'SucceededWithIssues' is treated as failure
# because the jobs run with continueOnError: true, which maps task failures to warnings.
$hasSuccess = $ran | Where-Object { $_ -eq 'Succeeded' }
if ($hasSuccess)
{
continue
}

$failedGroups += $groupName
}

if ($failedGroups.Count -gt 0)
{
Write-Error "No jobs succeeded for group(s): $($failedGroups -join ', ')."
exit 1
$jobResults = @{
Windows = "$(WindowsResult)"
Helix_Windows = "$(HelixWindowsResult)"
Windows_SqlServer = "$(WindowsSqlServerResult)"
Helix_Windows_SqlServer = "$(HelixWindowsSqlServerResult)"
Helix_Windows_Cosmos = "$(HelixWindowsCosmosResult)"
Linux = "$(LinuxResult)"
Helix_Ubuntu = "$(HelixUbuntuResult)"
Helix_Ubuntu_SqlServer = "$(HelixUbuntuSqlServerResult)"
macOS = "$(MacOsResult)"
Helix_macOS_x64 = "$(HelixMacOsX64Result)"
Helix_macOS_ARM64 = "$(HelixMacOsArm64Result)"
Helix_Windows_Arm64 = "$(HelixWindowsArm64Result)"
Helix_Ubuntu_Cosmos = "$(HelixUbuntuCosmosResult)"
}

Write-Host 'Group validation passed.'
& eng/aggregate-azdo-tests.ps1 -JobResults $jobResults
displayName: Evaluate grouped job outcomes
env:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
76 changes: 36 additions & 40 deletions azure-pipelines-public.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
parameters:
- name: jobs
type: object
default: []

variables:
- name: _BuildConfig
value: Release
Expand Down Expand Up @@ -52,6 +57,7 @@ stages:
helixRepo: dotnet/efcore
jobs:
- job: Windows
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Windows')) }}
enablePublishTestResults: true
timeoutInMinutes: 90
pool:
Expand Down Expand Up @@ -113,6 +119,7 @@ stages:

- job: Windows_SqlServer
displayName: 'Windows SQL Server'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Windows_SqlServer')) }}
enablePublishTestResults: true
timeoutInMinutes: 120
pool:
Expand Down Expand Up @@ -156,6 +163,7 @@ stages:
parallel: true

- job: macOS
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'macOS')) }}
enablePublishTestResults: true
pool:
vmImage: macOS-15
Expand Down Expand Up @@ -198,6 +206,7 @@ stages:
parallel: true

- job: Linux
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Linux')) }}
timeoutInMinutes: 120
enablePublishTestResults: true
pool:
Expand Down Expand Up @@ -241,6 +250,7 @@ stages:

- job: Helix_Windows
displayName: 'Helix Windows'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngPublicBuildPool)
Expand Down Expand Up @@ -271,6 +281,7 @@ stages:

- job: Helix_Windows_SqlServer
displayName: 'Helix Windows SQL Server'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_SqlServer')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngPublicBuildPool)
Expand Down Expand Up @@ -301,6 +312,7 @@ stages:

- job: Helix_Windows_Arm64
displayName: 'Helix Windows ARM64'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_Arm64')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngPublicBuildPool)
Expand Down Expand Up @@ -331,6 +343,7 @@ stages:

- job: Helix_Windows_Cosmos
displayName: 'Helix Windows Cosmos'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Windows_Cosmos')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngPublicBuildPool)
Expand Down Expand Up @@ -361,6 +374,7 @@ stages:

- job: Helix_macOS_x64
displayName: 'Helix macOS x64'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_macOS_x64')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngPublicBuildPool)
Expand Down Expand Up @@ -391,6 +405,7 @@ stages:

- job: Helix_macOS_ARM64
displayName: 'Helix macOS ARM64'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_macOS_ARM64')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngPublicBuildPool)
Expand Down Expand Up @@ -421,6 +436,7 @@ stages:

- job: Helix_Ubuntu_SqlServer
displayName: 'Helix Ubuntu SQL Server'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu_SqlServer')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngPublicBuildPool)
Expand Down Expand Up @@ -452,6 +468,7 @@ stages:

- job: Helix_Ubuntu_Cosmos
displayName: 'Helix Ubuntu Cosmos'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu_Cosmos')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngPublicBuildPool)
Expand Down Expand Up @@ -482,6 +499,7 @@ stages:

- job: Helix_Ubuntu
displayName: 'Helix Ubuntu'
condition: ${{ or(eq(length(parameters.jobs), 0), containsValue(parameters.jobs, 'Helix_Ubuntu')) }}
timeoutInMinutes: 180
pool:
name: $(DncEngPublicBuildPool)
Expand Down Expand Up @@ -516,6 +534,7 @@ stages:
jobs:
- job: Validate_Job_Results
displayName: Job Results
timeoutInMinutes: 200
pool:
name: $(DncEngPublicBuildPool)
demands: ImageOverride -equals 1es-windows-2022-open
Expand All @@ -535,46 +554,23 @@ stages:
HelixUbuntuCosmosResult: $[ stageDependencies.build.Helix_Ubuntu_Cosmos.result ]
steps:
- pwsh: |
$groupResults = @{
Windows = @("$(WindowsResult)", "$(HelixWindowsResult)")
Linux = @("$(LinuxResult)", "$(HelixUbuntuResult)")
MacOS = @("$(MacOsResult)", "$(HelixMacOsX64Result)")
Arm64 = @("$(HelixWindowsArm64Result)", "$(HelixMacOsArm64Result)")
Cosmos = @("$(HelixWindowsCosmosResult)", "$(HelixUbuntuCosmosResult)")
SqlServer = @("$(WindowsSqlServerResult)", "$(HelixWindowsSqlServerResult)", "$(HelixUbuntuSqlServerResult)")
}

$failedGroups = @()

foreach ($groupName in $groupResults.Keys)
{
$results = $groupResults[$groupName]
Write-Host "$groupName results: $($results -join ', ')"

# Ignore jobs that were skipped (e.g. condition-gated)
$ran = $results | Where-Object { $_ -ne 'Skipped' }
if (-not $ran)
{
Write-Host " -> all jobs skipped, treating group as successful"
continue
}

# Only 'Succeeded' counts as success. 'SucceededWithIssues' is treated as failure
# because the jobs run with continueOnError: true, which maps task failures to warnings.
$hasSuccess = $ran | Where-Object { $_ -eq 'Succeeded' }
if ($hasSuccess)
{
continue
}

$failedGroups += $groupName
}

if ($failedGroups.Count -gt 0)
{
Write-Error "No jobs succeeded for group(s): $($failedGroups -join ', ')."
exit 1
$jobResults = @{
Windows = "$(WindowsResult)"
Helix_Windows = "$(HelixWindowsResult)"
Windows_SqlServer = "$(WindowsSqlServerResult)"
Helix_Windows_SqlServer = "$(HelixWindowsSqlServerResult)"
Helix_Windows_Cosmos = "$(HelixWindowsCosmosResult)"
Linux = "$(LinuxResult)"
Helix_Ubuntu = "$(HelixUbuntuResult)"
Helix_Ubuntu_SqlServer = "$(HelixUbuntuSqlServerResult)"
macOS = "$(MacOsResult)"
Helix_macOS_x64 = "$(HelixMacOsX64Result)"
Helix_macOS_ARM64 = "$(HelixMacOsArm64Result)"
Helix_Windows_Arm64 = "$(HelixWindowsArm64Result)"
Helix_Ubuntu_Cosmos = "$(HelixUbuntuCosmosResult)"
}

Write-Host 'Group validation passed.'
& eng/aggregate-azdo-tests.ps1 -JobResults $jobResults
displayName: Evaluate grouped job outcomes
env:
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
Loading
Loading