From 8a19c3c59c7376e9167002fc1808de66ad066467 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 13:39:59 -0700 Subject: [PATCH 1/8] benchmark template --- .../templates/jobs/benchmark-job.yml | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 eng/common/pipelines/templates/jobs/benchmark-job.yml diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml new file mode 100644 index 000000000000..e5371b7c88dd --- /dev/null +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -0,0 +1,84 @@ +parameters: + - name: BenchmarkProject + type: string + default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks' + - name: ScenarioName + type: string + default: '' + - name: Tags + type: string + default: '' + - name: Model + type: string + default: '' + - name: Parallelism + type: number + default: 5 + +jobs: + - job: Run_Benchmark + variables: + - template: /eng/pipelines/templates/variables/globals.yml + - template: /eng/pipelines/templates/variables/image.yml + - group: 'AzSDK_Eval_Variable_group' + displayName: 'Run Benchmark' + pool: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux + + steps: + - checkout: none + - task: UseDotNet@2 + displayName: "Use .NET SDK 8.0.x" + retryCountOnTaskFailure: 3 + inputs: + packageType: sdk + version: 8.0.x + performMultiLevelLookup: true + + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + SkipCheckoutNone: true + Repositories: + - Name: $(Build.Repository.Name) + Commitish: $(Build.SourceVersion) + WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name) + Paths: + - 'tools/azsdk-cli/**' + - 'eng/common/**' + + - task: PowerShell@2 + displayName: 'Run benchmarks' + condition: succeeded() + inputs: + targetType: 'inline' + workingDirectory: '$(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/${{ parameters.BenchmarkProject }}' + pwsh: true + script: | + $args = @("run") + + # Determine scenario selection: name, tags, or all + if ("${{ parameters.ScenarioName }}") { + $args += "${{ parameters.ScenarioName }}" + } elseif ("${{ parameters.Tags }}") { + $args += "--tags", "${{ parameters.Tags }}" + } else { + $args += "--all" + } + + # Optional model override + if ("${{ parameters.Model }}") { + $args += "--model", "${{ parameters.Model }}" + } + + $args += "--parallel", "${{ parameters.Parallelism }}" + $args += "--cleanup", "always" + + Write-Host "Running: dotnet run -- $($args -join ' ')" + dotnet run -- @args + env: + DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 + DOTNET_CLI_TELEMETRY_OPTOUT: 1 + DOTNET_MULTILEVEL_LOOKUP: 0 + COPILOT_GITHUB_TOKEN: $(azuresdk-copilot-github-pat) From 6f133496b2934d15f3a0b8a518175d640c9d233f Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:01:14 -0700 Subject: [PATCH 2/8] cli args instead of args --- .../pipelines/templates/jobs/benchmark-job.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index e5371b7c88dd..4b4701a8cfa9 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -56,27 +56,27 @@ jobs: workingDirectory: '$(System.DefaultWorkingDirectory)/$(Build.Repository.Name)/${{ parameters.BenchmarkProject }}' pwsh: true script: | - $args = @("run") + $cliArgs = @("run") # Determine scenario selection: name, tags, or all if ("${{ parameters.ScenarioName }}") { - $args += "${{ parameters.ScenarioName }}" + $cliArgs += "${{ parameters.ScenarioName }}" } elseif ("${{ parameters.Tags }}") { - $args += "--tags", "${{ parameters.Tags }}" + $cliArgs += "--tags", "${{ parameters.Tags }}" } else { - $args += "--all" + $cliArgs += "--all" } # Optional model override if ("${{ parameters.Model }}") { - $args += "--model", "${{ parameters.Model }}" + $cliArgs += "--model", "${{ parameters.Model }}" } - $args += "--parallel", "${{ parameters.Parallelism }}" - $args += "--cleanup", "always" + $cliArgs += "--parallel", "${{ parameters.Parallelism }}" + $cliArgs += "--cleanup", "always" - Write-Host "Running: dotnet run -- $($args -join ' ')" - dotnet run -- @args + Write-Host "Running: dotnet run -- $($cliArgs -join ' ')" + dotnet run -- @cliArgs env: DOTNET_SKIP_FIRST_TIME_EXPERIENCE: 1 DOTNET_CLI_TELEMETRY_OPTOUT: 1 From 1850561156e8d04ffaae88419a67267d4378d57f Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:05:10 -0700 Subject: [PATCH 3/8] target path --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 4b4701a8cfa9..2beba3f43f82 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -45,7 +45,7 @@ jobs: Commitish: $(Build.SourceVersion) WorkingDirectory: $(System.DefaultWorkingDirectory)/$(Build.Repository.Name) Paths: - - 'tools/azsdk-cli/**' + - '${{ parameters.BenchmarkProject }}/**' - 'eng/common/**' - task: PowerShell@2 From 40b8ee15a79e961120531e390b5663c353d6cf00 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:06:58 -0700 Subject: [PATCH 4/8] .net 9 --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 2beba3f43f82..99f6cb3c2678 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -30,14 +30,14 @@ jobs: steps: - checkout: none - task: UseDotNet@2 - displayName: "Use .NET SDK 8.0.x" + displayName: "Use .NET SDK 9.0.x" retryCountOnTaskFailure: 3 inputs: packageType: sdk - version: 8.0.x + version: 9.0.x performMultiLevelLookup: true - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + - template:/eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: SkipCheckoutNone: true Repositories: From c665e73d7cf340e14b1f200ad803708bb3be76c5 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:08:33 -0700 Subject: [PATCH 5/8] fix --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 99f6cb3c2678..d7d15bc90a78 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -37,7 +37,7 @@ jobs: version: 9.0.x performMultiLevelLookup: true - - template:/eng/common/pipelines/templates/steps/sparse-checkout.yml + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: SkipCheckoutNone: true Repositories: From c110a8abb2d3e2b4d414f48e698c0e4f8dc0585f Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 14:16:21 -0700 Subject: [PATCH 6/8] blank defaults --- .../pipelines/templates/jobs/benchmark-job.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index d7d15bc90a78..ef6ce1739583 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -4,13 +4,13 @@ parameters: default: 'tools/azsdk-cli/Azure.Sdk.Tools.Cli.Benchmarks' - name: ScenarioName type: string - default: '' + default: ' ' - name: Tags type: string - default: '' + default: ' ' - name: Model type: string - default: '' + default: ' ' - name: Parallelism type: number default: 5 @@ -59,17 +59,17 @@ jobs: $cliArgs = @("run") # Determine scenario selection: name, tags, or all - if ("${{ parameters.ScenarioName }}") { - $cliArgs += "${{ parameters.ScenarioName }}" - } elseif ("${{ parameters.Tags }}") { - $cliArgs += "--tags", "${{ parameters.Tags }}" + if ("${{ parameters.ScenarioName }}".Trim()) { + $cliArgs += "${{ parameters.ScenarioName }}".Trim() + } elseif ("${{ parameters.Tags }}".Trim()) { + $cliArgs += "--tags", "${{ parameters.Tags }}".Trim() } else { $cliArgs += "--all" } # Optional model override - if ("${{ parameters.Model }}") { - $cliArgs += "--model", "${{ parameters.Model }}" + if ("${{ parameters.Model }}".Trim()) { + $cliArgs += "--model", "${{ parameters.Model }}".Trim() } $cliArgs += "--parallel", "${{ parameters.Parallelism }}" From 1246dae08f0ddc321266bbd04c8087131ce11d98 Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Mon, 9 Mar 2026 16:32:08 -0700 Subject: [PATCH 7/8] back to 8 --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index ef6ce1739583..97fc8c7d2ff0 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -29,13 +29,7 @@ jobs: steps: - checkout: none - - task: UseDotNet@2 - displayName: "Use .NET SDK 9.0.x" - retryCountOnTaskFailure: 3 - inputs: - packageType: sdk - version: 9.0.x - performMultiLevelLookup: true + - template: /eng/pipelines/templates/steps/install-dotnet.yml - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: From da907cfe38e51002c24a1b7ef9e767913f7f675b Mon Sep 17 00:00:00 2001 From: Juan Ospina <70209456+jeo02@users.noreply.github.com> Date: Wed, 11 Mar 2026 13:48:58 -0700 Subject: [PATCH 8/8] install after checkout --- eng/common/pipelines/templates/jobs/benchmark-job.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/jobs/benchmark-job.yml b/eng/common/pipelines/templates/jobs/benchmark-job.yml index 97fc8c7d2ff0..d490d8977921 100644 --- a/eng/common/pipelines/templates/jobs/benchmark-job.yml +++ b/eng/common/pipelines/templates/jobs/benchmark-job.yml @@ -29,7 +29,6 @@ jobs: steps: - checkout: none - - template: /eng/pipelines/templates/steps/install-dotnet.yml - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml parameters: @@ -42,6 +41,8 @@ jobs: - '${{ parameters.BenchmarkProject }}/**' - 'eng/common/**' + - template: /eng/pipelines/templates/steps/install-dotnet.yml + - task: PowerShell@2 displayName: 'Run benchmarks' condition: succeeded()