From ea1c00d52f2141949a7d4885ebbb1017ab825be1 Mon Sep 17 00:00:00 2001 From: Ben Thomas <25218250+alliscode@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:13:42 -0700 Subject: [PATCH 01/12] .NET: feat(evals): RubricScore type + EvalScoreResult.Dimensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the core rubric-evaluator surface that mirrors the Python work in PR #6101 (commit e45b934cc). Provider-agnostic types only — no Foundry coupling. Subsequent commits will wire these into FoundryEvals. - RubricScore: per-dimension score record (Id, Score?, Applicable, Weight, Reason). - EvalScoreResult.Dimensions: optional init-only list of RubricScore. Null for non-rubric (built-in) evaluators. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Evaluation/EvalItemResult.cs | 20 +++++- .../Evaluation/RubricScore.cs | 34 ++++++++++ .../EvaluationTests.cs | 62 +++++++++++++++++++ 3 files changed, 115 insertions(+), 1 deletion(-) create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/RubricScore.cs diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs index 64e317be2b..9557f8494c 100644 --- a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs @@ -66,7 +66,25 @@ public EvalItemResult(string itemId, string status, IReadOnlyListThe evaluator name that produced this score. /// The numeric score value. /// Whether the evaluator considered this a pass, or null if not determined. -public record EvalScoreResult(string Name, double Score, bool? Passed = null); +public record EvalScoreResult(string Name, double Score, bool? Passed = null) +{ + /// + /// Gets the per-dimension breakdown when this evaluator is a rubric-based evaluator. + /// + /// + /// + /// Rubric evaluators (for example, generated rubric evaluators authored in the Azure AI + /// Foundry portal) emit one per dimension per item alongside + /// the overall weighted . Each entry preserves the dimension's + /// applicability, weight, and the evaluator-supplied rationale. + /// + /// + /// Non-rubric evaluators (built-in quality, safety, or agent-behavior evaluators) leave + /// this property . + /// + /// + public IReadOnlyList? Dimensions { get; init; } +} /// /// Per-evaluator pass/fail breakdown from an evaluation run. diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/RubricScore.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/RubricScore.cs new file mode 100644 index 0000000000..d873af2644 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/RubricScore.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Agents.AI; + +/// +/// A single dimension's score from a rubric-based evaluator run. +/// +/// +/// +/// Rubric evaluators (such as the generated rubric evaluators produced by Azure AI Foundry's +/// adaptive evals) emit one per dimension per item, alongside an +/// overall weighted score. Attach instances to as +/// a typed view of the per-dimension breakdown returned by the provider +/// (e.g. properties.dimension_scores). +/// +/// +/// Non-rubric evaluators (built-in quality, safety, or agent-behavior evaluators) leave +/// as . +/// +/// +/// Dimension identifier — matches the id defined on the rubric. +/// +/// Numeric score for the dimension, or when the dimension was marked +/// non-applicable for this item. Foundry rubric evaluators emit integer scores on a 1–5 scale. +/// +/// Whether the dimension applied to this item. +/// Dimension weight, mirroring the rubric definition. +/// Short rationale produced by the evaluator. +public sealed record RubricScore( + string Id, + int? Score, + bool Applicable, + int Weight, + string Reason); diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs index 071e9b723a..df25a34364 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs @@ -550,6 +550,68 @@ public void AgentEvaluationResults_SubResults_AllPassedChecksChildren() Assert.False(results.AllPassed); } + // --------------------------------------------------------------- + // RubricScore tests + // --------------------------------------------------------------- + + [Fact] + public void RubricScore_Constructor_SetsAllProperties() + { + // Arrange & Act + var dim = new RubricScore("clarity", Score: 4, Applicable: true, Weight: 2, Reason: "clear"); + + // Assert + Assert.Equal("clarity", dim.Id); + Assert.Equal(4, dim.Score); + Assert.True(dim.Applicable); + Assert.Equal(2, dim.Weight); + Assert.Equal("clear", dim.Reason); + } + + [Fact] + public void RubricScore_NonApplicable_AllowsNullScore() + { + // Arrange & Act + var dim = new RubricScore("safety", Score: null, Applicable: false, Weight: 1, Reason: "n/a"); + + // Assert + Assert.Null(dim.Score); + Assert.False(dim.Applicable); + } + + [Fact] + public void EvalScoreResult_Dimensions_DefaultsToNull() + { + // Arrange & Act + var score = new EvalScoreResult("relevance", 0.8, Passed: true); + + // Assert + Assert.Null(score.Dimensions); + } + + [Fact] + public void EvalScoreResult_Dimensions_CanBeInitialized() + { + // Arrange + var dimensions = new List + { + new("clarity", 4, true, 1, "ok"), + new("safety", null, false, 1, "n/a"), + }; + + // Act + var score = new EvalScoreResult("custom-rubric", 0.75, Passed: true) + { + Dimensions = dimensions, + }; + + // Assert + Assert.NotNull(score.Dimensions); + Assert.Equal(2, score.Dimensions.Count); + Assert.Equal("clarity", score.Dimensions[0].Id); + Assert.False(score.Dimensions[1].Applicable); + } + // --------------------------------------------------------------- // Mixed evaluator tests // --------------------------------------------------------------- From e29eae82188487f67b604f67bf3286a0bf5640b0 Mon Sep 17 00:00:00 2001 From: Ben Thomas <25218250+alliscode@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:20:30 -0700 Subject: [PATCH 02/12] .NET: feat(evals): GeneratedEvaluatorRef + assertion helpers Adds the provider-agnostic surface for referencing a pre-existing rubric evaluator and gating CI on per-item / per-dimension thresholds. Mirrors Python PR #6101 commits e5830dd7f (ref type) and 4bc60462d (asserts). - GeneratedEvaluatorRef: name + optional version/display-name, plus a Latest(name) factory for versionless refs (discouraged for CI; consumers should warn at run time). - AgentEvaluationResults.AssertScoreAtLeast: walks DetailedItems[].Scores, optionally filtered by evaluator name, recurses into SubResults. - AgentEvaluationResults.AssertDimensionScoreAtLeast: walks each score's Dimensions list, skips non-applicable dimensions by default, supports requireApplicable to flip that, recurses into SubResults. - AgentEvaluationResults.AssertNoFailedItems: walks DetailedItems for fail/error statuses, recurses into SubResults. All helpers throw InvalidOperationException (matches existing AssertAllPassed). Truncates offender lists to the first 5 with a '+N more' suffix to keep CI output readable, mirroring the Python helpers. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Evaluation/AgentEvaluationResults.cs | 248 +++++++++++++++++ .../Evaluation/EvalItemResult.cs | 4 +- .../Evaluation/GeneratedEvaluatorRef.cs | 55 ++++ .../EvaluationTests.cs | 252 ++++++++++++++++++ 4 files changed, 558 insertions(+), 1 deletion(-) create mode 100644 dotnet/src/Microsoft.Agents.AI/Evaluation/GeneratedEvaluatorRef.cs diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs index f33d69a2e3..10fa02b986 100644 --- a/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Globalization; using System.Linq; using Microsoft.Extensions.AI.Evaluation; @@ -118,6 +119,253 @@ public void AssertAllPassed(string? message = null) } } + /// + /// Asserts that every per-evaluator score on every item is at least . + /// + /// + /// + /// Designed for CI gates on generated rubric evaluators (for example + /// results.AssertScoreAtLeast(0.80)). Walks across this + /// result and any from workflow evaluations. + /// + /// + /// When is , the assertion is a no-op for + /// this level. Providers that surface per-evaluator scores (such as Foundry) populate + /// ; providers that only emit aggregate + /// metrics do not. + /// + /// + /// Minimum acceptable score (inclusive). + /// + /// When set, only check scores whose matches. + /// + /// Optional custom failure message. + /// + /// Thrown when any matching score is below the threshold. + /// + public void AssertScoreAtLeast(double minScore, string? evaluator = null, string? message = null) + { + var offenders = new List(); + CollectScoreOffenders(this, minScore, evaluator, offenders); + + if (offenders.Count > 0) + { + throw new InvalidOperationException( + message ?? FormatOffenders( + $"{offenders.Count} score(s) below threshold {minScore.ToString(CultureInfo.InvariantCulture)}" + + (evaluator is not null ? $" for {evaluator}" : string.Empty), + offenders)); + } + } + + /// + /// Asserts that every item's score for the given rubric dimension is at least + /// . + /// + /// + /// Walks across + /// (and any ) looking for the named dimension. Non-applicable + /// dimensions are skipped by default; pass = + /// to fail when no applicable score is produced for an item. + /// + /// Dimension id — matches the rubric definition. + /// Minimum acceptable dimension score (inclusive). + /// + /// When set, only consider scores whose matches. + /// + /// + /// When , items with no applicable score for the dimension also fail + /// the assertion. Defaults to (skip). + /// + /// Optional custom failure message. + /// + /// Thrown when the dimension fails the threshold on any item. + /// + public void AssertDimensionScoreAtLeast( + string dimensionId, + double minScore, + string? evaluator = null, + bool requireApplicable = false, + string? message = null) + { + var offenders = new List(); + var missing = new List(); + CollectDimensionOffenders(this, dimensionId, minScore, evaluator, requireApplicable, offenders, missing); + + var problems = new List(); + if (offenders.Count > 0) + { + problems.Add(FormatOffenders( + $"{offenders.Count} dimension score(s) for '{dimensionId}' below {minScore.ToString(CultureInfo.InvariantCulture)}", + offenders)); + } + + if (missing.Count > 0) + { + problems.Add(FormatOffenders( + $"Dimension '{dimensionId}' not applicable on {missing.Count} item(s)", + missing)); + } + + if (problems.Count > 0) + { + throw new InvalidOperationException(message ?? string.Join("; ", problems)); + } + } + + /// + /// Asserts that no item ended in a failed or errored state. Includes any sub-results + /// from workflow evaluations. + /// + /// Optional custom failure message. + /// + /// Thrown when any item failed or errored. + /// + public void AssertNoFailedItems(string? message = null) + { + var bad = new List(); + CollectFailedItems(this, bad); + + if (bad.Count > 0) + { + throw new InvalidOperationException( + message ?? FormatOffenders($"{bad.Count} item(s) failed or errored", bad)); + } + } + + private static void CollectScoreOffenders( + AgentEvaluationResults results, + double minScore, + string? evaluator, + List offenders) + { + if (results.DetailedItems is not null) + { + foreach (var item in results.DetailedItems) + { + foreach (var score in item.Scores) + { + if (evaluator is not null && score.Name != evaluator) + { + continue; + } + + if (score.Score < minScore) + { + offenders.Add($"{item.ItemId}/{score.Name}={score.Score.ToString("F3", CultureInfo.InvariantCulture)}"); + } + } + } + } + + if (results.SubResults is not null) + { + foreach (var sub in results.SubResults.Values) + { + CollectScoreOffenders(sub, minScore, evaluator, offenders); + } + } + } + + private static void CollectDimensionOffenders( + AgentEvaluationResults results, + string dimensionId, + double minScore, + string? evaluator, + bool requireApplicable, + List offenders, + List missing) + { + if (results.DetailedItems is not null) + { + foreach (var item in results.DetailedItems) + { + bool foundApplicable = false; + foreach (var score in item.Scores) + { + if (evaluator is not null && score.Name != evaluator) + { + continue; + } + + if (score.Dimensions is null) + { + continue; + } + + foreach (var rs in score.Dimensions) + { + if (rs.Id != dimensionId) + { + continue; + } + + if (!rs.Applicable) + { + continue; + } + + foundApplicable = true; + if (rs.Score is null || rs.Score.Value < minScore) + { + var actual = rs.Score is null + ? "null" + : rs.Score.Value.ToString(CultureInfo.InvariantCulture); + offenders.Add($"{item.ItemId}/{score.Name}/{dimensionId}={actual}"); + } + } + } + + if (requireApplicable && !foundApplicable) + { + missing.Add(item.ItemId); + } + } + } + + if (results.SubResults is not null) + { + foreach (var sub in results.SubResults.Values) + { + CollectDimensionOffenders(sub, dimensionId, minScore, evaluator, requireApplicable, offenders, missing); + } + } + } + + private static void CollectFailedItems(AgentEvaluationResults results, List bad) + { + if (results.DetailedItems is not null) + { + foreach (var item in results.DetailedItems) + { + if (item.IsFailed || item.IsError) + { + bad.Add($"{item.ItemId}:{item.Status}"); + } + } + } + + if (results.SubResults is not null) + { + foreach (var sub in results.SubResults.Values) + { + CollectFailedItems(sub, bad); + } + } + } + + private static string FormatOffenders(string prefix, List offenders) + { + const int MaxShown = 5; + if (offenders.Count <= MaxShown) + { + return $"{prefix}: {string.Join(", ", offenders)}"; + } + + var shown = string.Join(", ", offenders.GetRange(0, MaxShown)); + return $"{prefix}: {shown} (+{offenders.Count - MaxShown} more)"; + } + private static bool ItemPassed(EvaluationResult result) { foreach (var metric in result.Metrics.Values) diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs index 9557f8494c..fb8014e2c3 100644 --- a/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/EvalItemResult.cs @@ -80,7 +80,9 @@ public record EvalScoreResult(string Name, double Score, bool? Passed = null) /// /// /// Non-rubric evaluators (built-in quality, safety, or agent-behavior evaluators) leave - /// this property . + /// this property . Use + /// + /// to gate CI on a specific dimension across all items. /// /// public IReadOnlyList? Dimensions { get; init; } diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/GeneratedEvaluatorRef.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/GeneratedEvaluatorRef.cs new file mode 100644 index 0000000000..a73d9cf590 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/GeneratedEvaluatorRef.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. + +namespace Microsoft.Agents.AI; + +/// +/// A reference to a generated rubric evaluator that already exists in the provider's registry. +/// +/// +/// +/// Pass instances of this class to a batch evaluator (for example +/// Microsoft.Agents.AI.Foundry.FoundryEvals) to score items with a pre-existing rubric +/// evaluator that was authored in the provider's portal or via the provider's dedicated SDK. +/// Agent Framework is a consumer here: it does not create or modify the evaluator definition; +/// it only references the persisted version by name. +/// +/// +/// Pinning is strongly recommended so evaluation runs are reproducible. +/// A resolves to whichever version is current at +/// execution time; consuming evaluators are expected to emit a warning when a versionless +/// reference is used. CI gates should always pass a concrete version. +/// +/// +/// +/// Evaluator name as stored in the provider's registry (for example +/// "reservation-policy-rubric"). Distinct from built-in evaluators such as +/// "relevance". +/// +/// +/// Pinned evaluator version. means "latest" — this is discouraged for +/// reproducible runs and consumers may emit a warning when used. +/// +/// +/// Optional human-readable name used in result summaries. Defaults to when +/// unset. +/// +public sealed record GeneratedEvaluatorRef( + string Name, + string? Version = null, + string? DisplayName = null) +{ + /// + /// Creates a versionless reference that resolves to the latest version of the evaluator at + /// run time. + /// + /// + /// Discouraged for reproducible runs. Prefer the primary constructor with an explicit + /// so CI and replay evaluations stay stable when the evaluator is + /// updated in the provider's registry. + /// + /// Evaluator name as stored in the provider's registry. + /// Optional human-readable name used in result summaries. + /// A new with unset. + public static GeneratedEvaluatorRef Latest(string name, string? displayName = null) + => new(name, Version: null, DisplayName: displayName); +} diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs index df25a34364..47def4e80d 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs @@ -612,6 +612,258 @@ public void EvalScoreResult_Dimensions_CanBeInitialized() Assert.False(score.Dimensions[1].Applicable); } + // --------------------------------------------------------------- + // GeneratedEvaluatorRef tests + // --------------------------------------------------------------- + + [Fact] + public void GeneratedEvaluatorRef_Constructor_DefaultsVersionAndDisplayNameToNull() + { + // Arrange & Act + var @ref = new GeneratedEvaluatorRef("policy-rubric"); + + // Assert + Assert.Equal("policy-rubric", @ref.Name); + Assert.Null(@ref.Version); + Assert.Null(@ref.DisplayName); + } + + [Fact] + public void GeneratedEvaluatorRef_Constructor_AcceptsVersionAndDisplayName() + { + // Arrange & Act + var @ref = new GeneratedEvaluatorRef("policy-rubric", Version: "3", DisplayName: "Policy Quality"); + + // Assert + Assert.Equal("3", @ref.Version); + Assert.Equal("Policy Quality", @ref.DisplayName); + } + + [Fact] + public void GeneratedEvaluatorRef_Latest_ProducesVersionlessReference() + { + // Arrange & Act + var @ref = GeneratedEvaluatorRef.Latest("policy-rubric", displayName: "Policy"); + + // Assert + Assert.Equal("policy-rubric", @ref.Name); + Assert.Null(@ref.Version); + Assert.Equal("Policy", @ref.DisplayName); + } + + // --------------------------------------------------------------- + // Assertion helper tests (AssertScoreAtLeast / AssertDimensionScoreAtLeast / AssertNoFailedItems) + // --------------------------------------------------------------- + + private static AgentEvaluationResults BuildResultsWithDetailed(params EvalItemResult[] detailed) + => new("test", Array.Empty()) + { + DetailedItems = detailed, + }; + + [Fact] + public void AssertScoreAtLeast_AboveThreshold_DoesNotThrow() + { + // Arrange + var detailed = new EvalItemResult("item-1", "pass", new[] + { + new EvalScoreResult("relevance", 0.9, Passed: true), + new EvalScoreResult("coherence", 0.85, Passed: true), + }); + var results = BuildResultsWithDetailed(detailed); + + // Act & Assert + results.AssertScoreAtLeast(0.8); + } + + [Fact] + public void AssertScoreAtLeast_BelowThreshold_ThrowsWithOffender() + { + // Arrange + var detailed = new EvalItemResult("item-1", "pass", new[] + { + new EvalScoreResult("relevance", 0.5, Passed: false), + new EvalScoreResult("coherence", 0.9, Passed: true), + }); + var results = BuildResultsWithDetailed(detailed); + + // Act & Assert + var ex = Assert.Throws(() => results.AssertScoreAtLeast(0.8)); + Assert.Contains("item-1/relevance=0.500", ex.Message); + Assert.Contains("0.8", ex.Message); + } + + [Fact] + public void AssertScoreAtLeast_EvaluatorFilter_OnlyChecksMatchingName() + { + // Arrange — coherence is below threshold but we filter to relevance. + var detailed = new EvalItemResult("item-1", "pass", new[] + { + new EvalScoreResult("relevance", 0.9, Passed: true), + new EvalScoreResult("coherence", 0.4, Passed: false), + }); + var results = BuildResultsWithDetailed(detailed); + + // Act & Assert — filtered to relevance, so no throw. + results.AssertScoreAtLeast(0.8, evaluator: "relevance"); + } + + [Fact] + public void AssertScoreAtLeast_RecursesIntoSubResults() + { + // Arrange + var failing = new EvalItemResult("sub-1", "pass", new[] + { + new EvalScoreResult("relevance", 0.2, Passed: false), + }); + var sub = BuildResultsWithDetailed(failing); + + var top = new AgentEvaluationResults("top", Array.Empty()) + { + SubResults = new Dictionary { ["agent"] = sub }, + }; + + // Act & Assert + var ex = Assert.Throws(() => top.AssertScoreAtLeast(0.8)); + Assert.Contains("sub-1/relevance", ex.Message); + } + + [Fact] + public void AssertDimensionScoreAtLeast_AboveThreshold_DoesNotThrow() + { + // Arrange + var detailed = new EvalItemResult("item-1", "pass", new[] + { + new EvalScoreResult("policy", 0.9, Passed: true) + { + Dimensions = + [ + new RubricScore("clarity", Score: 4, Applicable: true, Weight: 1, Reason: "ok"), + new RubricScore("safety", Score: 5, Applicable: true, Weight: 1, Reason: "ok"), + ], + }, + }); + var results = BuildResultsWithDetailed(detailed); + + // Act & Assert + results.AssertDimensionScoreAtLeast("clarity", 3.0); + } + + [Fact] + public void AssertDimensionScoreAtLeast_BelowThreshold_ThrowsWithOffender() + { + // Arrange + var detailed = new EvalItemResult("item-1", "pass", new[] + { + new EvalScoreResult("policy", 0.6, Passed: false) + { + Dimensions = + [ + new RubricScore("clarity", Score: 2, Applicable: true, Weight: 1, Reason: "weak"), + ], + }, + }); + var results = BuildResultsWithDetailed(detailed); + + // Act & Assert + var ex = Assert.Throws( + () => results.AssertDimensionScoreAtLeast("clarity", 3.0, evaluator: "policy")); + Assert.Contains("item-1/policy/clarity=2", ex.Message); + } + + [Fact] + public void AssertDimensionScoreAtLeast_NonApplicable_SkippedByDefault() + { + // Arrange — score=null + applicable=false should NOT trip the assertion by default. + var detailed = new EvalItemResult("item-1", "pass", new[] + { + new EvalScoreResult("policy", 1.0, Passed: true) + { + Dimensions = + [ + new RubricScore("optional", Score: null, Applicable: false, Weight: 1, Reason: "n/a"), + ], + }, + }); + var results = BuildResultsWithDetailed(detailed); + + // Act & Assert + results.AssertDimensionScoreAtLeast("optional", 3.0); + } + + [Fact] + public void AssertDimensionScoreAtLeast_RequireApplicable_ThrowsWhenMissing() + { + // Arrange — dimension never produced an applicable score on the item. + var detailed = new EvalItemResult("item-1", "pass", new[] + { + new EvalScoreResult("policy", 1.0, Passed: true) + { + Dimensions = + [ + new RubricScore("optional", Score: null, Applicable: false, Weight: 1, Reason: "n/a"), + ], + }, + }); + var results = BuildResultsWithDetailed(detailed); + + // Act & Assert + var ex = Assert.Throws( + () => results.AssertDimensionScoreAtLeast("optional", 3.0, requireApplicable: true)); + Assert.Contains("not applicable", ex.Message); + Assert.Contains("item-1", ex.Message); + } + + [Fact] + public void AssertNoFailedItems_AllPassing_DoesNotThrow() + { + // Arrange + var detailed = new EvalItemResult("item-1", "pass", new[] + { + new EvalScoreResult("relevance", 0.9, Passed: true), + }); + var results = BuildResultsWithDetailed(detailed); + + // Act & Assert + results.AssertNoFailedItems(); + } + + [Fact] + public void AssertNoFailedItems_FailedOrErrored_ThrowsWithStatuses() + { + // Arrange + var failed = new EvalItemResult("item-1", "fail", new[] + { + new EvalScoreResult("relevance", 0.2, Passed: false), + }); + var errored = new EvalItemResult("item-2", "errored", Array.Empty()); + var results = BuildResultsWithDetailed(failed, errored); + + // Act & Assert + var ex = Assert.Throws(() => results.AssertNoFailedItems()); + Assert.Contains("item-1:fail", ex.Message); + Assert.Contains("item-2:errored", ex.Message); + } + + [Fact] + public void AssertNoFailedItems_RecursesIntoSubResults() + { + // Arrange + var failed = new EvalItemResult("sub-1", "fail", new[] + { + new EvalScoreResult("relevance", 0.1, Passed: false), + }); + var sub = BuildResultsWithDetailed(failed); + var top = new AgentEvaluationResults("top", Array.Empty()) + { + SubResults = new Dictionary { ["agent"] = sub }, + }; + + // Act & Assert + var ex = Assert.Throws(() => top.AssertNoFailedItems()); + Assert.Contains("sub-1:fail", ex.Message); + } + // --------------------------------------------------------------- // Mixed evaluator tests // --------------------------------------------------------------- From c01b3924df419b0d07b1b39c09ea942ffd2bee5d Mon Sep 17 00:00:00 2001 From: Ben Thomas <25218250+alliscode@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:29:26 -0700 Subject: [PATCH 03/12] .NET: feat(foundry-evals): accept GeneratedEvaluatorRef in evaluators= Adds FoundryEvaluatorSpec, a readonly-struct union with implicit conversions from both string and GeneratedEvaluatorRef so call sites can mix built-in evaluator names with rubric evaluator references: var evals = new FoundryEvals( projectClient, model, new GeneratedEvaluatorRef("policy-rubric", "3"), FoundryEvals.Relevance, FoundryEvals.Coherence); FoundryEvals constructors (3 overloads), EvaluateTracesAsync, and EvaluateFoundryTargetAsync now take FoundryEvaluatorSpec[]/params instead of string[]/params. Existing call sites using string literals or string[] keep working unchanged via implicit conversion. FoundryEvalConverter.BuildTestingCriteria emits the documented Foundry wire format for rubric refs: { "type": "azure_ai_evaluator", "name": , "evaluator_name": , "evaluator_version": , // omitted when null "initialization_parameters": { "deployment_name": }, "data_mapping": { conversation arrays, optional tool_definitions } } WireTestingCriterion gains an optional EvaluatorVersion field. Rubric refs are preserved through FilterToolEvaluators (tool-aware but not tool-required) and ignored by FindMissingGroundTruthEvaluators. A versionless ref emits a Trace.TraceWarning at criterion-build time so CI authors notice the floating version (mirrors the Python warning). Adds 6 new Foundry unit tests (3 BuildTestingCriteria rubric paths, 1 FindMissingGroundTruthEvaluators, 1 FilterToolEvaluators preservation, 1 mixed-order). 369/369 Foundry tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Evaluation/FoundryEvalConverter.cs | 71 ++++++++++++-- .../Evaluation/FoundryEvalWireModels.cs | 3 + .../Evaluation/FoundryEvals.cs | 85 ++++++++++++----- .../Evaluation/FoundryEvaluatorSpec.cs | 95 +++++++++++++++++++ .../FoundryEvalConverterTests.cs | 77 +++++++++++++++ .../FoundryEvalsTests.cs | 28 ++++-- 6 files changed, 323 insertions(+), 36 deletions(-) create mode 100644 dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvaluatorSpec.cs diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalConverter.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalConverter.cs index 0754e2bc76..75fe99db74 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalConverter.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalConverter.cs @@ -148,19 +148,68 @@ internal static WireEvalItemPayload ConvertEvalItem(EvalItem item, IConversation /// /// Builds the testing_criteria array for evals.create(). /// - /// Evaluator names (short or fully-qualified). + /// + /// Evaluator specs — built-in evaluator names (short or fully-qualified) and/or + /// instances for pre-existing rubric evaluators. + /// /// Model deployment name for the LLM judge. /// /// Whether to include field-level data mapping (required for JSONL data source). /// + /// + /// Whether the mapped data items include tool definitions. Used to add a + /// tool_definitions mapping entry for rubric evaluators (built-in evaluators + /// derive this from their own membership). + /// internal static List BuildTestingCriteria( - IEnumerable evaluators, + IEnumerable evaluators, string model, - bool includeDataMapping = false) + bool includeDataMapping = false, + bool includeToolDefinitions = false) { var criteria = new List(); - foreach (var name in evaluators) + foreach (var spec in evaluators) { + if (spec.IsRubric) + { + var @ref = spec.GeneratedRef!; + Dictionary? refMapping = null; + if (includeDataMapping) + { + // Rubric evaluators accept conversation arrays like agent evaluators, + // plus tool_definitions when items are tool-aware. + refMapping = new Dictionary + { + ["query"] = "{{item.query_messages}}", + ["response"] = "{{item.response_messages}}", + }; + + if (includeToolDefinitions) + { + refMapping["tool_definitions"] = "{{item.tool_definitions}}"; + } + } + + criteria.Add(new WireTestingCriterion + { + Name = @ref.DisplayName ?? @ref.Name, + EvaluatorName = @ref.Name, + EvaluatorVersion = @ref.Version, + InitializationParameters = new WireInitParams { DeploymentName = model }, + DataMapping = refMapping, + }); + + if (@ref.Version is null) + { + System.Diagnostics.Trace.TraceWarning( + "GeneratedEvaluatorRef '{0}' has no pinned version; the eval run will resolve to whichever version is current at execution time. Pin the version for reproducible runs.", + @ref.Name); + } + + continue; + } + + var name = spec.BuiltinName!; var qualified = ResolveEvaluator(name); var shortName = name.StartsWith("builtin.", StringComparison.Ordinal) ? name.Substring("builtin.".Length) @@ -248,8 +297,12 @@ internal static WireItemSchema BuildItemSchema(bool hasContext = false, bool has /// Returns the subset of that require a ground-truth /// (reference) value but cannot be evaluated because no item provided one. /// + /// + /// Rubric references () are skipped — they are not + /// ground-truth–dependent on the wire. + /// internal static List FindMissingGroundTruthEvaluators( - IEnumerable evaluators, + IEnumerable evaluators, bool hasGroundTruth) { if (hasGroundTruth) @@ -258,8 +311,14 @@ internal static List FindMissingGroundTruthEvaluators( } var missing = new List(); - foreach (var name in evaluators) + foreach (var spec in evaluators) { + if (spec.IsRubric) + { + continue; + } + + var name = spec.BuiltinName!; if (GroundTruthEvaluators.Contains(ResolveEvaluator(name))) { missing.Add(name); diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalWireModels.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalWireModels.cs index c05232575c..f08b7e04f3 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalWireModels.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvalWireModels.cs @@ -137,6 +137,9 @@ internal sealed class WireTestingCriterion [JsonPropertyName("evaluator_name")] public required string EvaluatorName { get; init; } + [JsonPropertyName("evaluator_version")] + public string? EvaluatorVersion { get; init; } + [JsonPropertyName("initialization_parameters")] public required WireInitParams InitializationParameters { get; init; } diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs index 675ae38dfe..49dd9ba68f 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs @@ -43,7 +43,7 @@ public sealed class FoundryEvals : IAgentEvaluator private readonly EvaluationClient _evaluationClient; private readonly string _model; - private readonly string[] _evaluatorNames; + private readonly FoundryEvaluatorSpec[] _evaluators; private readonly IConversationSplitter? _splitter; private readonly double _pollIntervalSeconds = 5.0; private readonly double _timeoutSeconds = 300.0; @@ -58,17 +58,19 @@ public sealed class FoundryEvals : IAgentEvaluator /// The Azure AI Foundry project client. /// Model deployment name for the LLM judge evaluator. /// - /// Names of evaluators to use (e.g., , ). - /// When empty, defaults to relevance and coherence. + /// Evaluator specs to use. Each entry can be a built-in evaluator name (string, for example + /// ) or a for a rubric evaluator + /// already registered in the Foundry project. When empty, defaults to relevance, coherence, + /// and task adherence. /// - public FoundryEvals(AIProjectClient projectClient, string model, params string[] evaluators) + public FoundryEvals(AIProjectClient projectClient, string model, params FoundryEvaluatorSpec[] evaluators) { ArgumentNullException.ThrowIfNull(projectClient); ArgumentException.ThrowIfNullOrWhiteSpace(model); this._evaluationClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient(); this._model = model; - this._evaluatorNames = evaluators.Length > 0 + this._evaluators = evaluators.Length > 0 ? evaluators : [Relevance, Coherence, TaskAdherence]; } @@ -84,14 +86,14 @@ public FoundryEvals(AIProjectClient projectClient, string model, params string[] /// or a custom implementation. /// /// - /// Names of evaluators to use (e.g., , ). - /// When empty, defaults to relevance and coherence. + /// Evaluator specs (built-in names and/or instances). + /// When empty, defaults to relevance, coherence, and task adherence. /// public FoundryEvals( AIProjectClient projectClient, string model, IConversationSplitter? splitter, - params string[] evaluators) + params FoundryEvaluatorSpec[] evaluators) : this(projectClient, model, evaluators) { this._splitter = splitter; @@ -107,14 +109,16 @@ public FoundryEvals( /// /// Seconds between status polls (default 5). /// Maximum seconds to wait for completion (default 300). - /// Evaluator names to use. + /// + /// Evaluator specs (built-in names and/or instances). + /// public FoundryEvals( AIProjectClient projectClient, string model, IConversationSplitter? splitter, double pollIntervalSeconds, double timeoutSeconds, - params string[] evaluators) + params FoundryEvaluatorSpec[] evaluators) : this(projectClient, model, splitter, evaluators) { ArgumentOutOfRangeException.ThrowIfLessThanOrEqual(pollIntervalSeconds, 0); @@ -149,10 +153,10 @@ public async Task EvaluateAsync( bool allHaveGroundTruth = payloads.Count > 0 && payloads.All(p => p.GroundTruth is not null); // Filter out tool evaluators if no items have tools; auto-add ToolCallAccuracy if tools present - var evaluators = FilterToolEvaluators(this._evaluatorNames, hasTools); - if (hasTools && !evaluators.Any(e => FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(e)))) + var evaluators = FilterToolEvaluators(this._evaluators, hasTools); + if (hasTools && !HasToolEvaluator(evaluators)) { - evaluators = [.. evaluators, ToolCallAccuracy]; + evaluators = [.. evaluators, (FoundryEvaluatorSpec)ToolCallAccuracy]; } // Fail fast if a ground-truth evaluator (e.g. similarity) is requested but not @@ -178,7 +182,7 @@ public async Task EvaluateAsync( ItemSchema = FoundryEvalConverter.BuildItemSchema(hasContext, hasTools, hasGroundTruth), }, TestingCriteria = FoundryEvalConverter.BuildTestingCriteria( - evaluators, this._model, includeDataMapping: true), + evaluators, this._model, includeDataMapping: true, includeToolDefinitions: hasTools), }; var createEvalJson = JsonSerializer.Serialize(createEvalPayload, s_jsonOptions); @@ -287,7 +291,11 @@ public async Task EvaluateAsync( /// Evaluate specific OTel trace IDs from App Insights. /// Filter traces by agent ID (used with ). /// Hours of trace history to evaluate (default 24). - /// Evaluator names. Defaults to relevance, coherence, and task adherence. + /// + /// Evaluator specs. Each entry can be a built-in evaluator name (string) or a + /// for a rubric evaluator. Defaults to relevance, + /// coherence, and task adherence. + /// /// Display name for the evaluation. /// Seconds between status polls (default 5). /// Maximum seconds to wait for completion (default 300). @@ -300,7 +308,7 @@ public static async Task EvaluateTracesAsync( IEnumerable? traceIds = null, string? agentId = null, int lookbackHours = 24, - string[]? evaluators = null, + FoundryEvaluatorSpec[]? evaluators = null, string evalName = "Agent Framework Trace Eval", double pollIntervalSeconds = 5.0, double timeoutSeconds = 300.0, @@ -320,7 +328,7 @@ public static async Task EvaluateTracesAsync( } var evalClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient(); - var resolvedEvaluators = evaluators is { Length: > 0 } + FoundryEvaluatorSpec[] resolvedEvaluators = evaluators is { Length: > 0 } ? evaluators : [Relevance, Coherence, TaskAdherence]; @@ -440,7 +448,10 @@ public static async Task EvaluateTracesAsync( /// Model deployment name for the LLM judge evaluator. /// Target configuration (must include a "type" key, e.g. "azure_ai_agent"). /// Queries for Foundry to send to the target. - /// Evaluator names. Defaults to relevance, coherence, and task adherence. + /// + /// Evaluator specs (built-in names and/or instances). + /// Defaults to relevance, coherence, and task adherence. + /// /// Display name for the evaluation. /// Seconds between status polls (default 5). /// Maximum seconds to wait for completion (default 300). @@ -451,7 +462,7 @@ public static async Task EvaluateFoundryTargetAsync( string model, IDictionary target, IEnumerable testQueries, - string[]? evaluators = null, + FoundryEvaluatorSpec[]? evaluators = null, string evalName = "Agent Framework Target Eval", double pollIntervalSeconds = 5.0, double timeoutSeconds = 300.0, @@ -473,7 +484,7 @@ public static async Task EvaluateFoundryTargetAsync( } var evalClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient(); - var resolvedEvaluators = evaluators is { Length: > 0 } + FoundryEvaluatorSpec[] resolvedEvaluators = evaluators is { Length: > 0 } ? evaluators : [Relevance, Coherence, TaskAdherence]; @@ -917,20 +928,46 @@ private static EvalItemResult ParseDetailedItem(JsonElement outputItem) return result; } - internal static string[] FilterToolEvaluators(string[] evaluators, bool hasTools) + internal static FoundryEvaluatorSpec[] FilterToolEvaluators(FoundryEvaluatorSpec[] evaluators, bool hasTools) { if (hasTools) { return evaluators; } - var filtered = Array.FindAll(evaluators, e => - !FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(e))); + var filtered = Array.FindAll(evaluators, spec => + { + if (spec.IsRubric) + { + // Rubric refs are tool-aware but not tool-required; preserve them. + return true; + } + + return !FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(spec.BuiltinName!)); + }); return filtered.Length > 0 ? filtered : throw new ArgumentException( "All configured evaluators require tool definitions, but no tool calls were found in the eval items. " - + $"Tool evaluators: {string.Join(", ", evaluators)}. Either add tool call content to your EvalItems or remove tool-type evaluators."); + + $"Tool evaluators: {string.Join(", ", evaluators.Select(e => e.ToString()))}. Either add tool call content to your EvalItems or remove tool-type evaluators."); + } + + private static bool HasToolEvaluator(FoundryEvaluatorSpec[] evaluators) + { + foreach (var spec in evaluators) + { + if (spec.IsRubric) + { + continue; + } + + if (FoundryEvalConverter.ToolEvaluators.Contains(FoundryEvalConverter.ResolveEvaluator(spec.BuiltinName!))) + { + return true; + } + } + + return false; } } diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvaluatorSpec.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvaluatorSpec.cs new file mode 100644 index 0000000000..5712c2f87f --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvaluatorSpec.cs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; + +namespace Microsoft.Agents.AI.Foundry; + +/// +/// Specifies a single evaluator for a run — either a built-in +/// Foundry evaluator (referenced by short or fully-qualified name) or a pre-existing rubric +/// evaluator (referenced by ). +/// +/// +/// +/// Both and are implicitly convertible +/// to , so call sites can mix the two: +/// +/// +/// var evals = new FoundryEvals( +/// projectClient, +/// "gpt-4o-mini", +/// new GeneratedEvaluatorRef("policy-rubric", Version: "3"), +/// FoundryEvals.Relevance, +/// FoundryEvals.Coherence); +/// +/// +public readonly struct FoundryEvaluatorSpec : IEquatable +{ + private FoundryEvaluatorSpec(string? builtinName, GeneratedEvaluatorRef? generatedRef) + { + this.BuiltinName = builtinName; + this.GeneratedRef = generatedRef; + } + + /// + /// Initializes a new for a built-in evaluator by name + /// (for example "relevance" or "builtin.relevance"). + /// + /// Built-in evaluator name. + public FoundryEvaluatorSpec(string builtinName) + : this(builtinName ?? throw new ArgumentNullException(nameof(builtinName)), null) + { + } + + /// + /// Initializes a new for a generated rubric evaluator + /// previously registered with the provider. + /// + /// Reference to the rubric evaluator. + public FoundryEvaluatorSpec(GeneratedEvaluatorRef generatedRef) + : this(null, generatedRef ?? throw new ArgumentNullException(nameof(generatedRef))) + { + } + + /// Gets the built-in evaluator name, or when this is a rubric reference. + public string? BuiltinName { get; } + + /// Gets the rubric reference, or when this is a built-in evaluator. + public GeneratedEvaluatorRef? GeneratedRef { get; } + + /// Gets whether this spec references a built-in evaluator. + public bool IsBuiltin => this.BuiltinName is not null; + + /// Gets whether this spec references a generated rubric evaluator. + public bool IsRubric => this.GeneratedRef is not null; + + /// Implicit conversion from a built-in evaluator name. + public static implicit operator FoundryEvaluatorSpec(string builtinName) => new(builtinName); + + /// Implicit conversion from a . + public static implicit operator FoundryEvaluatorSpec(GeneratedEvaluatorRef generatedRef) => new(generatedRef); + + /// + public bool Equals(FoundryEvaluatorSpec other) + => this.BuiltinName == other.BuiltinName + && Equals(this.GeneratedRef, other.GeneratedRef); + + /// + public override bool Equals(object? obj) => obj is FoundryEvaluatorSpec other && this.Equals(other); + + /// + public override int GetHashCode() + => HashCode.Combine(this.BuiltinName, this.GeneratedRef); + + /// Equality operator. + public static bool operator ==(FoundryEvaluatorSpec left, FoundryEvaluatorSpec right) => left.Equals(right); + + /// Inequality operator. + public static bool operator !=(FoundryEvaluatorSpec left, FoundryEvaluatorSpec right) => !left.Equals(right); + + /// + public override string ToString() + => this.IsRubric + ? $"GeneratedEvaluatorRef({this.GeneratedRef!.Name}@{this.GeneratedRef.Version ?? "latest"})" + : this.BuiltinName ?? ""; +} diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalConverterTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalConverterTests.cs index aea1459e5e..4a54e21f9f 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalConverterTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalConverterTests.cs @@ -305,6 +305,71 @@ public void BuildTestingCriteria_WithoutDataMapping_OmitsMappingField() Assert.Null(criteria[0].DataMapping); } + [Fact] + public void BuildTestingCriteria_WithRubricRef_EmitsAzureAiEvaluatorWithVersion() + { + var rubric = new GeneratedEvaluatorRef("policy-rubric", Version: "3", DisplayName: "Policy"); + var criteria = FoundryEvalConverter.BuildTestingCriteria( + [rubric], "gpt-4o-mini", includeDataMapping: true); + + Assert.Single(criteria); + var entry = criteria[0]; + Assert.Equal("azure_ai_evaluator", entry.Type); + Assert.Equal("Policy", entry.Name); + Assert.Equal("policy-rubric", entry.EvaluatorName); + Assert.Equal("3", entry.EvaluatorVersion); + Assert.Equal("gpt-4o-mini", entry.InitializationParameters.DeploymentName); + + var mapping = entry.DataMapping; + Assert.NotNull(mapping); + Assert.Equal("{{item.query_messages}}", mapping["query"]); + Assert.Equal("{{item.response_messages}}", mapping["response"]); + Assert.False(mapping.ContainsKey("tool_definitions")); + } + + [Fact] + public void BuildTestingCriteria_WithVersionlessRubricRef_OmitsVersionField() + { + var rubric = GeneratedEvaluatorRef.Latest("policy-rubric"); + var criteria = FoundryEvalConverter.BuildTestingCriteria( + [rubric], "gpt-4o-mini", includeDataMapping: false); + + Assert.Single(criteria); + var entry = criteria[0]; + Assert.Equal("policy-rubric", entry.Name); // falls back to Name when DisplayName is null + Assert.Equal("policy-rubric", entry.EvaluatorName); + Assert.Null(entry.EvaluatorVersion); + Assert.Null(entry.DataMapping); + } + + [Fact] + public void BuildTestingCriteria_RubricRefWithTools_IncludesToolDefinitions() + { + var rubric = new GeneratedEvaluatorRef("tool-aware-rubric", Version: "1"); + var criteria = FoundryEvalConverter.BuildTestingCriteria( + [rubric], "gpt-4o-mini", includeDataMapping: true, includeToolDefinitions: true); + + Assert.Single(criteria); + var mapping = criteria[0].DataMapping; + Assert.NotNull(mapping); + Assert.True(mapping.ContainsKey("tool_definitions")); + Assert.Equal("{{item.tool_definitions}}", mapping["tool_definitions"]); + } + + [Fact] + public void BuildTestingCriteria_MixedSpecs_PreservesOrder() + { + var rubric = new GeneratedEvaluatorRef("policy-rubric", Version: "2"); + var criteria = FoundryEvalConverter.BuildTestingCriteria( + ["relevance", rubric, "coherence"], "gpt-4o-mini", includeDataMapping: false); + + Assert.Equal(3, criteria.Count); + Assert.Equal("builtin.relevance", criteria[0].EvaluatorName); + Assert.Equal("policy-rubric", criteria[1].EvaluatorName); + Assert.Equal("2", criteria[1].EvaluatorVersion); + Assert.Equal("builtin.coherence", criteria[2].EvaluatorName); + } + // --------------------------------------------------------------- // FoundryEvalConverter.BuildItemSchema tests // --------------------------------------------------------------- @@ -391,6 +456,18 @@ public void FindMissingGroundTruthEvaluators_NoGroundTruthEvaluators_ReturnsEmpt Assert.Empty(missing); } + [Fact] + public void FindMissingGroundTruthEvaluators_IgnoresRubricRefs() + { + // Rubric refs are not ground-truth–dependent and must be skipped even when + // no items carry ExpectedOutput. + var rubric = new GeneratedEvaluatorRef("policy-rubric", Version: "1"); + var missing = FoundryEvalConverter.FindMissingGroundTruthEvaluators( + [rubric, "relevance"], hasGroundTruth: false); + + Assert.Empty(missing); + } + // --------------------------------------------------------------- // FoundryEvalConverter.ConvertMessage DataContent test // --------------------------------------------------------------- diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs index a09dcf03fc..8fdf633ab0 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs @@ -13,7 +13,7 @@ public sealed class FoundryEvalsTests public void FilterToolEvaluators_AllToolEvaluators_NoTools_ThrowsArgumentException() { // All configured evaluators are tool-type, but no items have tools. - var evaluators = new[] { "tool_call_accuracy", "tool_selection" }; + var evaluators = new FoundryEvaluatorSpec[] { "tool_call_accuracy", "tool_selection" }; var ex = Assert.Throws( () => FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false)); @@ -24,23 +24,39 @@ public void FilterToolEvaluators_AllToolEvaluators_NoTools_ThrowsArgumentExcepti [Fact] public void FilterToolEvaluators_MixedEvaluators_NoTools_FiltersToolOnes() { - var evaluators = new[] { "relevance", "tool_call_accuracy", "coherence" }; + var evaluators = new FoundryEvaluatorSpec[] { "relevance", "tool_call_accuracy", "coherence" }; var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false); Assert.Equal(2, result.Length); - Assert.Contains("relevance", result); - Assert.Contains("coherence", result); - Assert.DoesNotContain("tool_call_accuracy", result); + Assert.Contains((FoundryEvaluatorSpec)"relevance", result); + Assert.Contains((FoundryEvaluatorSpec)"coherence", result); + Assert.DoesNotContain((FoundryEvaluatorSpec)"tool_call_accuracy", result); } [Fact] public void FilterToolEvaluators_HasTools_ReturnsAllEvaluators() { - var evaluators = new[] { "relevance", "tool_call_accuracy" }; + var evaluators = new FoundryEvaluatorSpec[] { "relevance", "tool_call_accuracy" }; var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: true); Assert.Equal(evaluators, result); } + + [Fact] + public void FilterToolEvaluators_PreservesRubricRefs_WhenNoTools() + { + // Rubric refs are tool-aware but never tool-required, so they must survive filtering + // when no items carry tool definitions. + var rubric = new GeneratedEvaluatorRef("policy-rubric", Version: "3"); + var evaluators = new FoundryEvaluatorSpec[] { "relevance", rubric, "tool_call_accuracy" }; + + var result = FoundryEvals.FilterToolEvaluators(evaluators, hasTools: false); + + Assert.Equal(2, result.Length); + Assert.Contains((FoundryEvaluatorSpec)"relevance", result); + Assert.Contains(result, s => s.IsRubric && s.GeneratedRef!.Name == "policy-rubric"); + Assert.DoesNotContain((FoundryEvaluatorSpec)"tool_call_accuracy", result); + } } From 55e829a509c2163cf22991c9661721f32a075fde Mon Sep 17 00:00:00 2001 From: Ben Thomas <25218250+alliscode@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:34:44 -0700 Subject: [PATCH 04/12] .NET: feat(foundry-evals): parse rubric dimension_scores into RubricScore Adds FoundryEvals.ParseRubricScores, called per result inside ParseDetailedItem. Each EvalScoreResult now populates Dimensions when the evaluator's sample carries a rubric breakdown. Accepts three shapes for forward compatibility with provider SDK iterations: 1. sample.properties.dimension_scores (canonical Foundry runtime shape) 2. sample.properties.rubric_scores (preview/legacy key) 3. top-level sample.dimension_scores / sample.rubric_scores (defensive fallback) Entries missing 'id', 'weight', or 'applicable' are skipped without invalidating well-formed siblings. Non-applicable dimensions may omit 'score' (parsed as null). Adds 6 unit tests covering canonical and legacy keys, top-level fallback, no-match returns null, malformed-entry skipping, and the non-applicable null-score path. 375/375 Foundry tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Evaluation/FoundryEvals.cs | 145 +++++++++++++++++- .../FoundryEvalsTests.cs | 136 ++++++++++++++++ 2 files changed, 280 insertions(+), 1 deletion(-) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs index 49dd9ba68f..b8c524ba8d 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs @@ -842,7 +842,13 @@ private static EvalItemResult ParseDetailedItem(JsonElement outputItem) passed = pp.ValueKind == JsonValueKind.True; } - scores.Add(new EvalScoreResult(name, score, passed)); + IReadOnlyList? dimensions = null; + if (r.TryGetProperty("sample", out var perResultSample)) + { + dimensions = ParseRubricScores(perResultSample); + } + + scores.Add(new EvalScoreResult(name, score, passed) { Dimensions = dimensions }); } } @@ -928,6 +934,143 @@ private static EvalItemResult ParseDetailedItem(JsonElement outputItem) return result; } + private static readonly string[] s_rubricDimensionKeys = ["dimension_scores", "rubric_scores"]; + + /// + /// Extracts the per-dimension list from a result-level sample + /// payload, when present. Accepts several legacy/canonical shapes for forward compatibility + /// with provider SDK changes: + /// + /// + /// + /// + /// sample.properties.dimension_scores (canonical Foundry shape). + /// + /// + /// sample.properties.rubric_scores (preview / legacy key). + /// + /// + /// Top-level sample.dimension_scores / sample.rubric_scores as a + /// defensive fallback. + /// + /// + /// Returns when no rubric scores are present (the evaluator was not + /// a rubric evaluator). Malformed entries (missing id, weight, or applicable) + /// are skipped without failing the whole list. + /// + internal static List? ParseRubricScores(JsonElement sample) + { + if (sample.ValueKind != JsonValueKind.Object) + { + return null; + } + + // Prefer sample.properties. then fall back to top-level sample.. + if (sample.TryGetProperty("properties", out var properties) + && properties.ValueKind == JsonValueKind.Object) + { + foreach (var key in s_rubricDimensionKeys) + { + if (properties.TryGetProperty(key, out var raw)) + { + var parsed = ParseDimensionEntries(raw); + if (parsed.Count > 0) + { + return parsed; + } + } + } + } + + foreach (var key in s_rubricDimensionKeys) + { + if (sample.TryGetProperty(key, out var raw)) + { + var parsed = ParseDimensionEntries(raw); + if (parsed.Count > 0) + { + return parsed; + } + } + } + + return null; + } + + private static List ParseDimensionEntries(JsonElement raw) + { + var parsed = new List(); + if (raw.ValueKind != JsonValueKind.Array) + { + return parsed; + } + + foreach (var entry in raw.EnumerateArray()) + { + if (entry.ValueKind != JsonValueKind.Object) + { + continue; + } + + if (!entry.TryGetProperty("id", out var idProp) + || !entry.TryGetProperty("weight", out var weightProp) + || !entry.TryGetProperty("applicable", out var applicableProp)) + { + continue; + } + + string? id = idProp.ValueKind switch + { + JsonValueKind.String => idProp.GetString(), + JsonValueKind.Number => idProp.GetRawText(), + _ => null, + }; + if (string.IsNullOrEmpty(id)) + { + continue; + } + + if (weightProp.ValueKind != JsonValueKind.Number + || !weightProp.TryGetInt32(out var weight)) + { + continue; + } + + if (applicableProp.ValueKind is not (JsonValueKind.True or JsonValueKind.False)) + { + continue; + } + + int? score = null; + if (entry.TryGetProperty("score", out var scoreProp) + && scoreProp.ValueKind == JsonValueKind.Number) + { + if (scoreProp.TryGetInt32(out var intScore)) + { + score = intScore; + } + else if (scoreProp.TryGetDouble(out var doubleScore)) + { + score = (int)doubleScore; + } + } + + string reason = entry.TryGetProperty("reason", out var reasonProp) + && reasonProp.ValueKind == JsonValueKind.String + ? reasonProp.GetString() ?? string.Empty + : string.Empty; + + parsed.Add(new RubricScore( + Id: id!, + Score: score, + Applicable: applicableProp.ValueKind == JsonValueKind.True, + Weight: weight, + Reason: reason)); + } + + return parsed; + } + internal static FoundryEvaluatorSpec[] FilterToolEvaluators(FoundryEvaluatorSpec[] evaluators, bool hasTools) { if (hasTools) diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs index 8fdf633ab0..7cca9ce901 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft. All rights reserved. using System; +using System.Linq; +using System.Text.Json; namespace Microsoft.Agents.AI.Foundry.UnitTests; @@ -59,4 +61,138 @@ public void FilterToolEvaluators_PreservesRubricRefs_WhenNoTools() Assert.Contains(result, s => s.IsRubric && s.GeneratedRef!.Name == "policy-rubric"); Assert.DoesNotContain((FoundryEvaluatorSpec)"tool_call_accuracy", result); } + + // --------------------------------------------------------------- + // FoundryEvals.ParseRubricScores tests + // --------------------------------------------------------------- + + [Fact] + public void ParseRubricScores_CanonicalDimensionScoresKey_ParsesAllFields() + { + // Per Microsoft Learn docs, runtime output uses properties.dimension_scores. + const string Json = """ + { + "properties": { + "dimension_scores": [ + { "id": "intent_recognition", "score": 5, "applicable": true, "weight": 9, "reason": "Identified correctly." }, + { "id": "general_quality", "score": 4, "applicable": true, "weight": 5, "reason": "Strong overall." } + ] + } + } + """; + using var doc = JsonDocument.Parse(Json); + + var result = FoundryEvals.ParseRubricScores(doc.RootElement); + + Assert.NotNull(result); + Assert.Equal(2, result!.Count); + Assert.Equal(["intent_recognition", "general_quality"], result.Select(r => r.Id)); + Assert.Equal([5, 4], result.Select(r => r.Score)); + Assert.Equal([9, 5], result.Select(r => r.Weight)); + Assert.True(result[0].Applicable); + Assert.Equal("Identified correctly.", result[0].Reason); + } + + [Fact] + public void ParseRubricScores_LegacyRubricScoresKey_StillSupported() + { + // Preview builds used the rubric_scores key; we still accept it for back-compat. + const string Json = """ + { + "properties": { + "rubric_scores": [ + { "id": "a", "score": 3, "applicable": true, "weight": 1, "reason": "r" } + ] + } + } + """; + using var doc = JsonDocument.Parse(Json); + + var result = FoundryEvals.ParseRubricScores(doc.RootElement); + + Assert.NotNull(result); + Assert.Single(result!); + Assert.Equal("a", result[0].Id); + } + + [Fact] + public void ParseRubricScores_TopLevelKey_FallsBack() + { + // Defensive fallback when SDK shape omits the 'properties' wrapper. + const string Json = """ + { + "dimension_scores": [ + { "id": "x", "score": 2, "applicable": true, "weight": 1, "reason": "" } + ] + } + """; + using var doc = JsonDocument.Parse(Json); + + var result = FoundryEvals.ParseRubricScores(doc.RootElement); + + Assert.NotNull(result); + Assert.Single(result!); + Assert.Equal("x", result[0].Id); + } + + [Fact] + public void ParseRubricScores_NoRubricKeys_ReturnsNull() + { + const string Json = """ + { "properties": { "other_field": [] } } + """; + using var doc = JsonDocument.Parse(Json); + + var result = FoundryEvals.ParseRubricScores(doc.RootElement); + + Assert.Null(result); + } + + [Fact] + public void ParseRubricScores_SkipsMalformedEntries() + { + // Entries missing weight or applicable are skipped, but well-formed siblings are kept. + const string Json = """ + { + "properties": { + "dimension_scores": [ + { "id": "good", "score": 3, "applicable": true, "weight": 1, "reason": "ok" }, + { "id": "bad-no-weight", "score": 2, "applicable": true, "reason": "x" }, + { "id": "bad-no-applicable", "score": 2, "weight": 1, "reason": "x" } + ] + } + } + """; + using var doc = JsonDocument.Parse(Json); + + var result = FoundryEvals.ParseRubricScores(doc.RootElement); + + Assert.NotNull(result); + Assert.Single(result!); + Assert.Equal("good", result[0].Id); + } + + [Fact] + public void ParseRubricScores_NonApplicableDimension_KeepsNullScoreWhenMissing() + { + // Non-applicable dimensions can legitimately omit score (or set it to null). + const string Json = """ + { + "properties": { + "dimension_scores": [ + { "id": "skipped", "applicable": false, "weight": 5, "reason": "n/a" } + ] + } + } + """; + using var doc = JsonDocument.Parse(Json); + + var result = FoundryEvals.ParseRubricScores(doc.RootElement); + + Assert.NotNull(result); + Assert.Single(result!); + Assert.Equal("skipped", result[0].Id); + Assert.False(result[0].Applicable); + Assert.Null(result[0].Score); + } } From 33f064df2e93c0e769caf298692caf79a8109416 Mon Sep 17 00:00:00 2001 From: Ben Thomas <25218250+alliscode@users.noreply.github.com> Date: Mon, 1 Jun 2026 15:41:27 -0700 Subject: [PATCH 05/12] .NET: feat(samples): Evaluation_FoundryRubric end-to-end sample Adds dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric mirroring the Python evaluate_with_rubric_sample.py: - Fetches a pre-existing Foundry agent via AgentAdministrationClient (GetAgentAsync for latest, GetAgentVersionAsync when FOUNDRY_AGENT_VERSION is pinned). - References a rubric evaluator by GeneratedEvaluatorRef(name, version); falls back to GeneratedEvaluatorRef.Latest(name) with the documented floating-version warning. - Mixes the rubric with FoundryEvals.Relevance and FoundryEvals.Coherence in a single FoundryEvals run (implicit string-and-ref conversion). - Prints per-dimension breakdowns from EvalScoreResult.Dimensions for each item. - Demonstrates a CI quality gate with AssertDimensionScoreAtLeast("general_quality", 3.0). Documents the FOUNDRY_PROJECT_ENDPOINT footgun (must be project-scoped URL .../api/projects/, not the bare Azure OpenAI endpoint) and the Eval-Definition-vs-Rubric-Evaluator distinction in the README. Ships a .env.example with the FOUNDRY_* variables. Registers the project in agent-framework-dotnet.slnx and cross-links from the sibling Evaluation_Multimodal / Evaluation_ExpectedOutputs READMEs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- dotnet/agent-framework-dotnet.slnx | 1 + .../Evaluation_ExpectedOutputs/README.md | 1 + .../Evaluation_Multimodal/README.md | 1 + .../Evaluation_FoundryRubric.csproj | 15 ++ .../Evaluation_FoundryRubric/Program.cs | 142 ++++++++++++++++++ .../Evaluation_FoundryRubric/README.md | 57 +++++++ 6 files changed, 217 insertions(+) create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Evaluation_FoundryRubric.csproj create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs create mode 100644 dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/README.md diff --git a/dotnet/agent-framework-dotnet.slnx b/dotnet/agent-framework-dotnet.slnx index e395627bc9..cf3cc1c773 100644 --- a/dotnet/agent-framework-dotnet.slnx +++ b/dotnet/agent-framework-dotnet.slnx @@ -403,6 +403,7 @@ + diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/README.md b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/README.md index 34f16865d2..77a2df6426 100644 --- a/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/README.md +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_ExpectedOutputs/README.md @@ -31,3 +31,4 @@ dotnet run --project .\Evaluation_ExpectedOutputs - [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation with built-in and custom checks - [Evaluation_FoundryQuality](../../../05-end-to-end/Evaluation/Evaluation_FoundryQuality/) — Cloud-based quality evaluation with Foundry evaluators +- [Evaluation_FoundryRubric](../../../05-end-to-end/Evaluation/Evaluation_FoundryRubric/) — Rubric (adaptive) evaluators with per-dimension scores diff --git a/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/README.md b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/README.md index d02447651b..0b7dabc808 100644 --- a/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/README.md +++ b/dotnet/samples/02-agents/Evaluation/Evaluation_Multimodal/README.md @@ -26,4 +26,5 @@ dotnet run --project .\Evaluation_Multimodal - [Evaluation_SimpleEval](../Evaluation_SimpleEval/) — Simplest evaluation with built-in checks and `agent.EvaluateAsync()` - [Evaluation_FoundryQuality](../../../05-end-to-end/Evaluation/Evaluation_FoundryQuality/) — Cloud-based quality evaluation with Foundry evaluators +- [Evaluation_FoundryRubric](../../../05-end-to-end/Evaluation/Evaluation_FoundryRubric/) — Rubric (adaptive) evaluators with per-dimension scores - [Evaluation_ConversationSplits](../../../05-end-to-end/Evaluation/Evaluation_ConversationSplits/) — Multi-turn conversation split strategies diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Evaluation_FoundryRubric.csproj b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Evaluation_FoundryRubric.csproj new file mode 100644 index 0000000000..6b4cb8f43e --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Evaluation_FoundryRubric.csproj @@ -0,0 +1,15 @@ + + + + Exe + net10.0 + + enable + enable + + + + + + + diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs new file mode 100644 index 0000000000..35c7dd96ab --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft. All rights reserved. + +// This sample evaluates a pre-existing Azure AI Foundry agent against a rubric evaluator +// that was authored in the Foundry portal. +// +// Rubric evaluators are LLM-as-judge evaluators with custom scoring dimensions you define +// for your domain. agent-framework consumes pre-existing rubric evaluators — they are +// authored in the Foundry portal (or via the dedicated SDK / REST surface) and referenced +// here by name and version. +// +// See: https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-evaluators/rubric-evaluators +// +// Prerequisites: +// - An Azure AI Foundry project with a deployed model. +// - A registered Foundry agent in that project (the rubric was created against this agent). +// - A rubric evaluator already created in the Foundry portal. +// - .env (or environment) populated with the FOUNDRY_* variables below. +// +// IMPORTANT: FOUNDRY_PROJECT_ENDPOINT must be the project-scoped URL +// https://.services.ai.azure.com/api/projects/ +// A bare Azure OpenAI endpoint silently fails eval submission with HTTP 500. + +using Azure.AI.Projects; +using Azure.AI.Projects.Agents; +using Azure.Identity; +using Microsoft.Agents.AI; +using Microsoft.Agents.AI.Foundry; +using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals; + +string projectEndpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") + ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); +string model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") + ?? throw new InvalidOperationException("FOUNDRY_MODEL is not set."); +string agentName = Environment.GetEnvironmentVariable("FOUNDRY_AGENT_NAME") + ?? throw new InvalidOperationException("FOUNDRY_AGENT_NAME is not set."); +string? agentVersion = Environment.GetEnvironmentVariable("FOUNDRY_AGENT_VERSION"); +string rubricName = Environment.GetEnvironmentVariable("FOUNDRY_RUBRIC_NAME") + ?? throw new InvalidOperationException("FOUNDRY_RUBRIC_NAME is not set."); +string? rubricVersion = Environment.GetEnvironmentVariable("FOUNDRY_RUBRIC_VERSION"); + +// WARNING: DefaultAzureCredential is convenient for development but requires careful +// consideration in production. Prefer ManagedIdentityCredential (or a specific credential) +// to avoid latency, unintended credential probing, and fallback security risks. +AIProjectClient projectClient = new(new Uri(projectEndpoint), new DefaultAzureCredential()); + +// 1. Connect to the pre-existing Foundry agent the rubric was created against. +FoundryAgent agent; +if (agentVersion is null) +{ + ProjectsAgentRecord agentRecord = await projectClient.AgentAdministrationClient.GetAgentAsync(agentName); + agent = projectClient.AsAIAgent(agentRecord); +} +else +{ + ProjectsAgentVersion versionRecord = await projectClient.AgentAdministrationClient.GetAgentVersionAsync(agentName, agentVersion); + agent = projectClient.AsAIAgent(versionRecord); +} + +// 2. Reference the pre-existing rubric evaluator by name + version. +// Always pin a version for reproducible CI runs; a versionless ref resolves to the +// current version at run time and emits a Trace.TraceWarning on each criterion build. +GeneratedEvaluatorRef rubric = rubricVersion is null + ? GeneratedEvaluatorRef.Latest(rubricName) + : new GeneratedEvaluatorRef(rubricName, rubricVersion); + +// 3. Mix the rubric with built-in evaluators in a single FoundryEvals config. +// The implicit conversion lets you pass strings and refs interchangeably. +FoundryEvals evals = new( + projectClient, + model, + rubric, + FoundryEvals.Relevance, + FoundryEvals.Coherence); + +// 4. Run two example queries against the agent and evaluate the outputs in one call. +string[] queries = +[ + "What's the weather like in Seattle?", + "Should I bring an umbrella to London tomorrow?", +]; + +Console.WriteLine(new string('=', 60)); +Console.WriteLine($"Evaluating '{agent.Name}' with rubric '{rubricName}' (version {rubricVersion ?? "latest"})"); +Console.WriteLine(new string('=', 60)); + +AgentEvaluationResults results = await agent.EvaluateAsync(queries, evals); + +Console.WriteLine($"Status: {results.Status}"); +Console.WriteLine($"Results: {results.Passed}/{results.Total} passed"); +if (results.ReportUrl is not null) +{ + Console.WriteLine($"Portal: {results.ReportUrl}"); +} + +Console.WriteLine(results.Passed == results.Total ? "[PASS] All passed" : $"[FAIL] {results.Failed} failed"); + +// 5. Print per-dimension breakdown for each evaluated item — this is the unique value +// of a rubric evaluator over the built-in numeric ones. +Console.WriteLine(); +Console.WriteLine(new string('=', 60)); +Console.WriteLine("Per-dimension scores"); +Console.WriteLine(new string('=', 60)); + +if (results.DetailedItems is { Count: > 0 }) +{ + for (int i = 0; i < results.DetailedItems.Count; i++) + { + EvalItemResult item = results.DetailedItems[i]; + Console.WriteLine($"Item {i + 1}{(i < queries.Length ? $" — \"{queries[i]}\"" : string.Empty)}"); + + foreach (EvalScoreResult score in item.Scores) + { + Console.WriteLine($" {score.Name}: {score.Score:F1}{(score.Passed is bool p ? (p ? " (pass)" : " (fail)") : string.Empty)}"); + if (score.Dimensions is { Count: > 0 } dims) + { + foreach (RubricScore d in dims) + { + string scoreStr = d.Score is int s ? s.ToString() : "n/a"; + Console.WriteLine($" - {d.Id}: {scoreStr} (weight={d.Weight}, applicable={d.Applicable})"); + } + } + } + + Console.WriteLine(); + } +} + +// 6. CI quality gate — fail the build if a critical dimension drops below threshold. +// Replace "general_quality" with whatever dimension id your rubric actually defines. +Console.WriteLine(new string('=', 60)); +Console.WriteLine("Per-dimension quality gate"); +Console.WriteLine(new string('=', 60)); + +try +{ + results.AssertDimensionScoreAtLeast("general_quality", minScore: 3.0, evaluator: rubricName); + Console.WriteLine($"[PASS] {results.ProviderName}: general_quality >= 3 on every item"); +} +catch (InvalidOperationException ex) +{ + Console.WriteLine($"[FAIL] {results.ProviderName}: dimension gate tripped: {ex.Message}"); +} diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/README.md b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/README.md new file mode 100644 index 0000000000..eb5f73123e --- /dev/null +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/README.md @@ -0,0 +1,57 @@ +# Evaluation — Foundry Rubric + +This sample evaluates a pre-existing Azure AI Foundry agent against a **rubric evaluator** +authored in the Foundry portal. Rubric evaluators are LLM-as-judge evaluators with custom +scoring dimensions you define for your domain; agent-framework references them by name and +version, mixes them with built-in evaluators, and exposes per-dimension scores you can gate +CI on. + +See the [rubric evaluator documentation](https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-evaluators/rubric-evaluators). + +## What this sample demonstrates + +- Connecting to a pre-existing Foundry agent (`AgentAdministrationClient.GetAgentAsync`). +- Referencing a pre-existing rubric evaluator via `GeneratedEvaluatorRef(name, version)`. +- Mixing the rubric with built-in evaluators (`Relevance`, `Coherence`) in one + `FoundryEvals` run. +- Reading per-dimension breakdowns from `EvalScoreResult.Dimensions`. +- Gating CI on a per-dimension threshold via + `AgentEvaluationResults.AssertDimensionScoreAtLeast(...)`. + +## Prerequisites + +- .NET 10 SDK or later. +- Azure CLI installed and authenticated (`az login`). +- An Azure AI Foundry project with a deployed model. +- A registered Foundry agent in that project (the agent the rubric was created against). +- A rubric evaluator created in the Foundry portal. Creating rubrics through the portal + currently requires picking a Foundry agent as the generation context, so this + prerequisite is implied by having a rubric at all. + +> [!IMPORTANT] +> `FOUNDRY_PROJECT_ENDPOINT` **must** be the project-scoped URL +> `https://.services.ai.azure.com/api/projects/`. A bare Azure OpenAI +> endpoint silently fails eval submission with HTTP 500. + +> [!NOTE] +> An **Eval Definition** (a saved bundle of testing_criteria with `"object": "eval"`) is +> not the same as a **Rubric Evaluator** (a standalone evaluator with dimensions, weights, +> and a version). `GeneratedEvaluatorRef` points at the latter. + +## Environment variables + +```powershell +$env:FOUNDRY_PROJECT_ENDPOINT="https://your-resource.services.ai.azure.com/api/projects/your-project" +$env:FOUNDRY_MODEL="gpt-4o-mini" +$env:FOUNDRY_AGENT_NAME="your-agent-name" +$env:FOUNDRY_AGENT_VERSION="1" # optional; omit for latest +$env:FOUNDRY_RUBRIC_NAME="your-rubric-name" +$env:FOUNDRY_RUBRIC_VERSION="1" # optional; omit for latest (CI: pin this) +``` + +## Run the sample + +```powershell +cd dotnet/samples/05-end-to-end/Evaluation +dotnet run --project .\Evaluation_FoundryRubric +``` From 3a63442eb424ff5f3c4323b46a222072bdd9832d Mon Sep 17 00:00:00 2001 From: alliscode <25218250+alliscode@users.noreply.github.com> Date: Tue, 2 Jun 2026 08:53:24 -0700 Subject: [PATCH 06/12] fix(foundry-evals): harden FoundryEvals public surface for review Address PR #6267 review comments on the .NET FoundryEvals integration: - Add source-compat overloads accepting `string[] evaluators` for `FoundryEvals` ctor, `EvaluateTracesAsync`, and `EvaluateFoundryTargetAsync` so existing callers passing string arrays keep compiling unchanged. New overloads forward via a private `ToSpecs` helper that wraps each name through the implicit `string -> FoundryEvaluatorSpec` conversion. - Guard against `default(FoundryEvaluatorSpec)` entries (both `BuiltinName` and `GeneratedRef` null) that would NRE the downstream converter. Adds `FoundryEvaluatorSpec.IsValid` / `EnsureValid` plus an internal `EnsureAllSpecsValid` helper, wired into the main ctor and both static evaluation entry points. - Add 6 unit tests covering the new validation surface. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Evaluation/FoundryEvals.cs | 174 ++++++++++++++++++ .../Evaluation/FoundryEvaluatorSpec.cs | 23 +++ .../FoundryEvalsTests.cs | 56 ++++++ 3 files changed, 253 insertions(+) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs index b8c524ba8d..e7871c527e 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs @@ -67,6 +67,8 @@ public FoundryEvals(AIProjectClient projectClient, string model, params FoundryE { ArgumentNullException.ThrowIfNull(projectClient); ArgumentException.ThrowIfNullOrWhiteSpace(model); + ArgumentNullException.ThrowIfNull(evaluators); + EnsureAllSpecsValid(evaluators, nameof(evaluators)); this._evaluationClient = projectClient.GetProjectOpenAIClient().GetEvaluationClient(); this._model = model; @@ -127,6 +129,81 @@ public FoundryEvals( this._timeoutSeconds = timeoutSeconds; } + // ----------------------------------------------------------------------- + // string[] constructor overloads (source-compat with older API that took + // `params string[] evaluators` before FoundryEvaluatorSpec was introduced). + // `params` is intentionally omitted to avoid overload ambiguity with the + // spec-based ctors at zero-args; individual string literals still resolve + // through `params FoundryEvaluatorSpec[]` via implicit conversion. + // ----------------------------------------------------------------------- + + /// + /// Initializes a new instance of the class using built-in evaluator + /// names. Preserves source compatibility for callers that pass a array. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// Built-in evaluator names (for example ). + public FoundryEvals(AIProjectClient projectClient, string model, string[] evaluators) + : this(projectClient, model, ToSpecs(evaluators)) + { + } + + /// + /// Initializes a new instance of the class with a splitter and + /// built-in evaluator names. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// Default conversation splitter for multi-turn conversations. + /// Built-in evaluator names. + public FoundryEvals( + AIProjectClient projectClient, + string model, + IConversationSplitter? splitter, + string[] evaluators) + : this(projectClient, model, splitter, ToSpecs(evaluators)) + { + } + + /// + /// Initializes a new instance of the class with full configuration + /// and built-in evaluator names. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// Default conversation splitter for multi-turn conversations. + /// Seconds between status polls. + /// Maximum seconds to wait for completion. + /// Built-in evaluator names. + public FoundryEvals( + AIProjectClient projectClient, + string model, + IConversationSplitter? splitter, + double pollIntervalSeconds, + double timeoutSeconds, + string[] evaluators) + : this(projectClient, model, splitter, pollIntervalSeconds, timeoutSeconds, ToSpecs(evaluators)) + { + } + + private static FoundryEvaluatorSpec[] ToSpecs(string[]? evaluators) + { + if (evaluators is null || evaluators.Length == 0) + { + return []; + } + + var specs = new FoundryEvaluatorSpec[evaluators.Length]; + for (int i = 0; i < evaluators.Length; i++) + { + specs[i] = evaluators[i] + ?? throw new ArgumentException($"Evaluator name at index {i} is null.", nameof(evaluators)); + } + + return specs; + } + // ----------------------------------------------------------------------- // IAgentEvaluator // ----------------------------------------------------------------------- @@ -274,6 +351,47 @@ public async Task EvaluateAsync( // Static evaluation methods (traces and targets) // ----------------------------------------------------------------------- + /// + /// Source-compat overload of + /// that accepts a array of built-in evaluator names. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// Evaluate specific Responses API response IDs. + /// Evaluate specific OTel trace IDs from App Insights. + /// Filter traces by agent ID (used with ). + /// Hours of trace history to evaluate. + /// Built-in evaluator names. Each is wrapped via . + /// Display name for the evaluation. + /// Seconds between status polls. + /// Maximum seconds to wait for completion. + /// Cancellation token. + /// Evaluation results with status, report URL, and per-item details. + public static Task EvaluateTracesAsync( + AIProjectClient projectClient, + string model, + IEnumerable? responseIds, + IEnumerable? traceIds, + string? agentId, + int lookbackHours, + string[]? evaluators, + string evalName = "Agent Framework Trace Eval", + double pollIntervalSeconds = 5.0, + double timeoutSeconds = 300.0, + CancellationToken cancellationToken = default) + => EvaluateTracesAsync( + projectClient, + model, + responseIds, + traceIds, + agentId, + lookbackHours, + ToSpecs(evaluators) is { Length: > 0 } specs ? specs : null, + evalName, + pollIntervalSeconds, + timeoutSeconds, + cancellationToken); + /// /// Evaluates agent behavior from Responses API response IDs, OTel traces, or agent activity. /// @@ -331,6 +449,7 @@ public static async Task EvaluateTracesAsync( FoundryEvaluatorSpec[] resolvedEvaluators = evaluators is { Length: > 0 } ? evaluators : [Relevance, Coherence, TaskAdherence]; + EnsureAllSpecsValid(resolvedEvaluators, nameof(evaluators)); // Create the evaluation definition with the appropriate data source scenario object dataSourceConfig; @@ -437,6 +556,41 @@ public static async Task EvaluateTracesAsync( }; } + /// + /// Source-compat overload of + /// that accepts a array of built-in evaluator names. + /// + /// The Azure AI Foundry project client. + /// Model deployment name for the LLM judge evaluator. + /// Target configuration (must include a "type" key). + /// Queries for Foundry to send to the target. + /// Built-in evaluator names. Each is wrapped via . + /// Display name for the evaluation. + /// Seconds between status polls. + /// Maximum seconds to wait for completion. + /// Cancellation token. + /// Evaluation results with status, report URL, and per-item details. + public static Task EvaluateFoundryTargetAsync( + AIProjectClient projectClient, + string model, + IDictionary target, + IEnumerable testQueries, + string[]? evaluators, + string evalName = "Agent Framework Target Eval", + double pollIntervalSeconds = 5.0, + double timeoutSeconds = 300.0, + CancellationToken cancellationToken = default) + => EvaluateFoundryTargetAsync( + projectClient, + model, + target, + testQueries, + ToSpecs(evaluators) is { Length: > 0 } specs ? specs : null, + evalName, + pollIntervalSeconds, + timeoutSeconds, + cancellationToken); + /// /// Evaluates a Foundry-registered agent or model deployment. /// @@ -487,6 +641,7 @@ public static async Task EvaluateFoundryTargetAsync( FoundryEvaluatorSpec[] resolvedEvaluators = evaluators is { Length: > 0 } ? evaluators : [Relevance, Coherence, TaskAdherence]; + EnsureAllSpecsValid(resolvedEvaluators, nameof(evaluators)); var createEvalPayload = new WireCreateEvalRequest { @@ -1096,6 +1251,25 @@ internal static FoundryEvaluatorSpec[] FilterToolEvaluators(FoundryEvaluatorSpec + $"Tool evaluators: {string.Join(", ", evaluators.Select(e => e.ToString()))}. Either add tool call content to your EvalItems or remove tool-type evaluators."); } + /// + /// Validates every spec in — defensively guards against + /// default(FoundryEvaluatorSpec) values that would otherwise NRE deep in the + /// dispatch pipeline (e.g. on spec.BuiltinName!). + /// + internal static void EnsureAllSpecsValid(FoundryEvaluatorSpec[] evaluators, string paramName) + { + for (int i = 0; i < evaluators.Length; i++) + { + if (!evaluators[i].IsValid) + { + throw new ArgumentException( + $"Invalid {nameof(FoundryEvaluatorSpec)} at index {i}: must be constructed with either a built-in " + + $"evaluator name or a {nameof(GeneratedEvaluatorRef)}. The default struct value is not a valid spec.", + paramName); + } + } + } + private static bool HasToolEvaluator(FoundryEvaluatorSpec[] evaluators) { foreach (var spec in evaluators) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvaluatorSpec.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvaluatorSpec.cs index 5712c2f87f..0cb9225386 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvaluatorSpec.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvaluatorSpec.cs @@ -63,6 +63,29 @@ public FoundryEvaluatorSpec(GeneratedEvaluatorRef generatedRef) /// Gets whether this spec references a generated rubric evaluator. public bool IsRubric => this.GeneratedRef is not null; + /// Gets whether this spec is valid (i.e. references either a built-in or a rubric). + /// + /// Because is a struct, default(FoundryEvaluatorSpec) + /// is a syntactically-valid but semantically-invalid value (both and + /// are ). Call at + /// API boundaries to fail fast instead of NRE-ing later. + /// + public bool IsValid => this.BuiltinName is not null || this.GeneratedRef is not null; + + /// Validates that this spec references either a built-in evaluator or a rubric. + /// Parameter name used in the thrown . + /// Thrown when neither nor is set. + public void EnsureValid(string? paramName = null) + { + if (!this.IsValid) + { + throw new ArgumentException( + $"Invalid {nameof(FoundryEvaluatorSpec)}: must be constructed with either a built-in evaluator name " + + $"or a {nameof(GeneratedEvaluatorRef)}. The default struct value is not a valid spec.", + paramName); + } + } + /// Implicit conversion from a built-in evaluator name. public static implicit operator FoundryEvaluatorSpec(string builtinName) => new(builtinName); diff --git a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs index 7cca9ce901..6b0c8ebbde 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Foundry.UnitTests/FoundryEvalsTests.cs @@ -195,4 +195,60 @@ public void ParseRubricScores_NonApplicableDimension_KeepsNullScoreWhenMissing() Assert.False(result[0].Applicable); Assert.Null(result[0].Score); } + + // --------------------------------------------------------------- + // FoundryEvaluatorSpec validation tests + // --------------------------------------------------------------- + + [Fact] + public void FoundryEvaluatorSpec_Default_IsNotValid() + { + var spec = default(FoundryEvaluatorSpec); + Assert.False(spec.IsValid); + Assert.Null(spec.BuiltinName); + Assert.Null(spec.GeneratedRef); + } + + [Fact] + public void FoundryEvaluatorSpec_EnsureValid_DefaultThrows() + { + var spec = default(FoundryEvaluatorSpec); + var ex = Assert.Throws(() => spec.EnsureValid("evaluators")); + Assert.Equal("evaluators", ex.ParamName); + } + + [Fact] + public void FoundryEvaluatorSpec_EnsureValid_BuiltinPasses() + { + var spec = (FoundryEvaluatorSpec)"relevance"; + spec.EnsureValid(); // does not throw + } + + [Fact] + public void FoundryEvaluatorSpec_EnsureValid_RubricPasses() + { + var spec = (FoundryEvaluatorSpec)new GeneratedEvaluatorRef("r", "1"); + spec.EnsureValid(); // does not throw + } + + [Fact] + public void EnsureAllSpecsValid_DefaultEntry_ThrowsWithParamName() + { + var specs = new FoundryEvaluatorSpec[] { "relevance", default }; + var ex = Assert.Throws( + () => FoundryEvals.EnsureAllSpecsValid(specs, "evaluators")); + Assert.Equal("evaluators", ex.ParamName); + Assert.Contains("index 1", ex.Message); + } + + [Fact] + public void EnsureAllSpecsValid_AllValid_DoesNotThrow() + { + var specs = new FoundryEvaluatorSpec[] + { + "relevance", + new GeneratedEvaluatorRef("policy", "1"), + }; + FoundryEvals.EnsureAllSpecsValid(specs, "evaluators"); + } } From d23c2d906f173be79957f10f853e87037a654cb6 Mon Sep 17 00:00:00 2001 From: alliscode <25218250+alliscode@users.noreply.github.com> Date: Tue, 2 Jun 2026 08:53:36 -0700 Subject: [PATCH 07/12] fix(sample): set ExitCode=1 when rubric dimension gate trips PR #6267 review comment: the FoundryRubric sample swallowed the AssertDimensionScoreAtLeast failure, so a CI run that included it as a quality gate would still exit 0 even when the rubric regressed. Set `System.Environment.ExitCode = 1` in the catch so CI fails while still letting the rest of the sample's logging complete cleanly. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs index 35c7dd96ab..5cdd21f04e 100644 --- a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs @@ -139,4 +139,5 @@ catch (InvalidOperationException ex) { Console.WriteLine($"[FAIL] {results.ProviderName}: dimension gate tripped: {ex.Message}"); + System.Environment.ExitCode = 1; } From 501af85f01629e81426bd3297ad98d9f687057cb Mon Sep 17 00:00:00 2001 From: alliscode <25218250+alliscode@users.noreply.github.com> Date: Tue, 2 Jun 2026 08:53:47 -0700 Subject: [PATCH 08/12] fix(foundry-evals): search typed Sample directly for rubric scores PR #6267 review comment: `_extract_rubric_scores` only searched the `properties` dict when the sample exposed one. When the Azure AI Projects typed SDK returns a Sample object that puts `dimension_scores` / `rubric_scores` directly on the instance (no `properties` wrapper), we missed them and surfaced no per-dimension scores. Add an `else: containers.append(sample)` branch so non-dict typed samples are also inspected for the score keys. Covered by two new tests: one with `dimension_scores` directly on a typed Sample without a `properties` wrapper, and one with the legacy `rubric_scores` key in the same shape. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../agent_framework_foundry/_foundry_evals.py | 2 + .../foundry/tests/test_foundry_evals.py | 38 +++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py index 8059c2ce99..b4015f1e23 100644 --- a/python/packages/foundry/agent_framework_foundry/_foundry_evals.py +++ b/python/packages/foundry/agent_framework_foundry/_foundry_evals.py @@ -541,6 +541,8 @@ def _extract_rubric_scores(sample: Any) -> list[RubricScore] | None: if props_dict is not None and props_dict is not properties: containers.append(props_dict) containers.append(sample_any) + else: + containers.append(sample) for container in containers: for key in _RUBRIC_DIMENSION_KEYS: diff --git a/python/packages/foundry/tests/test_foundry_evals.py b/python/packages/foundry/tests/test_foundry_evals.py index 8734650aaf..6e24a10ec7 100644 --- a/python/packages/foundry/tests/test_foundry_evals.py +++ b/python/packages/foundry/tests/test_foundry_evals.py @@ -2619,6 +2619,44 @@ def test_dimension_scores_via_attribute(self) -> None: assert result[0].id == "policy_enforcement" assert result[0].score == 1 + def test_dimension_scores_directly_on_typed_sample_no_properties_wrapper(self) -> None: + """Typed SDK sample with ``dimension_scores`` directly on the instance (no ``properties``).""" + + rs = MagicMock() + rs.id = "intent_recognition" + rs.score = 4 + rs.applicable = True + rs.weight = 2 + rs.reason = "ok" + + # spec= restricts available attributes — no `properties`, just `dimension_scores`. + sample = MagicMock(spec=["dimension_scores"]) + sample.dimension_scores = [rs] + + result = _extract_rubric_scores(sample) + assert result is not None + assert result[0].id == "intent_recognition" + assert result[0].score == 4 + assert result[0].weight == 2 + + def test_rubric_scores_directly_on_typed_sample_legacy_key(self) -> None: + """Same fallback works for the legacy ``rubric_scores`` key.""" + + rs = MagicMock() + rs.id = "policy" + rs.score = 2 + rs.applicable = True + rs.weight = 1 + rs.reason = "partial" + + sample = MagicMock(spec=["rubric_scores"]) + sample.rubric_scores = [rs] + + result = _extract_rubric_scores(sample) + assert result is not None + assert result[0].id == "policy" + assert result[0].score == 2 + # --------------------------------------------------------------------------- # _poll_eval_run — timeout / failed / canceled paths From db25a7130375e986b0cef9f19fa61e14d08cd284 Mon Sep 17 00:00:00 2001 From: alliscode <25218250+alliscode@users.noreply.github.com> Date: Tue, 2 Jun 2026 08:53:59 -0700 Subject: [PATCH 09/12] test(evals): cover assert_score_at_least and assert_no_failed_items PR #6267 review comments: both assertion helpers shipped without unit tests. Add `TestAssertScoreAtLeast` (above threshold, below w/ offenders, evaluator filter, sub_results recursion) and `TestAssertNoFailedItems` (all passing, failed/errored statuses, sub_results recursion) with a shared `_score_results` fixture builder. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../core/tests/core/test_local_eval.py | 111 ++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/python/packages/core/tests/core/test_local_eval.py b/python/packages/core/tests/core/test_local_eval.py index e60fb35d51..595e1c8884 100644 --- a/python/packages/core/tests/core/test_local_eval.py +++ b/python/packages/core/tests/core/test_local_eval.py @@ -1113,3 +1113,114 @@ def test_evaluator_filter_isolates_offenders(self) -> None: ) # The low-scoring "other" evaluator is filtered out; "policy" passes. results.assert_dimension_score_at_least("clarity", 3, evaluator="policy") + + +def _score_results( + *scores_per_item: list[EvalScoreResult], + sub_results: dict[str, EvalResults] | None = None, +) -> EvalResults: + """Build an EvalResults shaped for score / status assertion tests.""" + items = [ + EvalItemResult(item_id=f"item-{i}", status="pass", scores=scores) for i, scores in enumerate(scores_per_item) + ] + return EvalResults( + provider="test", + eval_id="ev1", + run_id="run1", + result_counts={"passed": len(items), "failed": 0, "errored": 0, "total": len(items)}, + items=items, + sub_results=sub_results or {}, + ) + + +class TestAssertScoreAtLeast: + """Tests for EvalResults.assert_score_at_least (mirrors .NET coverage).""" + + def test_all_above_threshold_passes(self) -> None: + results = _score_results( + [EvalScoreResult(name="relevance", score=0.9)], + [EvalScoreResult(name="relevance", score=0.85)], + ) + # Should not raise. + results.assert_score_at_least(0.8) + + def test_below_threshold_raises_with_offenders(self) -> None: + results = _score_results( + [EvalScoreResult(name="relevance", score=0.4)], + [EvalScoreResult(name="relevance", score=0.9)], + ) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_score_at_least(0.5) + msg = str(exc.value) + assert "item-0" in msg + assert "relevance" in msg + assert "0.400" in msg + + def test_evaluator_filter_isolates_offenders(self) -> None: + results = _score_results( + [ + EvalScoreResult(name="other", score=0.1), + EvalScoreResult(name="relevance", score=0.95), + ], + ) + # The low-scoring "other" evaluator is filtered out; "relevance" passes. + results.assert_score_at_least(0.8, evaluator="relevance") + + def test_recursion_into_sub_results(self) -> None: + sub = _score_results([EvalScoreResult(name="relevance", score=0.2)]) + parent = _score_results( + [EvalScoreResult(name="relevance", score=0.9)], + sub_results={"sub_executor": sub}, + ) + with pytest.raises(EvalNotPassedError) as exc: + parent.assert_score_at_least(0.5) + # Offender from sub-result is surfaced. + assert "0.200" in str(exc.value) + + +class TestAssertNoFailedItems: + """Tests for EvalResults.assert_no_failed_items (mirrors .NET coverage).""" + + def test_all_passing_does_not_raise(self) -> None: + results = _score_results( + [EvalScoreResult(name="relevance", score=0.9)], + [EvalScoreResult(name="relevance", score=0.85)], + ) + # Should not raise. + results.assert_no_failed_items() + + def test_failed_and_errored_items_raise_with_statuses(self) -> None: + items = [ + EvalItemResult(item_id="ok", status="pass", scores=[]), + EvalItemResult(item_id="bad", status="fail", scores=[]), + EvalItemResult(item_id="boom", status="error", scores=[], error_code="timeout"), + ] + results = EvalResults( + provider="test", + eval_id="ev1", + run_id="run1", + result_counts={"passed": 1, "failed": 1, "errored": 1, "total": 3}, + items=items, + ) + with pytest.raises(EvalNotPassedError) as exc: + results.assert_no_failed_items() + msg = str(exc.value) + assert "bad:fail" in msg + assert "boom:error" in msg + + def test_recursion_into_sub_results(self) -> None: + sub_items = [EvalItemResult(item_id="sub-bad", status="fail", scores=[])] + sub = EvalResults( + provider="test", + eval_id="ev2", + run_id="run2", + result_counts={"passed": 0, "failed": 1, "errored": 0, "total": 1}, + items=sub_items, + ) + parent = _score_results( + [EvalScoreResult(name="relevance", score=0.9)], + sub_results={"sub_executor": sub}, + ) + with pytest.raises(EvalNotPassedError) as exc: + parent.assert_no_failed_items() + assert "sub-bad:fail" in str(exc.value) From 984967fe9768353ab740450b5c3f548ecf5fb83b Mon Sep 17 00:00:00 2001 From: Ben Thomas <25218250+alliscode@users.noreply.github.com> Date: Tue, 2 Jun 2026 10:57:48 -0700 Subject: [PATCH 10/12] docs(samples): remove dead rubric-evaluator doc link from FoundryRubric sample The Azure AI Foundry rubric evaluator concept doc page has not yet been published, so the link in the sample README and Program.cs comment 404s. Drop the references until the upstream doc is live. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Evaluation/Evaluation_FoundryRubric/Program.cs | 6 ++---- .../Evaluation/Evaluation_FoundryRubric/README.md | 2 -- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs index 5cdd21f04e..f8e250f7f5 100644 --- a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs @@ -8,8 +8,6 @@ // authored in the Foundry portal (or via the dedicated SDK / REST surface) and referenced // here by name and version. // -// See: https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-evaluators/rubric-evaluators -// // Prerequisites: // - An Azure AI Foundry project with a deployed model. // - A registered Foundry agent in that project (the rubric was created against this agent). @@ -27,9 +25,9 @@ using Microsoft.Agents.AI.Foundry; using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals; -string projectEndpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") +string projectEndpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT_3") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); -string model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") +string model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL_3") ?? throw new InvalidOperationException("FOUNDRY_MODEL is not set."); string agentName = Environment.GetEnvironmentVariable("FOUNDRY_AGENT_NAME") ?? throw new InvalidOperationException("FOUNDRY_AGENT_NAME is not set."); diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/README.md b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/README.md index eb5f73123e..a05cfca5e6 100644 --- a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/README.md +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/README.md @@ -6,8 +6,6 @@ scoring dimensions you define for your domain; agent-framework references them b version, mixes them with built-in evaluators, and exposes per-dimension scores you can gate CI on. -See the [rubric evaluator documentation](https://learn.microsoft.com/azure/ai-foundry/concepts/evaluation-evaluators/rubric-evaluators). - ## What this sample demonstrates - Connecting to a pre-existing Foundry agent (`AgentAdministrationClient.GetAgentAsync`). From 25b40a3117ceb2e708df10a36f4be2a32c99579d Mon Sep 17 00:00:00 2001 From: Ben Thomas Date: Thu, 4 Jun 2026 08:09:21 -0700 Subject: [PATCH 11/12] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../Evaluation/Evaluation_FoundryRubric/Program.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs index f8e250f7f5..d321234019 100644 --- a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs @@ -25,9 +25,9 @@ using Microsoft.Agents.AI.Foundry; using FoundryEvals = Microsoft.Agents.AI.Foundry.FoundryEvals; -string projectEndpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT_3") +string projectEndpoint = Environment.GetEnvironmentVariable("FOUNDRY_PROJECT_ENDPOINT") ?? throw new InvalidOperationException("FOUNDRY_PROJECT_ENDPOINT is not set."); -string model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL_3") +string model = Environment.GetEnvironmentVariable("FOUNDRY_MODEL") ?? throw new InvalidOperationException("FOUNDRY_MODEL is not set."); string agentName = Environment.GetEnvironmentVariable("FOUNDRY_AGENT_NAME") ?? throw new InvalidOperationException("FOUNDRY_AGENT_NAME is not set."); From 3c4e20d9c99b56003b7794f5662d5ea8a29c4ea9 Mon Sep 17 00:00:00 2001 From: Ben Thomas <25218250+alliscode@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:01:36 -0700 Subject: [PATCH 12/12] Address PR 6267 review nits Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../Evaluation_FoundryRubric/Program.cs | 2 +- .../Evaluation/FoundryEvals.cs | 4 +- .../Evaluation/AgentEvaluationResults.cs | 87 ++++++++++++++++++- .../EvaluationTests.cs | 23 +++++ 4 files changed, 112 insertions(+), 4 deletions(-) diff --git a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs index d321234019..5e1aef1736 100644 --- a/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs +++ b/dotnet/samples/05-end-to-end/Evaluation/Evaluation_FoundryRubric/Program.cs @@ -131,7 +131,7 @@ try { - results.AssertDimensionScoreAtLeast("general_quality", minScore: 3.0, evaluator: rubricName); + results.AssertDimensionScoreAtLeast("general_quality", minScore: 3.0, evaluator: rubricName, requireApplicable: true); Console.WriteLine($"[PASS] {results.ProviderName}: general_quality >= 3 on every item"); } catch (InvalidOperationException ex) diff --git a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs index e7871c527e..d1542fc4df 100644 --- a/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs +++ b/dotnet/src/Microsoft.Agents.AI.Foundry/Evaluation/FoundryEvals.cs @@ -374,7 +374,7 @@ public static Task EvaluateTracesAsync( IEnumerable? traceIds, string? agentId, int lookbackHours, - string[]? evaluators, + string[]? evaluators = null, string evalName = "Agent Framework Trace Eval", double pollIntervalSeconds = 5.0, double timeoutSeconds = 300.0, @@ -575,7 +575,7 @@ public static Task EvaluateFoundryTargetAsync( string model, IDictionary target, IEnumerable testQueries, - string[]? evaluators, + string[]? evaluators = null, string evalName = "Agent Framework Target Eval", double pollIntervalSeconds = 5.0, double timeoutSeconds = 300.0, diff --git a/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs index 10fa02b986..552479ad69 100644 --- a/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs +++ b/dotnet/src/Microsoft.Agents.AI/Evaluation/AgentEvaluationResults.cs @@ -166,7 +166,9 @@ public void AssertScoreAtLeast(double minScore, string? evaluator = null, string /// Walks across /// (and any ) looking for the named dimension. Non-applicable /// dimensions are skipped by default; pass = - /// to fail when no applicable score is produced for an item. + /// to fail when no applicable score is produced for an item. If dimension data exists + /// but the requested is never present, the assertion fails + /// to surface likely typos or evaluator mismatches. /// /// Dimension id — matches the rubric definition. /// Minimum acceptable dimension score (inclusive). @@ -193,6 +195,14 @@ public void AssertDimensionScoreAtLeast( CollectDimensionOffenders(this, dimensionId, minScore, evaluator, requireApplicable, offenders, missing); var problems = new List(); + bool hasAnyDimensionData = HasAnyDimensionData(this, evaluator); + if (hasAnyDimensionData && !HasDimension(this, dimensionId, evaluator)) + { + problems.Add( + $"Dimension '{dimensionId}' was not found in results" + + (evaluator is not null ? $" for evaluator '{evaluator}'." : ".")); + } + if (offenders.Count > 0) { problems.Add(FormatOffenders( @@ -354,6 +364,81 @@ private static void CollectFailedItems(AgentEvaluationResults results, List 0 }) + { + return true; + } + } + } + } + + if (results.SubResults is not null) + { + foreach (var sub in results.SubResults.Values) + { + if (HasAnyDimensionData(sub, evaluator)) + { + return true; + } + } + } + + return false; + } + + private static bool HasDimension(AgentEvaluationResults results, string dimensionId, string? evaluator) + { + if (results.DetailedItems is not null) + { + foreach (var item in results.DetailedItems) + { + foreach (var score in item.Scores) + { + if (evaluator is not null && score.Name != evaluator) + { + continue; + } + + if (score.Dimensions is null) + { + continue; + } + + if (score.Dimensions.Any(rs => rs.Id == dimensionId)) + { + return true; + } + } + } + } + + if (results.SubResults is not null) + { + foreach (var sub in results.SubResults.Values) + { + if (HasDimension(sub, dimensionId, evaluator)) + { + return true; + } + } + } + + return false; + } + private static string FormatOffenders(string prefix, List offenders) { const int MaxShown = 5; diff --git a/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs b/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs index 47def4e80d..ff84b7d685 100644 --- a/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.UnitTests/EvaluationTests.cs @@ -814,6 +814,29 @@ public void AssertDimensionScoreAtLeast_RequireApplicable_ThrowsWhenMissing() Assert.Contains("item-1", ex.Message); } + [Fact] + public void AssertDimensionScoreAtLeast_UnknownDimension_ThrowsWhenDimensionDataExists() + { + // Arrange + var detailed = new EvalItemResult("item-1", "pass", new[] + { + new EvalScoreResult("policy", 1.0, Passed: true) + { + Dimensions = + [ + new RubricScore("clarity", Score: 4, Applicable: true, Weight: 1, Reason: "ok"), + ], + }, + }); + var results = BuildResultsWithDetailed(detailed); + + // Act & Assert + var ex = Assert.Throws( + () => results.AssertDimensionScoreAtLeast("typo_dimension", 3.0)); + Assert.Contains("typo_dimension", ex.Message); + Assert.Contains("not found", ex.Message); + } + [Fact] public void AssertNoFailedItems_AllPassing_DoesNotThrow() {