From 94fbe0e1f46550ad02f8b02730e732ce26ffa119 Mon Sep 17 00:00:00 2001
From: Saurabh Badenkal <sbadenkal@microsoft.com>
Date: Tue, 2 Jun 2026 08:52:54 -0700
Subject: [PATCH 1/2] validation: build LER from PR #416 branch to verify
 DeterministicAssertionEvaluator fix end-to-end

DO NOT MERGE. This is a throwaway validation PR. Points the pipeline's LocalEvalRunner repo ref to https://microsoft.ghe.com/bic/LocalEvalRunner/pull/416 (users/sbadenkal/service-only-deterministic-assertion), checks out LER source explicitly, and enables useBuildFromSource so the pipeline runs LER built from the PR-416 branch instead of the published NuGet. Also copies the new-format dv_data.biceval.json from PR #70 so the run actually exercises the DeterministicAssertionEvaluator code path. PR #70 is untouched. Expected outcome: pipeline succeeds where build 20283404 crashed at evaluator load.
---
 .azdo/DataversePluginEvals_PR.yml | 13 +++++++++++--
 evals/tests/dv_data.biceval.json  |  8 ++++++++
 2 files changed, 19 insertions(+), 2 deletions(-)

diff --git a/.azdo/DataversePluginEvals_PR.yml b/.azdo/DataversePluginEvals_PR.yml
index 01fb4bc..2b8d5da 100644
--- a/.azdo/DataversePluginEvals_PR.yml
+++ b/.azdo/DataversePluginEvals_PR.yml
@@ -56,7 +56,10 @@ resources:
     type: github
     endpoint: bic-proxima-shared-stamp
     name: bic/LocalEvalRunner
-    ref: refs/heads/main
+    # Temporary: pinned to LER PR #416 branch to validate the
+    # DeterministicAssertionEvaluator service-only routing fix end-to-end.
+    # Revert to refs/heads/main before merge.
+    ref: refs/heads/users/sbadenkal/service-only-deterministic-assertion
   - repository: BicEvalsService
     type: github
     endpoint: bic-proxima-shared-stamp
@@ -67,15 +70,21 @@ steps:
 - checkout: self
   displayName: 'Checkout Dataverse-skills'
 
+- checkout: LocalEvalRunner
+  displayName: 'Checkout LocalEvalRunner source (for build-from-source)'
+  fetchDepth: 1
+
 - checkout: BicEvalsService
   displayName: 'Checkout evaluator prompts'
   fetchDepth: 1
 
-# Setup: .NET, NuGet auth, LocalEvalRunner install, Copilot CLI, auth token
+# Setup: .NET, NuGet auth, LocalEvalRunner BUILD FROM SOURCE, Copilot CLI, auth token
 - template: .pipelines/templates/steps/BicEval-SetupTemplate.yml@LocalEvalRunner
   parameters:
     cliAgent: 'copilot'
     cliAgentTokenVariable: 'COPILOT_GH_PAT'
+    useBuildFromSource: true
+    sourceRepoPath: '$(Build.SourcesDirectory)/LocalEvalRunner'
 
 # Configure: merge replacements, prepare agent config
 - template: .pipelines/templates/steps/BicEval-ConfigureTemplate.yml@LocalEvalRunner
diff --git a/evals/tests/dv_data.biceval.json b/evals/tests/dv_data.biceval.json
index 00d6b5b..82a8853 100644
--- a/evals/tests/dv_data.biceval.json
+++ b/evals/tests/dv_data.biceval.json
@@ -3,6 +3,14 @@
   "scenarioName": "Data",
   "description": "Verifies SDK patterns for record creation in Dataverse tables.",
   "enabled_evaluators": [
+    {
+      "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator",
+      "passing_score": 3,
+      "priority": 1,
+      "settings": {
+        "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED"
+      }
+    },
     {
       "name": "CortexConfigurations:Common/Skills/correctness.prompty",
       "passing_score": 3,

From 4cfeccb9f2235a232dabf31e84f1a82ac7ad6a89 Mon Sep 17 00:00:00 2001
From: Saurabh Badenkal <sbadenkal@microsoft.com>
Date: Tue, 2 Jun 2026 10:23:07 -0700
Subject: [PATCH 2/2] validation: add LMChecklist evaluator so natural-language
 assertions actually get graded

Per Lekina's feedback on PR 393 thread: DeterministicAssertionEvaluator only grades verb-prefixed assertions (CONTAINS/NOT_CONTAINS/SKILL_LOADED); correctness.prompty scores against expected_response without seeing individual assertions; therefore natural-language assertions were silently unscored. Adding LMChecklist.prompty (score 0/1 per-assertion, threshold 1) closes that gap. passing_score=1 matches the prompty's documented range.
---
 evals/tests/dv_data.biceval.json | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/evals/tests/dv_data.biceval.json b/evals/tests/dv_data.biceval.json
index 82a8853..741d628 100644
--- a/evals/tests/dv_data.biceval.json
+++ b/evals/tests/dv_data.biceval.json
@@ -15,6 +15,11 @@
       "name": "CortexConfigurations:Common/Skills/correctness.prompty",
       "passing_score": 3,
       "priority": 1
+    },
+    {
+      "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty",
+      "passing_score": 1,
+      "priority": 1
     }
   ],
   "tests": [