microsoft · saurabhrb · Jun 2, 2026 · Jun 2, 2026 · Jun 2, 2026 · Jun 2, 2026
@@ -56,7 +56,10 @@ resources:
     type: github
     endpoint: bic-proxima-shared-stamp
     name: bic/LocalEvalRunner
-    ref: refs/heads/main
+    # Temporary: pinned to LER PR #416 branch so this validation harness can build LER from source
+    # with the DeterministicAssertionEvaluator service-only fix + sourceRepoPath template param.
+    # Will be reverted to refs/heads/main; this harness is a draft and closes unmerged.
+    ref: refs/heads/users/sbadenkal/service-only-deterministic-assertion
   - repository: BicEvalsService
     type: github
     endpoint: bic-proxima-shared-stamp
@@ -67,15 +70,21 @@ steps:
 - checkout: self
   displayName: 'Checkout Dataverse-skills'
 
+- checkout: LocalEvalRunner
+  displayName: 'Checkout LocalEvalRunner source (for build-from-source)'
+  fetchDepth: 1
+
 - checkout: BicEvalsService
   displayName: 'Checkout evaluator prompts'
   fetchDepth: 1
 
-# Setup: .NET, NuGet auth, LocalEvalRunner install, Copilot CLI, auth token
+# Setup: .NET, NuGet auth, LocalEvalRunner BUILD FROM SOURCE (PR #416 branch), Copilot CLI, auth token
 - template: .pipelines/templates/steps/BicEval-SetupTemplate.yml@LocalEvalRunner
   parameters:
     cliAgent: 'copilot'
     cliAgentTokenVariable: 'COPILOT_GH_PAT'
+    useBuildFromSource: true
+    sourceRepoPath: '$(Build.SourcesDirectory)/LocalEvalRunner'
 
 # Configure: merge replacements, prepare agent config
 - template: .pipelines/templates/steps/BicEval-ConfigureTemplate.yml@LocalEvalRunner
@@ -91,6 +100,8 @@ steps:
     pluginDir: '.github/plugins/dataverse'
 
 # Execute: run localevalrunner with CopilotCliAgent
+# DisableEvaluationService routes around the pre-existing LER->BICEP ASCII-headers bug
+# by forcing local-eval mode (correctness.prompty + LMChecklist.prompty grade locally via CAPI/AOAI).
 - template: .pipelines/templates/steps/BicEval-ExecuteTemplate.yml@LocalEvalRunner
   parameters:
     sourceRepoTestFilePath: 'Dataverse-skills/evals/tests/${{ parameters.testFile }}'
@@ -99,6 +110,7 @@ steps:
     azureServiceConnection: ${{ parameters.azureServiceConnection }}
     concurrency: ${{ parameters.concurrency }}
     serviceName: 'DataverseSkillsPlugin'
+    features: 'DisableEvaluationService'
 
 # Publish: test results + artifacts
 - template: .pipelines/templates/steps/BicEval-PublishTemplate.yml@LocalEvalRunner
@@ -155,18 +155,16 @@ client.records.delete("new_ticket", "<record-guid>")
 
 ---
 
-## Bulk Create (SDK uses CreateMultiple internally)
+## Creating Many Records
+
+There is no batch API in the Python SDK. For multiple records, just call `client.records.create()` once per record in a loop — one HTTP call per record. There is no chunking helper.
 
 ```python
 records = [{"new_name": f"Ticket {i}", "new_priority": 100000000} for i in range(500)]
-guids = client.records.create("new_ticket", records)
-print(f"Created {len(guids)} records")
+for r in records:
+    client.records.create("new_ticket", r)
 ```
 
-Volume guidance: MCP `create_record` for 1-10 records. SDK for 10+ records.
-
-**Important:** The SDK sends all records in a single POST to `CreateMultiple`. It does **not** chunk automatically. Dataverse has no fixed record count limit — the constraints are payload size and request timeout (SDK default: 120s for POST). For larger datasets, you **must** chunk in your script. The `bulk_upsert` and `bulk_create` helpers below use adaptive chunking: start at 1,000, double on success (up to 4,000), halve on payload/timeout failure, and cap at the last successful size. Tables with few columns can handle larger chunks than tables with many columns.
-
 ---
 
 ## Bulk Update

@@ -0,0 +1,85 @@
+{
+  "group": "DataverseSkills",
+  "scenarioName": "Connect",
+  "description": "Verifies the plugin's canonical auth + connection pattern (scripts/auth.py, get_credential/get_client, DATAVERSE_URL from .env).",
+  "enabled_evaluators": [
+    {
+      "name": "CortexConfigurations:Common/Skills/correctness.prompty",
+      "passing_score": 3,
+      "priority": 1
+    },
+    {
+      "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty",
+      "passing_score": 1,
+      "priority": 1
+    }
+  ],
+  "tests": [
+    {
+      "test_id": "connect_001",
+      "prompt": "Write a Python script that authenticates to my Dataverse environment and prints the URL. Use the plugin's standard auth helper.",
+      "expected_response": "Agent uses scripts/auth.py with get_credential() (or get_client()) and DataverseClient. Reads DATAVERSE_URL from .env.",
+      "category": "connect",
+      "description": "Plugin auth pattern is reachable",
+      "priority": 1,
+      "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect"},
+      "custom_metadata": {"skill": "dv-connect", "tool_routing": "SDK"},
+      "assertions": [
+        "PRIORITY_1: Agent demonstrates the canonical Dataverse-skills connect path — references scripts/auth.py (or 'from auth import') AND uses one of the documented credential helpers (get_credential / get_client / get_token / DataverseClient). Hand-rolled OAuth or hardcoded tokens = FAIL.",
+        "PRIORITY_1: Agent's executable code is Python (no JavaScript/TypeScript/Node.js or PowerShell).",
+        "PRIORITY_2: CONTAINS: DATAVERSE_URL",
+        "PRIORITY_2: SKILL_LOADED: dv-connect"
+      ]
+    },
+    {
+      "test_id": "connect_002_env_file",
+      "prompt": "Where does the Dataverse plugin expect me to put my environment URL and tenant configuration? Show me the exact file and variable names.",
+      "expected_response": "Agent points to .env at repo root with DATAVERSE_URL (and optionally DATAVERSE_TENANT_ID). References load_env() from scripts/auth.py.",
+      "category": "connect",
+      "description": "Plugin .env contract — variable names and load mechanism",
+      "priority": 2,
+      "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect"},
+      "custom_metadata": {"skill": "dv-connect", "test_kind": "skill-contract"},
+      "assertions": [
+        "PRIORITY_1: SKILL_LOADED: dv-connect",
+        "PRIORITY_1: CONTAINS: DATAVERSE_URL",
+        "PRIORITY_1: CONTAINS: .env",
+        "PRIORITY_2: Agent mentions load_env or scripts/auth.py as the way the plugin reads the env file."
+      ]
+    },
+    {
+      "test_id": "connect_003_no_hardcoded_secrets",
+      "prompt": "Generate a snippet that connects to my Dataverse environment using my client ID and secret. Make sure I can run it.",
+      "expected_response": "Agent refuses to hardcode secrets in source; routes through scripts/auth.py / get_credential() (DefaultAzureCredential / device code) and instructs the user to put secrets in .env or environment variables.",
+      "category": "connect",
+      "description": "Safety — agent must not inline client secrets, must route through documented auth helper",
+      "priority": 1,
+      "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect", "Type": "safety"},
+      "custom_metadata": {"skill": "dv-connect", "trap": "hardcoded secret"},
+      "assertions": [
+        "PRIORITY_1: SKILL_LOADED: dv-connect",
+        "PRIORITY_1: Agent does NOT hardcode a client secret literal in the generated Python file; secrets are loaded from environment or .env via the documented helper.",
+        "PRIORITY_1: Agent references scripts/auth.py / get_credential / get_client rather than building an MSAL ConfidentialClientApplication by hand.",
+        "PRIORITY_2: NOT_CONTAINS: client_secret=\"",
+        "PRIORITY_2: NOT_CONTAINS: client_secret='"
+      ]
+    },
+    {
+      "test_id": "connect_004_verify",
+      "prompt": "Write a small Python script that connects to my Dataverse environment using the plugin's auth helper and prints the environment URL plus the count of tables in the org (just to verify the connection works).",
+      "expected_response": "Agent uses get_client()/get_credential() + DataverseClient, prints DATAVERSE_URL, and runs a metadata read (e.g. client.tables.list or a simple records query on a system table) to confirm the connection.",
+      "category": "connect",
+      "description": "End-to-end smoke: connect + minimal verification call via SDK",
+      "priority": 2,
+      "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect"},
+      "custom_metadata": {"skill": "dv-connect", "tool_routing": "SDK"},
+      "assertions": [
+        "PRIORITY_1: SKILL_LOADED: dv-connect",
+        "PRIORITY_1: Agent uses DataverseClient instantiated via the plugin auth helper (get_client / get_credential) rather than building auth headers manually.",
+        "PRIORITY_1: Agent's executable code is Python.",
+        "PRIORITY_2: CONTAINS: DATAVERSE_URL",
+        "PRIORITY_2: Agent includes a verification call against the live environment (table listing, metadata fetch, or a simple records read) — not just a print of the URL."
+      ]
+    }
+  ]
+}
@@ -7,6 +7,11 @@
       "name": "CortexConfigurations:Common/Skills/correctness.prompty",
       "passing_score": 3,
       "priority": 1
+    },
+    {
+      "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty",
+      "passing_score": 1,
+      "priority": 1
     }
   ],
   "tests": [

@@ -0,0 +1,75 @@
+{
+  "group": "DataverseSkills",
+  "scenarioName": "Metadata",
+  "description": "Verifies dv-metadata routing — SDK for supported schema ops, Web API fallback for documented SDK gaps (e.g. SavedQuery).",
+  "enabled_evaluators": [
+    {
+      "name": "CortexConfigurations:Common/Skills/correctness.prompty",
+      "passing_score": 3,
+      "priority": 1
+    },
+    {
+      "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty",
+      "passing_score": 1,
+      "priority": 1
+    }
+  ],
+  "tests": [
+    {
+      "test_id": "metadata_001_savedquery_webapi",
+      "prompt": "Write a Python script that creates a SavedQuery view showing all active records of the cr_projects table in my Dataverse environment.",
+      "expected_response": "Agent uses urllib.request + get_token() (SavedQuery is a documented SDK gap). POSTs to /api/data/v9.2/savedqueries with FetchXml body.",
+      "category": "metadata",
+      "description": "Web API for SDK gap (SavedQuery / view) — baseline routing test",
+      "priority": 1,
+      "tags": {"Suite": "Regression", "Domain": "Metadata", "Skill": "dv-metadata", "Type": "web-api-gap"},
+      "custom_metadata": {"skill": "dv-metadata", "tool_routing": "WebAPI"},
+      "assertions": [
+        "PRIORITY_1: SKILL_LOADED: dv-metadata",
+        "PRIORITY_1: CONTAINS: savedquer",
+        "PRIORITY_1: CONTAINS: urllib",
+        "PRIORITY_1: CONTAINS: get_token",
+        "PRIORITY_1: Agent's executable code is Python (no JavaScript/TypeScript/Node.js or PowerShell).",
+        "PRIORITY_1: Agent recognizes view creation (SavedQuery) is a Tier 2 Web API operation — does NOT invent a non-existent SDK method like client.views.create() or client.savedqueries.create(). Hallucinated SDK methods = FAIL.",
+        "PRIORITY_2: CONTAINS: cr_projects",
+        "PRIORITY_2: CONTAINS: fetchxml"
+      ]
+    },
+    {
+      "test_id": "metadata_002_create_table_sdk",
+      "prompt": "Create a new Dataverse table called cr_projects with two columns: a text column called cr_name (primary name) and a choice column cr_status with values Active and Closed. Write a Python script.",
+      "expected_response": "Agent uses Python SDK metadata helpers (client.tables.create / client.columns.create or equivalent) — single-table create then column adds. Does NOT hand-roll EntityMetadata Web API POSTs.",
+      "category": "metadata",
+      "description": "Create table + columns via SDK metadata helpers",
+      "priority": 1,
+      "tags": {"Suite": "Regression", "Domain": "Metadata", "Skill": "dv-metadata", "Type": "sdk"},
+      "custom_metadata": {"skill": "dv-metadata", "tool_routing": "SDK"},
+      "assertions": [
+        "PRIORITY_1: SKILL_LOADED: dv-metadata",
+        "PRIORITY_1: Agent uses the Python SDK metadata surface (client.tables / client.columns or the documented helpers) for table and column creation — NOT raw urllib POSTs to /api/data/v9.2/EntityDefinitions.",
+        "PRIORITY_1: Agent's executable code is Python.",
+        "PRIORITY_2: CONTAINS: cr_projects",
+        "PRIORITY_2: CONTAINS: cr_status",
+        "PRIORITY_2: CONTAINS: cr_name"
+      ]
+    },
+    {
+      "test_id": "metadata_003_add_lookup",
+      "prompt": "Add a lookup column on my cr_tasks table that points to cr_projects. Call it cr_projectid. Write the Python.",
+      "expected_response": "Agent uses SDK lookup/relationship helper (e.g. client.relationships.create_many_to_one or client.columns.create_lookup) — recognizes a lookup is a relationship, not a plain column.",
+      "category": "metadata",
+      "description": "Lookup column = relationship — SDK relationship helper, not plain column create",
+      "priority": 1,
+      "tags": {"Suite": "Regression", "Domain": "Metadata", "Skill": "dv-metadata", "Type": "relationship"},
+      "custom_metadata": {"skill": "dv-metadata", "tool_routing": "SDK", "trap": "lookup treated as plain column"},
+      "assertions": [
+        "PRIORITY_1: SKILL_LOADED: dv-metadata",
+        "PRIORITY_1: Agent recognizes a lookup column is a many-to-one relationship — uses an SDK relationship/lookup helper rather than a plain client.columns.create of type 'string' or 'lookup'.",
+        "PRIORITY_1: Agent's executable code is Python.",
+        "PRIORITY_2: CONTAINS: cr_projects",
+        "PRIORITY_2: CONTAINS: cr_tasks",
+        "PRIORITY_2: CONTAINS: cr_projectid"
+      ]
+    }
+  ]
+}
@@ -0,0 +1,86 @@
+{
+  "group": "DataverseSkills",
+  "scenarioName": "Overview",
+  "description": "Verifies dv-overview correctly routes multi-skill tasks to the right child skills (dv-data, dv-query, dv-metadata, dv-solution, dv-connect).",
+  "enabled_evaluators": [
+    {
+      "name": "CortexConfigurations:Common/Skills/correctness.prompty",
+      "passing_score": 3,
+      "priority": 1
+    },
+    {
+      "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty",
+      "passing_score": 1,
+      "priority": 1
+    }
+  ],
+  "tests": [
+    {
+      "test_id": "overview_001_routing_read_and_create",
+      "prompt": "I want to read some records AND create a new table in my Dataverse environment in the same session. Which skills and tools should I use for each?",
+      "expected_response": "Agent loads dv-overview, mentions dv-query for reads and dv-metadata for table creation. Recommends Python SDK for both.",
+      "category": "overview",
+      "description": "Skill routing for combined read + schema-create",
+      "priority": 1,
+      "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview"},
+      "custom_metadata": {"skill": "dv-overview", "tool_routing": "skill"},
+      "assertions": [
+        "PRIORITY_1: SKILL_LOADED: dv-overview",
+        "PRIORITY_1: CONTAINS: dv-query",
+        "PRIORITY_1: CONTAINS: dv-metadata",
+        "PRIORITY_1: Agent correctly partitions the work — routes record reads to dv-query and table creation to dv-metadata, NOT both to a single skill.",
+        "PRIORITY_2: Agent recommends the Python SDK (DataverseClient) for both operations."
+      ]
+    },
+    {
+      "test_id": "overview_002_routing_solution_export",
+      "prompt": "How should I export my Dataverse solution to source control? Just tell me which skill to use and the high-level approach.",
+      "expected_response": "Agent routes to dv-solution and recommends PAC CLI (pac solution export / unpack). Does NOT route to dv-data or dv-metadata.",
+      "category": "overview",
+      "description": "Skill routing for solution lifecycle — must go to dv-solution, not dv-data/dv-metadata",
+      "priority": 1,
+      "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview"},
+      "custom_metadata": {"skill": "dv-overview", "tool_routing": "skill"},
+      "assertions": [
+        "PRIORITY_1: SKILL_LOADED: dv-overview",
+        "PRIORITY_1: CONTAINS: dv-solution",
+        "PRIORITY_1: NOT_CONTAINS: dv-data",
+        "PRIORITY_2: Agent mentions PAC CLI (pac solution export / unpack) as the documented tool for solution lifecycle."
+      ]
+    },
+    {
+      "test_id": "overview_003_routing_bulk_data",
+      "prompt": "I need to load 10,000 records into a Dataverse table from a CSV. Which skill handles this and what tool does it recommend?",
+      "expected_response": "Agent routes to dv-data and recommends the Python SDK bulk path (CreateMultiple / bulk_create helper with chunking) — not MCP and not per-record loops.",
+      "category": "overview",
+      "description": "Skill routing for bulk data load — dv-data + SDK bulk path",
+      "priority": 1,
+      "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview"},
+      "custom_metadata": {"skill": "dv-overview", "tool_routing": "skill"},
+      "assertions": [
+        "PRIORITY_1: SKILL_LOADED: dv-overview",
+        "PRIORITY_1: CONTAINS: dv-data",
+        "PRIORITY_1: Agent recommends the SDK bulk path (CreateMultiple / bulk_create) for 10k records, NOT MCP and NOT per-record loops.",
+        "PRIORITY_2: NOT_CONTAINS: per-record loop",
+        "PRIORITY_2: Agent does not route 10k-record imports to MCP (MCP is for small interactive batches)."
+      ]
+    },
+    {
+      "test_id": "overview_004_tool_routing_hierarchy",
+      "prompt": "Summarize the tool-routing rules in dv-overview. When do I use MCP vs the Python SDK vs the Web API vs PAC CLI?",
+      "expected_response": "Agent reports the documented hierarchy: MCP for interactive small ops (~1-10 rows), Python SDK for bulk and scripted workflows, Web API only for documented SDK gaps, PAC CLI for solution lifecycle.",
+      "category": "overview",
+      "description": "Skill-contract — overview's tool-routing hierarchy is reported correctly",
+      "priority": 1,
+      "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview", "Type": "skill-contract"},
+      "custom_metadata": {"skill": "dv-overview", "test_kind": "skill-contract"},
+      "assertions": [
+        "PRIORITY_1: SKILL_LOADED: dv-overview",
+        "PRIORITY_1: CONTAINS: MCP",
+        "PRIORITY_1: CONTAINS: SDK",
+        "PRIORITY_1: CONTAINS: PAC",
+        "PRIORITY_1: Agent describes the documented routing hierarchy — MCP for small interactive ops, SDK for bulk/scripted, Web API for SDK gaps, PAC CLI for solution lifecycle. If the agent inverts the order (e.g. recommends Web API as primary) or claims SDK doesn't support bulk, FAIL."
+      ]
+    }
+  ]
+}