From 99abf099ae08ad905ce46c9676d061f108c4fcfc Mon Sep 17 00:00:00 2001 From: Saurabh Badenkal Date: Mon, 1 Jun 2026 22:38:56 -0700 Subject: [PATCH 1/6] add: deterministic-evaluator eval tests for connect/metadata/overview/query/solution; convert dv_data to new format Adds 3-4 tests per skill using the new generic deterministic-evaluator format introduced in LocalEvalRunner: each test file now enables CortexConfigurations:Common/DeterministicAssertionEvaluator with settings.supported_verbs=CONTAINS,NOT_CONTAINS,SKILL_LOADED alongside the correctness.prompty semantic judge. Ports the dev/evalsV0 baseline tests (connect_001, metadata_001, overview_001, query_001, solution_001) and adds 2-3 natural follow-up tests per skill covering env-file contract, no-hardcoded-secrets, schema create + lookup relationship, filtered/aggregate reads, and solution unpack/import/routing-trap. dv_data.biceval.json is updated in place to add the deterministic evaluator. --- evals/tests/dv_connect.biceval.json | 88 +++++++++++++++++++++++++++ evals/tests/dv_data.biceval.json | 8 +++ evals/tests/dv_metadata.biceval.json | 78 ++++++++++++++++++++++++ evals/tests/dv_overview.biceval.json | 89 +++++++++++++++++++++++++++ evals/tests/dv_query.biceval.json | 90 ++++++++++++++++++++++++++++ evals/tests/dv_solution.biceval.json | 89 +++++++++++++++++++++++++++ 6 files changed, 442 insertions(+) create mode 100644 evals/tests/dv_connect.biceval.json create mode 100644 evals/tests/dv_metadata.biceval.json create mode 100644 evals/tests/dv_overview.biceval.json create mode 100644 evals/tests/dv_query.biceval.json create mode 100644 evals/tests/dv_solution.biceval.json diff --git a/evals/tests/dv_connect.biceval.json b/evals/tests/dv_connect.biceval.json new file mode 100644 index 0000000..e627e8f --- /dev/null +++ b/evals/tests/dv_connect.biceval.json @@ -0,0 +1,88 @@ +{ + "group": "DataverseSkills", + "scenarioName": "Connect", + "description": "Verifies the plugin's canonical auth + connection pattern (scripts/auth.py, get_credential/get_client, DATAVERSE_URL from .env).", + "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", + "passing_score": 3, + "priority": 1, + "settings": { + "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" + } + }, + { + "name": "CortexConfigurations:Common/Skills/correctness.prompty", + "passing_score": 3, + "priority": 1 + } + ], + "tests": [ + { + "test_id": "connect_001", + "prompt": "Write a Python script that authenticates to my Dataverse environment and prints the URL. Use the plugin's standard auth helper.", + "expected_response": "Agent uses scripts/auth.py with get_credential() (or get_client()) and DataverseClient. Reads DATAVERSE_URL from .env.", + "category": "connect", + "description": "Plugin auth pattern is reachable", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect"}, + "custom_metadata": {"skill": "dv-connect", "tool_routing": "SDK"}, + "assertions": [ + "PRIORITY_1: Agent demonstrates the canonical Dataverse-skills connect path — references scripts/auth.py (or 'from auth import') AND uses one of the documented credential helpers (get_credential / get_client / get_token / DataverseClient). Hand-rolled OAuth or hardcoded tokens = FAIL.", + "PRIORITY_1: Agent's executable code is Python (no JavaScript/TypeScript/Node.js or PowerShell).", + "PRIORITY_2: CONTAINS: DATAVERSE_URL", + "PRIORITY_2: SKILL_LOADED: dv-connect" + ] + }, + { + "test_id": "connect_002_env_file", + "prompt": "Where does the Dataverse plugin expect me to put my environment URL and tenant configuration? Show me the exact file and variable names.", + "expected_response": "Agent points to .env at repo root with DATAVERSE_URL (and optionally DATAVERSE_TENANT_ID). References load_env() from scripts/auth.py.", + "category": "connect", + "description": "Plugin .env contract — variable names and load mechanism", + "priority": 2, + "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect"}, + "custom_metadata": {"skill": "dv-connect", "test_kind": "skill-contract"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-connect", + "PRIORITY_1: CONTAINS: DATAVERSE_URL", + "PRIORITY_1: CONTAINS: .env", + "PRIORITY_2: Agent mentions load_env or scripts/auth.py as the way the plugin reads the env file." + ] + }, + { + "test_id": "connect_003_no_hardcoded_secrets", + "prompt": "Generate a snippet that connects to my Dataverse environment using my client ID and secret. Make sure I can run it.", + "expected_response": "Agent refuses to hardcode secrets in source; routes through scripts/auth.py / get_credential() (DefaultAzureCredential / device code) and instructs the user to put secrets in .env or environment variables.", + "category": "connect", + "description": "Safety — agent must not inline client secrets, must route through documented auth helper", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect", "Type": "safety"}, + "custom_metadata": {"skill": "dv-connect", "trap": "hardcoded secret"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-connect", + "PRIORITY_1: Agent does NOT hardcode a client secret literal in the generated Python file; secrets are loaded from environment or .env via the documented helper.", + "PRIORITY_1: Agent references scripts/auth.py / get_credential / get_client rather than building an MSAL ConfidentialClientApplication by hand.", + "PRIORITY_2: NOT_CONTAINS: client_secret=\"", + "PRIORITY_2: NOT_CONTAINS: client_secret='" + ] + }, + { + "test_id": "connect_004_verify", + "prompt": "Write a small Python script that connects to my Dataverse environment using the plugin's auth helper and prints the environment URL plus the count of tables in the org (just to verify the connection works).", + "expected_response": "Agent uses get_client()/get_credential() + DataverseClient, prints DATAVERSE_URL, and runs a metadata read (e.g. client.tables.list or a simple records query on a system table) to confirm the connection.", + "category": "connect", + "description": "End-to-end smoke: connect + minimal verification call via SDK", + "priority": 2, + "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect"}, + "custom_metadata": {"skill": "dv-connect", "tool_routing": "SDK"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-connect", + "PRIORITY_1: Agent uses DataverseClient instantiated via the plugin auth helper (get_client / get_credential) rather than building auth headers manually.", + "PRIORITY_1: Agent's executable code is Python.", + "PRIORITY_2: CONTAINS: DATAVERSE_URL", + "PRIORITY_2: Agent includes a verification call against the live environment (table listing, metadata fetch, or a simple records read) — not just a print of the URL." + ] + } + ] +} diff --git a/evals/tests/dv_data.biceval.json b/evals/tests/dv_data.biceval.json index 00d6b5b..82a8853 100644 --- a/evals/tests/dv_data.biceval.json +++ b/evals/tests/dv_data.biceval.json @@ -3,6 +3,14 @@ "scenarioName": "Data", "description": "Verifies SDK patterns for record creation in Dataverse tables.", "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", + "passing_score": 3, + "priority": 1, + "settings": { + "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" + } + }, { "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, diff --git a/evals/tests/dv_metadata.biceval.json b/evals/tests/dv_metadata.biceval.json new file mode 100644 index 0000000..3ed21a5 --- /dev/null +++ b/evals/tests/dv_metadata.biceval.json @@ -0,0 +1,78 @@ +{ + "group": "DataverseSkills", + "scenarioName": "Metadata", + "description": "Verifies dv-metadata routing — SDK for supported schema ops, Web API fallback for documented SDK gaps (e.g. SavedQuery).", + "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", + "passing_score": 3, + "priority": 1, + "settings": { + "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" + } + }, + { + "name": "CortexConfigurations:Common/Skills/correctness.prompty", + "passing_score": 3, + "priority": 1 + } + ], + "tests": [ + { + "test_id": "metadata_001_savedquery_webapi", + "prompt": "Write a Python script that creates a SavedQuery view showing all active records of the cr_projects table in my Dataverse environment.", + "expected_response": "Agent uses urllib.request + get_token() (SavedQuery is a documented SDK gap). POSTs to /api/data/v9.2/savedqueries with FetchXml body.", + "category": "metadata", + "description": "Web API for SDK gap (SavedQuery / view) — baseline routing test", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Metadata", "Skill": "dv-metadata", "Type": "web-api-gap"}, + "custom_metadata": {"skill": "dv-metadata", "tool_routing": "WebAPI"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-metadata", + "PRIORITY_1: CONTAINS: savedquer", + "PRIORITY_1: CONTAINS: urllib", + "PRIORITY_1: CONTAINS: get_token", + "PRIORITY_1: Agent's executable code is Python (no JavaScript/TypeScript/Node.js or PowerShell).", + "PRIORITY_1: Agent recognizes view creation (SavedQuery) is a Tier 2 Web API operation — does NOT invent a non-existent SDK method like client.views.create() or client.savedqueries.create(). Hallucinated SDK methods = FAIL.", + "PRIORITY_2: CONTAINS: cr_projects", + "PRIORITY_2: CONTAINS: fetchxml" + ] + }, + { + "test_id": "metadata_002_create_table_sdk", + "prompt": "Create a new Dataverse table called cr_projects with two columns: a text column called cr_name (primary name) and a choice column cr_status with values Active and Closed. Write a Python script.", + "expected_response": "Agent uses Python SDK metadata helpers (client.tables.create / client.columns.create or equivalent) — single-table create then column adds. Does NOT hand-roll EntityMetadata Web API POSTs.", + "category": "metadata", + "description": "Create table + columns via SDK metadata helpers", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Metadata", "Skill": "dv-metadata", "Type": "sdk"}, + "custom_metadata": {"skill": "dv-metadata", "tool_routing": "SDK"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-metadata", + "PRIORITY_1: Agent uses the Python SDK metadata surface (client.tables / client.columns or the documented helpers) for table and column creation — NOT raw urllib POSTs to /api/data/v9.2/EntityDefinitions.", + "PRIORITY_1: Agent's executable code is Python.", + "PRIORITY_2: CONTAINS: cr_projects", + "PRIORITY_2: CONTAINS: cr_status", + "PRIORITY_2: CONTAINS: cr_name" + ] + }, + { + "test_id": "metadata_003_add_lookup", + "prompt": "Add a lookup column on my cr_tasks table that points to cr_projects. Call it cr_projectid. Write the Python.", + "expected_response": "Agent uses SDK lookup/relationship helper (e.g. client.relationships.create_many_to_one or client.columns.create_lookup) — recognizes a lookup is a relationship, not a plain column.", + "category": "metadata", + "description": "Lookup column = relationship — SDK relationship helper, not plain column create", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Metadata", "Skill": "dv-metadata", "Type": "relationship"}, + "custom_metadata": {"skill": "dv-metadata", "tool_routing": "SDK", "trap": "lookup treated as plain column"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-metadata", + "PRIORITY_1: Agent recognizes a lookup column is a many-to-one relationship — uses an SDK relationship/lookup helper rather than a plain client.columns.create of type 'string' or 'lookup'.", + "PRIORITY_1: Agent's executable code is Python.", + "PRIORITY_2: CONTAINS: cr_projects", + "PRIORITY_2: CONTAINS: cr_tasks", + "PRIORITY_2: CONTAINS: cr_projectid" + ] + } + ] +} diff --git a/evals/tests/dv_overview.biceval.json b/evals/tests/dv_overview.biceval.json new file mode 100644 index 0000000..393b586 --- /dev/null +++ b/evals/tests/dv_overview.biceval.json @@ -0,0 +1,89 @@ +{ + "group": "DataverseSkills", + "scenarioName": "Overview", + "description": "Verifies dv-overview correctly routes multi-skill tasks to the right child skills (dv-data, dv-query, dv-metadata, dv-solution, dv-connect).", + "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", + "passing_score": 3, + "priority": 1, + "settings": { + "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" + } + }, + { + "name": "CortexConfigurations:Common/Skills/correctness.prompty", + "passing_score": 3, + "priority": 1 + } + ], + "tests": [ + { + "test_id": "overview_001_routing_read_and_create", + "prompt": "I want to read some records AND create a new table in my Dataverse environment in the same session. Which skills and tools should I use for each?", + "expected_response": "Agent loads dv-overview, mentions dv-query for reads and dv-metadata for table creation. Recommends Python SDK for both.", + "category": "overview", + "description": "Skill routing for combined read + schema-create", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview"}, + "custom_metadata": {"skill": "dv-overview", "tool_routing": "skill"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-overview", + "PRIORITY_1: CONTAINS: dv-query", + "PRIORITY_1: CONTAINS: dv-metadata", + "PRIORITY_1: Agent correctly partitions the work — routes record reads to dv-query and table creation to dv-metadata, NOT both to a single skill.", + "PRIORITY_2: Agent recommends the Python SDK (DataverseClient) for both operations." + ] + }, + { + "test_id": "overview_002_routing_solution_export", + "prompt": "How should I export my Dataverse solution to source control? Just tell me which skill to use and the high-level approach.", + "expected_response": "Agent routes to dv-solution and recommends PAC CLI (pac solution export / unpack). Does NOT route to dv-data or dv-metadata.", + "category": "overview", + "description": "Skill routing for solution lifecycle — must go to dv-solution, not dv-data/dv-metadata", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview"}, + "custom_metadata": {"skill": "dv-overview", "tool_routing": "skill"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-overview", + "PRIORITY_1: CONTAINS: dv-solution", + "PRIORITY_1: NOT_CONTAINS: dv-data", + "PRIORITY_2: Agent mentions PAC CLI (pac solution export / unpack) as the documented tool for solution lifecycle." + ] + }, + { + "test_id": "overview_003_routing_bulk_data", + "prompt": "I need to load 10,000 records into a Dataverse table from a CSV. Which skill handles this and what tool does it recommend?", + "expected_response": "Agent routes to dv-data and recommends the Python SDK bulk path (CreateMultiple / bulk_create helper with chunking) — not MCP and not per-record loops.", + "category": "overview", + "description": "Skill routing for bulk data load — dv-data + SDK bulk path", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview"}, + "custom_metadata": {"skill": "dv-overview", "tool_routing": "skill"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-overview", + "PRIORITY_1: CONTAINS: dv-data", + "PRIORITY_1: Agent recommends the SDK bulk path (CreateMultiple / bulk_create) for 10k records, NOT MCP and NOT per-record loops.", + "PRIORITY_2: NOT_CONTAINS: per-record loop", + "PRIORITY_2: Agent does not route 10k-record imports to MCP (MCP is for small interactive batches)." + ] + }, + { + "test_id": "overview_004_tool_routing_hierarchy", + "prompt": "Summarize the tool-routing rules in dv-overview. When do I use MCP vs the Python SDK vs the Web API vs PAC CLI?", + "expected_response": "Agent reports the documented hierarchy: MCP for interactive small ops (~1-10 rows), Python SDK for bulk and scripted workflows, Web API only for documented SDK gaps, PAC CLI for solution lifecycle.", + "category": "overview", + "description": "Skill-contract — overview's tool-routing hierarchy is reported correctly", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview", "Type": "skill-contract"}, + "custom_metadata": {"skill": "dv-overview", "test_kind": "skill-contract"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-overview", + "PRIORITY_1: CONTAINS: MCP", + "PRIORITY_1: CONTAINS: SDK", + "PRIORITY_1: CONTAINS: PAC", + "PRIORITY_1: Agent describes the documented routing hierarchy — MCP for small interactive ops, SDK for bulk/scripted, Web API for SDK gaps, PAC CLI for solution lifecycle. If the agent inverts the order (e.g. recommends Web API as primary) or claims SDK doesn't support bulk, FAIL." + ] + } + ] +} diff --git a/evals/tests/dv_query.biceval.json b/evals/tests/dv_query.biceval.json new file mode 100644 index 0000000..766fdf5 --- /dev/null +++ b/evals/tests/dv_query.biceval.json @@ -0,0 +1,90 @@ +{ + "group": "DataverseSkills", + "scenarioName": "Query", + "description": "Verifies dv-query SDK patterns for reading records — auto-paginating bulk reads, filters, and FetchXML/Web API fallback only where the SDK has documented gaps.", + "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", + "passing_score": 3, + "priority": 1, + "settings": { + "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" + } + }, + { + "name": "CortexConfigurations:Common/Skills/correctness.prompty", + "passing_score": 3, + "priority": 1 + } + ], + "tests": [ + { + "test_id": "query_001_bulk_read", + "prompt": "Show me all records in the cr_customers table from my Dataverse environment. Write a Python script.", + "expected_response": "Agent uses Python SDK: DataverseClient + client.records.get('cr_customers') (or client.dataframe.get / equivalent). Auto-paginates — does NOT manually construct $skip/$top URL parameters.", + "category": "query", + "description": "Bulk read via SDK — auto-paginating, no manual OData paging", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Query", "Skill": "dv-query"}, + "custom_metadata": {"skill": "dv-query", "tool_routing": "SDK"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-query", + "PRIORITY_1: CONTAINS: cr_customers", + "PRIORITY_1: Agent uses the official Python SDK (DataverseClient.records.get / dataframe.get / equivalent) — raw urllib/requests for a basic read the SDK already supports = FAIL.", + "PRIORITY_1: Agent's executable code is Python.", + "PRIORITY_2: Agent does NOT manually construct OData $skip/$top URL parameters for a basic 'show me records' query — relies on the SDK's auto-pagination." + ] + }, + { + "test_id": "query_002_filtered_read", + "prompt": "Read all cr_customers where statecode = 0 (active) and city = 'Seattle'. Python.", + "expected_response": "Agent uses the SDK's filter parameter (e.g. client.records.get('cr_customers', filter=\"statecode eq 0 and city eq 'Seattle'\")) — server-side filter, not a client-side Python comprehension after fetching everything.", + "category": "query", + "description": "Filtered read — server-side filter via SDK, not client-side filter on a full table fetch", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Query", "Skill": "dv-query", "Type": "filter"}, + "custom_metadata": {"skill": "dv-query", "tool_routing": "SDK", "trap": "client-side filter antipattern"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-query", + "PRIORITY_1: CONTAINS: cr_customers", + "PRIORITY_1: Agent pushes the filter to the server via the SDK filter/OData parameter — does NOT fetch the whole table and filter in a Python list comprehension or pandas mask.", + "PRIORITY_2: CONTAINS: statecode", + "PRIORITY_2: CONTAINS: Seattle" + ] + }, + { + "test_id": "query_003_aggregate_fetchxml", + "prompt": "Get the count of cr_orders per cr_customerid for the top 10 customers by order count. Write Python.", + "expected_response": "Agent recognizes aggregates require FetchXML (Web API) — uses urllib.request + get_token to POST FetchXML with + + count(1) groupby, OR uses the documented SDK FetchXML helper.", + "category": "query", + "description": "Aggregation requires FetchXML — SDK gap, must use Web API or SDK FetchXML helper", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Query", "Skill": "dv-query", "Type": "aggregation"}, + "custom_metadata": {"skill": "dv-query", "tool_routing": "WebAPI"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-query", + "PRIORITY_1: CONTAINS: fetchxml", + "PRIORITY_1: Agent uses FetchXML (via Web API urllib+get_token, or the SDK's FetchXML helper) for the aggregation — does NOT fetch all rows and aggregate client-side in pandas.", + "PRIORITY_2: CONTAINS: cr_orders", + "PRIORITY_2: CONTAINS: cr_customerid", + "PRIORITY_2: Agent's FetchXML includes an aggregate attribute (count) and a groupby." + ] + }, + { + "test_id": "query_004_single_record_mcp_or_sdk", + "prompt": "Look up a single Dataverse cr_customers record by id 11111111-2222-3333-4444-555555555555 and print its name.", + "expected_response": "Agent uses a targeted single-record read — MCP (read_record) for an interactive lookup, or SDK client.records.get_by_id. Does NOT fetch the entire table and filter.", + "category": "query", + "description": "Single-record by id — targeted read, not full-table scan", + "priority": 2, + "tags": {"Suite": "Regression", "Domain": "Query", "Skill": "dv-query", "Type": "single-record"}, + "custom_metadata": {"skill": "dv-query", "tool_routing": "SDK"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-query", + "PRIORITY_1: CONTAINS: cr_customers", + "PRIORITY_1: Agent performs a targeted single-record read (by id) — does NOT fetch the whole cr_customers table and filter in memory.", + "PRIORITY_2: Agent uses the Python SDK get-by-id pattern or MCP read_record — not a hand-rolled urllib request when the SDK supports it." + ] + } + ] +} diff --git a/evals/tests/dv_solution.biceval.json b/evals/tests/dv_solution.biceval.json new file mode 100644 index 0000000..0fe1e88 --- /dev/null +++ b/evals/tests/dv_solution.biceval.json @@ -0,0 +1,89 @@ +{ + "group": "DataverseSkills", + "scenarioName": "Solution", + "description": "Verifies dv-solution routes solution lifecycle (export/import/pack/unpack) to PAC CLI — NOT the Python SDK or Web API.", + "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", + "passing_score": 3, + "priority": 1, + "settings": { + "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" + } + }, + { + "name": "CortexConfigurations:Common/Skills/correctness.prompty", + "passing_score": 3, + "priority": 1 + } + ], + "tests": [ + { + "test_id": "solution_001_export", + "prompt": "How do I export a Dataverse solution called ContosoApp from my dev environment to a zip file?", + "expected_response": "Agent recommends PAC CLI: 'pac solution export --name ContosoApp --path X.zip'.", + "category": "solution", + "description": "Solution export via PAC CLI", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Solution", "Skill": "dv-solution"}, + "custom_metadata": {"skill": "dv-solution", "tool_routing": "PAC"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-solution", + "PRIORITY_1: CONTAINS: pac solution export", + "PRIORITY_1: CONTAINS: ContosoApp", + "PRIORITY_1: Agent uses PAC CLI for the solution export, NOT the Python SDK or Web API. Solution lifecycle is Tier 3 PAC CLI territory.", + "PRIORITY_2: Agent mentions the --path / output zip argument." + ] + }, + { + "test_id": "solution_002_unpack_for_source_control", + "prompt": "I exported ContosoApp.zip. How do I unpack it so I can check the contents into Git?", + "expected_response": "Agent recommends 'pac solution unpack --zipfile ContosoApp.zip --folder ./solutions/ContosoApp --packagetype Both' (or similar). Routes to PAC CLI, not manual zip extraction.", + "category": "solution", + "description": "Unpack solution for source control via PAC CLI", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Solution", "Skill": "dv-solution"}, + "custom_metadata": {"skill": "dv-solution", "tool_routing": "PAC"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-solution", + "PRIORITY_1: CONTAINS: pac solution unpack", + "PRIORITY_1: Agent uses 'pac solution unpack' rather than recommending a manual zip extraction or a Python unzip script — unpack normalizes the XML for diff-friendly source control.", + "PRIORITY_2: CONTAINS: ContosoApp" + ] + }, + { + "test_id": "solution_003_import", + "prompt": "How do I import the ContosoApp solution zip into my test environment?", + "expected_response": "Agent recommends 'pac solution import --path ContosoApp.zip' (optionally with --async / --publish-changes). Routes to PAC CLI.", + "category": "solution", + "description": "Solution import via PAC CLI", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Solution", "Skill": "dv-solution"}, + "custom_metadata": {"skill": "dv-solution", "tool_routing": "PAC"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-solution", + "PRIORITY_1: CONTAINS: pac solution import", + "PRIORITY_1: Agent uses PAC CLI for the import, NOT the Python SDK or Web API.", + "PRIORITY_2: CONTAINS: ContosoApp" + ] + }, + { + "test_id": "solution_004_routing_violation_trap", + "prompt": "Write a Python script using the Dataverse SDK to export the ContosoApp solution to a zip file. I want to do this from a Python notebook, not the CLI.", + "expected_response": "Agent refuses to fabricate a non-existent SDK solution API and routes the user to PAC CLI (optionally via subprocess from Python) — solution export is not in the Python SDK surface.", + "category": "solution", + "description": "Routing-violation trap — agent must NOT hallucinate an SDK solution.export() method", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Solution", "Skill": "dv-solution", "Type": "routing-trap"}, + "custom_metadata": {"skill": "dv-solution", "trap": "hallucinated SDK solution API"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-solution", + "PRIORITY_1: Agent does NOT invent a non-existent SDK call like client.solutions.export() or client.solution.export() — solution lifecycle is not in the Python SDK.", + "PRIORITY_1: Agent routes to PAC CLI (either as a recommendation, or as a subprocess.run('pac solution export ...') wrapper inside the Python script).", + "PRIORITY_2: CONTAINS: pac solution export", + "PRIORITY_2: NOT_CONTAINS: client.solutions.export", + "PRIORITY_2: NOT_CONTAINS: client.solution.export" + ] + } + ] +} From 06d5524b9e907eddf68583a197af78c6793d6b56 Mon Sep 17 00:00:00 2001 From: Saurabh Badenkal Date: Tue, 2 Jun 2026 11:36:21 -0700 Subject: [PATCH 2/6] add: LMChecklist evaluator so natural-language assertions actually get graded Per LER PR-393 author guidance: DeterministicAssertionEvaluator only grades verb-prefixed assertions (CONTAINS/NOT_CONTAINS/SKILL_LOADED), and correctness.prompty scores against expected_response without seeing individual assertions. Without LMChecklist, natural-language assertions (those without a verb prefix) are unscored. Adds LMChecklist (Common/SEVAL/LMChecklist.prompty) to all six test files using the exact name + passing_score=3 + priority=1 shape the LER author published. Loader registration is already proven against the new format in Dataverse-skills PR #71's draft validation runs (LocalEvalRunner builds 20289122 and 20290419). --- evals/tests/dv_connect.biceval.json | 5 +++++ evals/tests/dv_data.biceval.json | 5 +++++ evals/tests/dv_metadata.biceval.json | 5 +++++ evals/tests/dv_overview.biceval.json | 5 +++++ evals/tests/dv_query.biceval.json | 5 +++++ evals/tests/dv_solution.biceval.json | 5 +++++ 6 files changed, 30 insertions(+) diff --git a/evals/tests/dv_connect.biceval.json b/evals/tests/dv_connect.biceval.json index e627e8f..05bcee5 100644 --- a/evals/tests/dv_connect.biceval.json +++ b/evals/tests/dv_connect.biceval.json @@ -15,6 +15,11 @@ "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 3, + "priority": 1 } ], "tests": [ diff --git a/evals/tests/dv_data.biceval.json b/evals/tests/dv_data.biceval.json index 82a8853..71d8531 100644 --- a/evals/tests/dv_data.biceval.json +++ b/evals/tests/dv_data.biceval.json @@ -15,6 +15,11 @@ "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 3, + "priority": 1 } ], "tests": [ diff --git a/evals/tests/dv_metadata.biceval.json b/evals/tests/dv_metadata.biceval.json index 3ed21a5..9f48a33 100644 --- a/evals/tests/dv_metadata.biceval.json +++ b/evals/tests/dv_metadata.biceval.json @@ -15,6 +15,11 @@ "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 3, + "priority": 1 } ], "tests": [ diff --git a/evals/tests/dv_overview.biceval.json b/evals/tests/dv_overview.biceval.json index 393b586..7a6d862 100644 --- a/evals/tests/dv_overview.biceval.json +++ b/evals/tests/dv_overview.biceval.json @@ -15,6 +15,11 @@ "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 3, + "priority": 1 } ], "tests": [ diff --git a/evals/tests/dv_query.biceval.json b/evals/tests/dv_query.biceval.json index 766fdf5..ab7848b 100644 --- a/evals/tests/dv_query.biceval.json +++ b/evals/tests/dv_query.biceval.json @@ -15,6 +15,11 @@ "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 3, + "priority": 1 } ], "tests": [ diff --git a/evals/tests/dv_solution.biceval.json b/evals/tests/dv_solution.biceval.json index 0fe1e88..f446275 100644 --- a/evals/tests/dv_solution.biceval.json +++ b/evals/tests/dv_solution.biceval.json @@ -15,6 +15,11 @@ "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 3, + "priority": 1 } ], "tests": [ From 357f9a6ebc441185e1e379735e0e3901778284ce Mon Sep 17 00:00:00 2001 From: Saurabh Badenkal Date: Tue, 2 Jun 2026 13:50:26 -0700 Subject: [PATCH 3/6] harness: drop DeterministicAssertionEvaluator + enable DisableEvaluationService for local-eval mode Bad-PR validation harness. Routes around the pre-existing LER->BICEP ASCII-headers bug by forcing local evaluation: LMChecklist.prompty grades each assertion (verb-prefixed and natural-language alike) via CAPI/AOAI locally. Drops the DeterministicAssertionEvaluator entry because in DisableEvaluationService mode it would fall through to a local file lookup and crash (same root issue LER PR #416 fixes for the service-enabled path). Harness PR is draft and closes unmerged; PR #70 (test files) and PR #416 (LER fixes) are untouched. --- .azdo/DataversePluginEvals_PR.yml | 16 ++++++++++++++-- evals/tests/dv_data.biceval.json | 8 -------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/.azdo/DataversePluginEvals_PR.yml b/.azdo/DataversePluginEvals_PR.yml index 01fb4bc..6f0625e 100644 --- a/.azdo/DataversePluginEvals_PR.yml +++ b/.azdo/DataversePluginEvals_PR.yml @@ -56,7 +56,10 @@ resources: type: github endpoint: bic-proxima-shared-stamp name: bic/LocalEvalRunner - ref: refs/heads/main + # Temporary: pinned to LER PR #416 branch so this validation harness can build LER from source + # with the DeterministicAssertionEvaluator service-only fix + sourceRepoPath template param. + # Will be reverted to refs/heads/main; this harness is a draft and closes unmerged. + ref: refs/heads/users/sbadenkal/service-only-deterministic-assertion - repository: BicEvalsService type: github endpoint: bic-proxima-shared-stamp @@ -67,15 +70,21 @@ steps: - checkout: self displayName: 'Checkout Dataverse-skills' +- checkout: LocalEvalRunner + displayName: 'Checkout LocalEvalRunner source (for build-from-source)' + fetchDepth: 1 + - checkout: BicEvalsService displayName: 'Checkout evaluator prompts' fetchDepth: 1 -# Setup: .NET, NuGet auth, LocalEvalRunner install, Copilot CLI, auth token +# Setup: .NET, NuGet auth, LocalEvalRunner BUILD FROM SOURCE (PR #416 branch), Copilot CLI, auth token - template: .pipelines/templates/steps/BicEval-SetupTemplate.yml@LocalEvalRunner parameters: cliAgent: 'copilot' cliAgentTokenVariable: 'COPILOT_GH_PAT' + useBuildFromSource: true + sourceRepoPath: '$(Build.SourcesDirectory)/LocalEvalRunner' # Configure: merge replacements, prepare agent config - template: .pipelines/templates/steps/BicEval-ConfigureTemplate.yml@LocalEvalRunner @@ -91,6 +100,8 @@ steps: pluginDir: '.github/plugins/dataverse' # Execute: run localevalrunner with CopilotCliAgent +# DisableEvaluationService routes around the pre-existing LER->BICEP ASCII-headers bug +# by forcing local-eval mode (correctness.prompty + LMChecklist.prompty grade locally via CAPI/AOAI). - template: .pipelines/templates/steps/BicEval-ExecuteTemplate.yml@LocalEvalRunner parameters: sourceRepoTestFilePath: 'Dataverse-skills/evals/tests/${{ parameters.testFile }}' @@ -99,6 +110,7 @@ steps: azureServiceConnection: ${{ parameters.azureServiceConnection }} concurrency: ${{ parameters.concurrency }} serviceName: 'DataverseSkillsPlugin' + features: 'DisableEvaluationService' # Publish: test results + artifacts - template: .pipelines/templates/steps/BicEval-PublishTemplate.yml@LocalEvalRunner diff --git a/evals/tests/dv_data.biceval.json b/evals/tests/dv_data.biceval.json index 71d8531..4fc311b 100644 --- a/evals/tests/dv_data.biceval.json +++ b/evals/tests/dv_data.biceval.json @@ -3,14 +3,6 @@ "scenarioName": "Data", "description": "Verifies SDK patterns for record creation in Dataverse tables.", "enabled_evaluators": [ - { - "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", - "passing_score": 3, - "priority": 1, - "settings": { - "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" - } - }, { "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, From 44c07c0db5429e46b2e537737d6fa836de643c94 Mon Sep 17 00:00:00 2001 From: Saurabh Badenkal Date: Tue, 2 Jun 2026 13:52:29 -0700 Subject: [PATCH 4/6] harness: drop DeterministicAssertionEvaluator from remaining 5 test files for local-eval mode --- evals/tests/dv_connect.biceval.json | 8 -------- evals/tests/dv_metadata.biceval.json | 8 -------- evals/tests/dv_overview.biceval.json | 8 -------- evals/tests/dv_query.biceval.json | 8 -------- evals/tests/dv_solution.biceval.json | 8 -------- 5 files changed, 40 deletions(-) diff --git a/evals/tests/dv_connect.biceval.json b/evals/tests/dv_connect.biceval.json index 05bcee5..53490fb 100644 --- a/evals/tests/dv_connect.biceval.json +++ b/evals/tests/dv_connect.biceval.json @@ -3,14 +3,6 @@ "scenarioName": "Connect", "description": "Verifies the plugin's canonical auth + connection pattern (scripts/auth.py, get_credential/get_client, DATAVERSE_URL from .env).", "enabled_evaluators": [ - { - "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", - "passing_score": 3, - "priority": 1, - "settings": { - "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" - } - }, { "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, diff --git a/evals/tests/dv_metadata.biceval.json b/evals/tests/dv_metadata.biceval.json index 9f48a33..9dca41d 100644 --- a/evals/tests/dv_metadata.biceval.json +++ b/evals/tests/dv_metadata.biceval.json @@ -3,14 +3,6 @@ "scenarioName": "Metadata", "description": "Verifies dv-metadata routing — SDK for supported schema ops, Web API fallback for documented SDK gaps (e.g. SavedQuery).", "enabled_evaluators": [ - { - "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", - "passing_score": 3, - "priority": 1, - "settings": { - "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" - } - }, { "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, diff --git a/evals/tests/dv_overview.biceval.json b/evals/tests/dv_overview.biceval.json index 7a6d862..435b40a 100644 --- a/evals/tests/dv_overview.biceval.json +++ b/evals/tests/dv_overview.biceval.json @@ -3,14 +3,6 @@ "scenarioName": "Overview", "description": "Verifies dv-overview correctly routes multi-skill tasks to the right child skills (dv-data, dv-query, dv-metadata, dv-solution, dv-connect).", "enabled_evaluators": [ - { - "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", - "passing_score": 3, - "priority": 1, - "settings": { - "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" - } - }, { "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, diff --git a/evals/tests/dv_query.biceval.json b/evals/tests/dv_query.biceval.json index ab7848b..d5fb408 100644 --- a/evals/tests/dv_query.biceval.json +++ b/evals/tests/dv_query.biceval.json @@ -3,14 +3,6 @@ "scenarioName": "Query", "description": "Verifies dv-query SDK patterns for reading records — auto-paginating bulk reads, filters, and FetchXML/Web API fallback only where the SDK has documented gaps.", "enabled_evaluators": [ - { - "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", - "passing_score": 3, - "priority": 1, - "settings": { - "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" - } - }, { "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, diff --git a/evals/tests/dv_solution.biceval.json b/evals/tests/dv_solution.biceval.json index f446275..c9ffaef 100644 --- a/evals/tests/dv_solution.biceval.json +++ b/evals/tests/dv_solution.biceval.json @@ -3,14 +3,6 @@ "scenarioName": "Solution", "description": "Verifies dv-solution routes solution lifecycle (export/import/pack/unpack) to PAC CLI — NOT the Python SDK or Web API.", "enabled_evaluators": [ - { - "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", - "passing_score": 3, - "priority": 1, - "settings": { - "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" - } - }, { "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, From b705f6f94f6fb9744bbfceff6b6721e5e91a9b19 Mon Sep 17 00:00:00 2001 From: Saurabh Badenkal Date: Tue, 2 Jun 2026 15:08:56 -0700 Subject: [PATCH 5/6] harness: lower LMChecklist passing_score 3 to 1 to match its 0/1 scoring range --- evals/tests/dv_connect.biceval.json | 2 +- evals/tests/dv_data.biceval.json | 2 +- evals/tests/dv_metadata.biceval.json | 2 +- evals/tests/dv_overview.biceval.json | 2 +- evals/tests/dv_query.biceval.json | 2 +- evals/tests/dv_solution.biceval.json | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/evals/tests/dv_connect.biceval.json b/evals/tests/dv_connect.biceval.json index 53490fb..1cf2378 100644 --- a/evals/tests/dv_connect.biceval.json +++ b/evals/tests/dv_connect.biceval.json @@ -10,7 +10,7 @@ }, { "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", - "passing_score": 3, + "passing_score": 1, "priority": 1 } ], diff --git a/evals/tests/dv_data.biceval.json b/evals/tests/dv_data.biceval.json index 4fc311b..fcf43ef 100644 --- a/evals/tests/dv_data.biceval.json +++ b/evals/tests/dv_data.biceval.json @@ -10,7 +10,7 @@ }, { "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", - "passing_score": 3, + "passing_score": 1, "priority": 1 } ], diff --git a/evals/tests/dv_metadata.biceval.json b/evals/tests/dv_metadata.biceval.json index 9dca41d..3802d02 100644 --- a/evals/tests/dv_metadata.biceval.json +++ b/evals/tests/dv_metadata.biceval.json @@ -10,7 +10,7 @@ }, { "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", - "passing_score": 3, + "passing_score": 1, "priority": 1 } ], diff --git a/evals/tests/dv_overview.biceval.json b/evals/tests/dv_overview.biceval.json index 435b40a..4a1ec2d 100644 --- a/evals/tests/dv_overview.biceval.json +++ b/evals/tests/dv_overview.biceval.json @@ -10,7 +10,7 @@ }, { "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", - "passing_score": 3, + "passing_score": 1, "priority": 1 } ], diff --git a/evals/tests/dv_query.biceval.json b/evals/tests/dv_query.biceval.json index d5fb408..58de036 100644 --- a/evals/tests/dv_query.biceval.json +++ b/evals/tests/dv_query.biceval.json @@ -10,7 +10,7 @@ }, { "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", - "passing_score": 3, + "passing_score": 1, "priority": 1 } ], diff --git a/evals/tests/dv_solution.biceval.json b/evals/tests/dv_solution.biceval.json index c9ffaef..4952709 100644 --- a/evals/tests/dv_solution.biceval.json +++ b/evals/tests/dv_solution.biceval.json @@ -10,7 +10,7 @@ }, { "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", - "passing_score": 3, + "passing_score": 1, "priority": 1 } ], From b566a812633eba0003772dbb9f2e6257a9b4ba60 Mon Sep 17 00:00:00 2001 From: Saurabh Badenkal Date: Tue, 2 Jun 2026 13:51:39 -0700 Subject: [PATCH 6/6] BAD PR (DO NOT MERGE): regress dv-data SKILL bulk-create guidance Intentional regression for bad-PR gate validation. Replaces the SDK bulk-create section (CreateMultiple + adaptive chunking helpers) with per-record-loop antipattern and 'no batch API' phrasing. Expected to fail: data_002_bulk_create (PRIORITY_1 bulk/list assertion) and data_003_skill_contract (PRIORITY_1 skill-content assertion). All other tests should still pass. --- .github/plugins/dataverse/skills/dv-data/SKILL.md | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/plugins/dataverse/skills/dv-data/SKILL.md b/.github/plugins/dataverse/skills/dv-data/SKILL.md index e97209b..6866bb9 100644 --- a/.github/plugins/dataverse/skills/dv-data/SKILL.md +++ b/.github/plugins/dataverse/skills/dv-data/SKILL.md @@ -155,18 +155,16 @@ client.records.delete("new_ticket", "") --- -## Bulk Create (SDK uses CreateMultiple internally) +## Creating Many Records + +There is no batch API in the Python SDK. For multiple records, just call `client.records.create()` once per record in a loop — one HTTP call per record. There is no chunking helper. ```python records = [{"new_name": f"Ticket {i}", "new_priority": 100000000} for i in range(500)] -guids = client.records.create("new_ticket", records) -print(f"Created {len(guids)} records") +for r in records: + client.records.create("new_ticket", r) ``` -Volume guidance: MCP `create_record` for 1-10 records. SDK for 10+ records. - -**Important:** The SDK sends all records in a single POST to `CreateMultiple`. It does **not** chunk automatically. Dataverse has no fixed record count limit — the constraints are payload size and request timeout (SDK default: 120s for POST). For larger datasets, you **must** chunk in your script. The `bulk_upsert` and `bulk_create` helpers below use adaptive chunking: start at 1,000, double on success (up to 4,000), halve on payload/timeout failure, and cap at the last successful size. Tables with few columns can handle larger chunks than tables with many columns. - --- ## Bulk Update