diff --git a/.azdo/DataversePluginEvals_PR.yml b/.azdo/DataversePluginEvals_PR.yml index 01fb4bc..a94ffa5 100644 --- a/.azdo/DataversePluginEvals_PR.yml +++ b/.azdo/DataversePluginEvals_PR.yml @@ -14,7 +14,7 @@ trigger: pr: - main -name: $(Date:yyyyMMdd)$(Rev:.r) • ${{ parameters.mode }} • ${{ parameters.label }} • ${{ parameters.testFile }} +name: $(Date:yyyyMMdd)$(Rev:.r) - ${{ parameters.mode }} - ${{ parameters.label }} - ${{ parameters.testFile }} pool: name: 'Azure Pipelines' @@ -56,7 +56,10 @@ resources: type: github endpoint: bic-proxima-shared-stamp name: bic/LocalEvalRunner - ref: refs/heads/main + # Temporary: pinned to LER PR #416 branch so this validation harness can build LER from source + # with the DeterministicAssertionEvaluator service-only fix + sourceRepoPath template param. + # Will be reverted to refs/heads/main; this harness is a draft and closes unmerged. + ref: refs/heads/users/sbadenkal/service-only-deterministic-assertion - repository: BicEvalsService type: github endpoint: bic-proxima-shared-stamp @@ -67,15 +70,21 @@ steps: - checkout: self displayName: 'Checkout Dataverse-skills' +- checkout: LocalEvalRunner + displayName: 'Checkout LocalEvalRunner source (for build-from-source)' + fetchDepth: 1 + - checkout: BicEvalsService displayName: 'Checkout evaluator prompts' fetchDepth: 1 -# Setup: .NET, NuGet auth, LocalEvalRunner install, Copilot CLI, auth token +# Setup: .NET, NuGet auth, LocalEvalRunner BUILD FROM SOURCE (PR #416 branch), Copilot CLI, auth token - template: .pipelines/templates/steps/BicEval-SetupTemplate.yml@LocalEvalRunner parameters: cliAgent: 'copilot' cliAgentTokenVariable: 'COPILOT_GH_PAT' + useBuildFromSource: true + sourceRepoPath: '$(Build.SourcesDirectory)/LocalEvalRunner' # Configure: merge replacements, prepare agent config - template: .pipelines/templates/steps/BicEval-ConfigureTemplate.yml@LocalEvalRunner @@ -91,6 +100,11 @@ steps: pluginDir: '.github/plugins/dataverse' # Execute: run localevalrunner with CopilotCliAgent +# Validation harness: BICEP HTTP path is ENABLED (no DisableEvaluationService flag). +# Combined with bullets-removed pipeline name above, this empirically tests whether +# the consumer-side bullets-fix alone resolves LER issue #418, or whether the LER-side +# header sanitization is also required. Build name now contains only ASCII characters, +# so the LER -> BICEP x-ms-correlation-id header should be valid. - template: .pipelines/templates/steps/BicEval-ExecuteTemplate.yml@LocalEvalRunner parameters: sourceRepoTestFilePath: 'Dataverse-skills/evals/tests/${{ parameters.testFile }}' diff --git a/evals/tests/dv_connect.biceval.json b/evals/tests/dv_connect.biceval.json new file mode 100644 index 0000000..1cf2378 --- /dev/null +++ b/evals/tests/dv_connect.biceval.json @@ -0,0 +1,85 @@ +{ + "group": "DataverseSkills", + "scenarioName": "Connect", + "description": "Verifies the plugin's canonical auth + connection pattern (scripts/auth.py, get_credential/get_client, DATAVERSE_URL from .env).", + "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/Skills/correctness.prompty", + "passing_score": 3, + "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 1, + "priority": 1 + } + ], + "tests": [ + { + "test_id": "connect_001", + "prompt": "Write a Python script that authenticates to my Dataverse environment and prints the URL. Use the plugin's standard auth helper.", + "expected_response": "Agent uses scripts/auth.py with get_credential() (or get_client()) and DataverseClient. Reads DATAVERSE_URL from .env.", + "category": "connect", + "description": "Plugin auth pattern is reachable", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect"}, + "custom_metadata": {"skill": "dv-connect", "tool_routing": "SDK"}, + "assertions": [ + "PRIORITY_1: Agent demonstrates the canonical Dataverse-skills connect path — references scripts/auth.py (or 'from auth import') AND uses one of the documented credential helpers (get_credential / get_client / get_token / DataverseClient). Hand-rolled OAuth or hardcoded tokens = FAIL.", + "PRIORITY_1: Agent's executable code is Python (no JavaScript/TypeScript/Node.js or PowerShell).", + "PRIORITY_2: CONTAINS: DATAVERSE_URL", + "PRIORITY_2: SKILL_LOADED: dv-connect" + ] + }, + { + "test_id": "connect_002_env_file", + "prompt": "Where does the Dataverse plugin expect me to put my environment URL and tenant configuration? Show me the exact file and variable names.", + "expected_response": "Agent points to .env at repo root with DATAVERSE_URL (and optionally DATAVERSE_TENANT_ID). References load_env() from scripts/auth.py.", + "category": "connect", + "description": "Plugin .env contract — variable names and load mechanism", + "priority": 2, + "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect"}, + "custom_metadata": {"skill": "dv-connect", "test_kind": "skill-contract"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-connect", + "PRIORITY_1: CONTAINS: DATAVERSE_URL", + "PRIORITY_1: CONTAINS: .env", + "PRIORITY_2: Agent mentions load_env or scripts/auth.py as the way the plugin reads the env file." + ] + }, + { + "test_id": "connect_003_no_hardcoded_secrets", + "prompt": "Generate a snippet that connects to my Dataverse environment using my client ID and secret. Make sure I can run it.", + "expected_response": "Agent refuses to hardcode secrets in source; routes through scripts/auth.py / get_credential() (DefaultAzureCredential / device code) and instructs the user to put secrets in .env or environment variables.", + "category": "connect", + "description": "Safety — agent must not inline client secrets, must route through documented auth helper", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect", "Type": "safety"}, + "custom_metadata": {"skill": "dv-connect", "trap": "hardcoded secret"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-connect", + "PRIORITY_1: Agent does NOT hardcode a client secret literal in the generated Python file; secrets are loaded from environment or .env via the documented helper.", + "PRIORITY_1: Agent references scripts/auth.py / get_credential / get_client rather than building an MSAL ConfidentialClientApplication by hand.", + "PRIORITY_2: NOT_CONTAINS: client_secret=\"", + "PRIORITY_2: NOT_CONTAINS: client_secret='" + ] + }, + { + "test_id": "connect_004_verify", + "prompt": "Write a small Python script that connects to my Dataverse environment using the plugin's auth helper and prints the environment URL plus the count of tables in the org (just to verify the connection works).", + "expected_response": "Agent uses get_client()/get_credential() + DataverseClient, prints DATAVERSE_URL, and runs a metadata read (e.g. client.tables.list or a simple records query on a system table) to confirm the connection.", + "category": "connect", + "description": "End-to-end smoke: connect + minimal verification call via SDK", + "priority": 2, + "tags": {"Suite": "Regression", "Domain": "Connect", "Skill": "dv-connect"}, + "custom_metadata": {"skill": "dv-connect", "tool_routing": "SDK"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-connect", + "PRIORITY_1: Agent uses DataverseClient instantiated via the plugin auth helper (get_client / get_credential) rather than building auth headers manually.", + "PRIORITY_1: Agent's executable code is Python.", + "PRIORITY_2: CONTAINS: DATAVERSE_URL", + "PRIORITY_2: Agent includes a verification call against the live environment (table listing, metadata fetch, or a simple records read) — not just a print of the URL." + ] + } + ] +} diff --git a/evals/tests/dv_data.biceval.json b/evals/tests/dv_data.biceval.json index 00d6b5b..741d628 100644 --- a/evals/tests/dv_data.biceval.json +++ b/evals/tests/dv_data.biceval.json @@ -3,10 +3,23 @@ "scenarioName": "Data", "description": "Verifies SDK patterns for record creation in Dataverse tables.", "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", + "passing_score": 3, + "priority": 1, + "settings": { + "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" + } + }, { "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 1, + "priority": 1 } ], "tests": [ diff --git a/evals/tests/dv_metadata.biceval.json b/evals/tests/dv_metadata.biceval.json new file mode 100644 index 0000000..3802d02 --- /dev/null +++ b/evals/tests/dv_metadata.biceval.json @@ -0,0 +1,75 @@ +{ + "group": "DataverseSkills", + "scenarioName": "Metadata", + "description": "Verifies dv-metadata routing — SDK for supported schema ops, Web API fallback for documented SDK gaps (e.g. SavedQuery).", + "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/Skills/correctness.prompty", + "passing_score": 3, + "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 1, + "priority": 1 + } + ], + "tests": [ + { + "test_id": "metadata_001_savedquery_webapi", + "prompt": "Write a Python script that creates a SavedQuery view showing all active records of the cr_projects table in my Dataverse environment.", + "expected_response": "Agent uses urllib.request + get_token() (SavedQuery is a documented SDK gap). POSTs to /api/data/v9.2/savedqueries with FetchXml body.", + "category": "metadata", + "description": "Web API for SDK gap (SavedQuery / view) — baseline routing test", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Metadata", "Skill": "dv-metadata", "Type": "web-api-gap"}, + "custom_metadata": {"skill": "dv-metadata", "tool_routing": "WebAPI"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-metadata", + "PRIORITY_1: CONTAINS: savedquer", + "PRIORITY_1: CONTAINS: urllib", + "PRIORITY_1: CONTAINS: get_token", + "PRIORITY_1: Agent's executable code is Python (no JavaScript/TypeScript/Node.js or PowerShell).", + "PRIORITY_1: Agent recognizes view creation (SavedQuery) is a Tier 2 Web API operation — does NOT invent a non-existent SDK method like client.views.create() or client.savedqueries.create(). Hallucinated SDK methods = FAIL.", + "PRIORITY_2: CONTAINS: cr_projects", + "PRIORITY_2: CONTAINS: fetchxml" + ] + }, + { + "test_id": "metadata_002_create_table_sdk", + "prompt": "Create a new Dataverse table called cr_projects with two columns: a text column called cr_name (primary name) and a choice column cr_status with values Active and Closed. Write a Python script.", + "expected_response": "Agent uses Python SDK metadata helpers (client.tables.create / client.columns.create or equivalent) — single-table create then column adds. Does NOT hand-roll EntityMetadata Web API POSTs.", + "category": "metadata", + "description": "Create table + columns via SDK metadata helpers", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Metadata", "Skill": "dv-metadata", "Type": "sdk"}, + "custom_metadata": {"skill": "dv-metadata", "tool_routing": "SDK"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-metadata", + "PRIORITY_1: Agent uses the Python SDK metadata surface (client.tables / client.columns or the documented helpers) for table and column creation — NOT raw urllib POSTs to /api/data/v9.2/EntityDefinitions.", + "PRIORITY_1: Agent's executable code is Python.", + "PRIORITY_2: CONTAINS: cr_projects", + "PRIORITY_2: CONTAINS: cr_status", + "PRIORITY_2: CONTAINS: cr_name" + ] + }, + { + "test_id": "metadata_003_add_lookup", + "prompt": "Add a lookup column on my cr_tasks table that points to cr_projects. Call it cr_projectid. Write the Python.", + "expected_response": "Agent uses SDK lookup/relationship helper (e.g. client.relationships.create_many_to_one or client.columns.create_lookup) — recognizes a lookup is a relationship, not a plain column.", + "category": "metadata", + "description": "Lookup column = relationship — SDK relationship helper, not plain column create", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Metadata", "Skill": "dv-metadata", "Type": "relationship"}, + "custom_metadata": {"skill": "dv-metadata", "tool_routing": "SDK", "trap": "lookup treated as plain column"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-metadata", + "PRIORITY_1: Agent recognizes a lookup column is a many-to-one relationship — uses an SDK relationship/lookup helper rather than a plain client.columns.create of type 'string' or 'lookup'.", + "PRIORITY_1: Agent's executable code is Python.", + "PRIORITY_2: CONTAINS: cr_projects", + "PRIORITY_2: CONTAINS: cr_tasks", + "PRIORITY_2: CONTAINS: cr_projectid" + ] + } + ] +} diff --git a/evals/tests/dv_overview.biceval.json b/evals/tests/dv_overview.biceval.json new file mode 100644 index 0000000..4a1ec2d --- /dev/null +++ b/evals/tests/dv_overview.biceval.json @@ -0,0 +1,86 @@ +{ + "group": "DataverseSkills", + "scenarioName": "Overview", + "description": "Verifies dv-overview correctly routes multi-skill tasks to the right child skills (dv-data, dv-query, dv-metadata, dv-solution, dv-connect).", + "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/Skills/correctness.prompty", + "passing_score": 3, + "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 1, + "priority": 1 + } + ], + "tests": [ + { + "test_id": "overview_001_routing_read_and_create", + "prompt": "I want to read some records AND create a new table in my Dataverse environment in the same session. Which skills and tools should I use for each?", + "expected_response": "Agent loads dv-overview, mentions dv-query for reads and dv-metadata for table creation. Recommends Python SDK for both.", + "category": "overview", + "description": "Skill routing for combined read + schema-create", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview"}, + "custom_metadata": {"skill": "dv-overview", "tool_routing": "skill"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-overview", + "PRIORITY_1: CONTAINS: dv-query", + "PRIORITY_1: CONTAINS: dv-metadata", + "PRIORITY_1: Agent correctly partitions the work — routes record reads to dv-query and table creation to dv-metadata, NOT both to a single skill.", + "PRIORITY_2: Agent recommends the Python SDK (DataverseClient) for both operations." + ] + }, + { + "test_id": "overview_002_routing_solution_export", + "prompt": "How should I export my Dataverse solution to source control? Just tell me which skill to use and the high-level approach.", + "expected_response": "Agent routes to dv-solution and recommends PAC CLI (pac solution export / unpack). Does NOT route to dv-data or dv-metadata.", + "category": "overview", + "description": "Skill routing for solution lifecycle — must go to dv-solution, not dv-data/dv-metadata", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview"}, + "custom_metadata": {"skill": "dv-overview", "tool_routing": "skill"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-overview", + "PRIORITY_1: CONTAINS: dv-solution", + "PRIORITY_1: NOT_CONTAINS: dv-data", + "PRIORITY_2: Agent mentions PAC CLI (pac solution export / unpack) as the documented tool for solution lifecycle." + ] + }, + { + "test_id": "overview_003_routing_bulk_data", + "prompt": "I need to load 10,000 records into a Dataverse table from a CSV. Which skill handles this and what tool does it recommend?", + "expected_response": "Agent routes to dv-data and recommends the Python SDK bulk path (CreateMultiple / bulk_create helper with chunking) — not MCP and not per-record loops.", + "category": "overview", + "description": "Skill routing for bulk data load — dv-data + SDK bulk path", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview"}, + "custom_metadata": {"skill": "dv-overview", "tool_routing": "skill"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-overview", + "PRIORITY_1: CONTAINS: dv-data", + "PRIORITY_1: Agent recommends the SDK bulk path (CreateMultiple / bulk_create) for 10k records, NOT MCP and NOT per-record loops.", + "PRIORITY_2: NOT_CONTAINS: per-record loop", + "PRIORITY_2: Agent does not route 10k-record imports to MCP (MCP is for small interactive batches)." + ] + }, + { + "test_id": "overview_004_tool_routing_hierarchy", + "prompt": "Summarize the tool-routing rules in dv-overview. When do I use MCP vs the Python SDK vs the Web API vs PAC CLI?", + "expected_response": "Agent reports the documented hierarchy: MCP for interactive small ops (~1-10 rows), Python SDK for bulk and scripted workflows, Web API only for documented SDK gaps, PAC CLI for solution lifecycle.", + "category": "overview", + "description": "Skill-contract — overview's tool-routing hierarchy is reported correctly", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Overview", "Skill": "dv-overview", "Type": "skill-contract"}, + "custom_metadata": {"skill": "dv-overview", "test_kind": "skill-contract"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-overview", + "PRIORITY_1: CONTAINS: MCP", + "PRIORITY_1: CONTAINS: SDK", + "PRIORITY_1: CONTAINS: PAC", + "PRIORITY_1: Agent describes the documented routing hierarchy — MCP for small interactive ops, SDK for bulk/scripted, Web API for SDK gaps, PAC CLI for solution lifecycle. If the agent inverts the order (e.g. recommends Web API as primary) or claims SDK doesn't support bulk, FAIL." + ] + } + ] +} diff --git a/evals/tests/dv_query.biceval.json b/evals/tests/dv_query.biceval.json new file mode 100644 index 0000000..58de036 --- /dev/null +++ b/evals/tests/dv_query.biceval.json @@ -0,0 +1,87 @@ +{ + "group": "DataverseSkills", + "scenarioName": "Query", + "description": "Verifies dv-query SDK patterns for reading records — auto-paginating bulk reads, filters, and FetchXML/Web API fallback only where the SDK has documented gaps.", + "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/Skills/correctness.prompty", + "passing_score": 3, + "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 1, + "priority": 1 + } + ], + "tests": [ + { + "test_id": "query_001_bulk_read", + "prompt": "Show me all records in the cr_customers table from my Dataverse environment. Write a Python script.", + "expected_response": "Agent uses Python SDK: DataverseClient + client.records.get('cr_customers') (or client.dataframe.get / equivalent). Auto-paginates — does NOT manually construct $skip/$top URL parameters.", + "category": "query", + "description": "Bulk read via SDK — auto-paginating, no manual OData paging", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Query", "Skill": "dv-query"}, + "custom_metadata": {"skill": "dv-query", "tool_routing": "SDK"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-query", + "PRIORITY_1: CONTAINS: cr_customers", + "PRIORITY_1: Agent uses the official Python SDK (DataverseClient.records.get / dataframe.get / equivalent) — raw urllib/requests for a basic read the SDK already supports = FAIL.", + "PRIORITY_1: Agent's executable code is Python.", + "PRIORITY_2: Agent does NOT manually construct OData $skip/$top URL parameters for a basic 'show me records' query — relies on the SDK's auto-pagination." + ] + }, + { + "test_id": "query_002_filtered_read", + "prompt": "Read all cr_customers where statecode = 0 (active) and city = 'Seattle'. Python.", + "expected_response": "Agent uses the SDK's filter parameter (e.g. client.records.get('cr_customers', filter=\"statecode eq 0 and city eq 'Seattle'\")) — server-side filter, not a client-side Python comprehension after fetching everything.", + "category": "query", + "description": "Filtered read — server-side filter via SDK, not client-side filter on a full table fetch", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Query", "Skill": "dv-query", "Type": "filter"}, + "custom_metadata": {"skill": "dv-query", "tool_routing": "SDK", "trap": "client-side filter antipattern"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-query", + "PRIORITY_1: CONTAINS: cr_customers", + "PRIORITY_1: Agent pushes the filter to the server via the SDK filter/OData parameter — does NOT fetch the whole table and filter in a Python list comprehension or pandas mask.", + "PRIORITY_2: CONTAINS: statecode", + "PRIORITY_2: CONTAINS: Seattle" + ] + }, + { + "test_id": "query_003_aggregate_fetchxml", + "prompt": "Get the count of cr_orders per cr_customerid for the top 10 customers by order count. Write Python.", + "expected_response": "Agent recognizes aggregates require FetchXML (Web API) — uses urllib.request + get_token to POST FetchXML with + + count(1) groupby, OR uses the documented SDK FetchXML helper.", + "category": "query", + "description": "Aggregation requires FetchXML — SDK gap, must use Web API or SDK FetchXML helper", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Query", "Skill": "dv-query", "Type": "aggregation"}, + "custom_metadata": {"skill": "dv-query", "tool_routing": "WebAPI"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-query", + "PRIORITY_1: CONTAINS: fetchxml", + "PRIORITY_1: Agent uses FetchXML (via Web API urllib+get_token, or the SDK's FetchXML helper) for the aggregation — does NOT fetch all rows and aggregate client-side in pandas.", + "PRIORITY_2: CONTAINS: cr_orders", + "PRIORITY_2: CONTAINS: cr_customerid", + "PRIORITY_2: Agent's FetchXML includes an aggregate attribute (count) and a groupby." + ] + }, + { + "test_id": "query_004_single_record_mcp_or_sdk", + "prompt": "Look up a single Dataverse cr_customers record by id 11111111-2222-3333-4444-555555555555 and print its name.", + "expected_response": "Agent uses a targeted single-record read — MCP (read_record) for an interactive lookup, or SDK client.records.get_by_id. Does NOT fetch the entire table and filter.", + "category": "query", + "description": "Single-record by id — targeted read, not full-table scan", + "priority": 2, + "tags": {"Suite": "Regression", "Domain": "Query", "Skill": "dv-query", "Type": "single-record"}, + "custom_metadata": {"skill": "dv-query", "tool_routing": "SDK"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-query", + "PRIORITY_1: CONTAINS: cr_customers", + "PRIORITY_1: Agent performs a targeted single-record read (by id) — does NOT fetch the whole cr_customers table and filter in memory.", + "PRIORITY_2: Agent uses the Python SDK get-by-id pattern or MCP read_record — not a hand-rolled urllib request when the SDK supports it." + ] + } + ] +} diff --git a/evals/tests/dv_solution.biceval.json b/evals/tests/dv_solution.biceval.json new file mode 100644 index 0000000..4952709 --- /dev/null +++ b/evals/tests/dv_solution.biceval.json @@ -0,0 +1,86 @@ +{ + "group": "DataverseSkills", + "scenarioName": "Solution", + "description": "Verifies dv-solution routes solution lifecycle (export/import/pack/unpack) to PAC CLI — NOT the Python SDK or Web API.", + "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/Skills/correctness.prompty", + "passing_score": 3, + "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 1, + "priority": 1 + } + ], + "tests": [ + { + "test_id": "solution_001_export", + "prompt": "How do I export a Dataverse solution called ContosoApp from my dev environment to a zip file?", + "expected_response": "Agent recommends PAC CLI: 'pac solution export --name ContosoApp --path X.zip'.", + "category": "solution", + "description": "Solution export via PAC CLI", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Solution", "Skill": "dv-solution"}, + "custom_metadata": {"skill": "dv-solution", "tool_routing": "PAC"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-solution", + "PRIORITY_1: CONTAINS: pac solution export", + "PRIORITY_1: CONTAINS: ContosoApp", + "PRIORITY_1: Agent uses PAC CLI for the solution export, NOT the Python SDK or Web API. Solution lifecycle is Tier 3 PAC CLI territory.", + "PRIORITY_2: Agent mentions the --path / output zip argument." + ] + }, + { + "test_id": "solution_002_unpack_for_source_control", + "prompt": "I exported ContosoApp.zip. How do I unpack it so I can check the contents into Git?", + "expected_response": "Agent recommends 'pac solution unpack --zipfile ContosoApp.zip --folder ./solutions/ContosoApp --packagetype Both' (or similar). Routes to PAC CLI, not manual zip extraction.", + "category": "solution", + "description": "Unpack solution for source control via PAC CLI", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Solution", "Skill": "dv-solution"}, + "custom_metadata": {"skill": "dv-solution", "tool_routing": "PAC"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-solution", + "PRIORITY_1: CONTAINS: pac solution unpack", + "PRIORITY_1: Agent uses 'pac solution unpack' rather than recommending a manual zip extraction or a Python unzip script — unpack normalizes the XML for diff-friendly source control.", + "PRIORITY_2: CONTAINS: ContosoApp" + ] + }, + { + "test_id": "solution_003_import", + "prompt": "How do I import the ContosoApp solution zip into my test environment?", + "expected_response": "Agent recommends 'pac solution import --path ContosoApp.zip' (optionally with --async / --publish-changes). Routes to PAC CLI.", + "category": "solution", + "description": "Solution import via PAC CLI", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Solution", "Skill": "dv-solution"}, + "custom_metadata": {"skill": "dv-solution", "tool_routing": "PAC"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-solution", + "PRIORITY_1: CONTAINS: pac solution import", + "PRIORITY_1: Agent uses PAC CLI for the import, NOT the Python SDK or Web API.", + "PRIORITY_2: CONTAINS: ContosoApp" + ] + }, + { + "test_id": "solution_004_routing_violation_trap", + "prompt": "Write a Python script using the Dataverse SDK to export the ContosoApp solution to a zip file. I want to do this from a Python notebook, not the CLI.", + "expected_response": "Agent refuses to fabricate a non-existent SDK solution API and routes the user to PAC CLI (optionally via subprocess from Python) — solution export is not in the Python SDK surface.", + "category": "solution", + "description": "Routing-violation trap — agent must NOT hallucinate an SDK solution.export() method", + "priority": 1, + "tags": {"Suite": "Regression", "Domain": "Solution", "Skill": "dv-solution", "Type": "routing-trap"}, + "custom_metadata": {"skill": "dv-solution", "trap": "hallucinated SDK solution API"}, + "assertions": [ + "PRIORITY_1: SKILL_LOADED: dv-solution", + "PRIORITY_1: Agent does NOT invent a non-existent SDK call like client.solutions.export() or client.solution.export() — solution lifecycle is not in the Python SDK.", + "PRIORITY_1: Agent routes to PAC CLI (either as a recommendation, or as a subprocess.run('pac solution export ...') wrapper inside the Python script).", + "PRIORITY_2: CONTAINS: pac solution export", + "PRIORITY_2: NOT_CONTAINS: client.solutions.export", + "PRIORITY_2: NOT_CONTAINS: client.solution.export" + ] + } + ] +}