diff --git a/.azdo/DataversePluginEvals_PR.yml b/.azdo/DataversePluginEvals_PR.yml index 01fb4bc..2b8d5da 100644 --- a/.azdo/DataversePluginEvals_PR.yml +++ b/.azdo/DataversePluginEvals_PR.yml @@ -56,7 +56,10 @@ resources: type: github endpoint: bic-proxima-shared-stamp name: bic/LocalEvalRunner - ref: refs/heads/main + # Temporary: pinned to LER PR #416 branch to validate the + # DeterministicAssertionEvaluator service-only routing fix end-to-end. + # Revert to refs/heads/main before merge. + ref: refs/heads/users/sbadenkal/service-only-deterministic-assertion - repository: BicEvalsService type: github endpoint: bic-proxima-shared-stamp @@ -67,15 +70,21 @@ steps: - checkout: self displayName: 'Checkout Dataverse-skills' +- checkout: LocalEvalRunner + displayName: 'Checkout LocalEvalRunner source (for build-from-source)' + fetchDepth: 1 + - checkout: BicEvalsService displayName: 'Checkout evaluator prompts' fetchDepth: 1 -# Setup: .NET, NuGet auth, LocalEvalRunner install, Copilot CLI, auth token +# Setup: .NET, NuGet auth, LocalEvalRunner BUILD FROM SOURCE, Copilot CLI, auth token - template: .pipelines/templates/steps/BicEval-SetupTemplate.yml@LocalEvalRunner parameters: cliAgent: 'copilot' cliAgentTokenVariable: 'COPILOT_GH_PAT' + useBuildFromSource: true + sourceRepoPath: '$(Build.SourcesDirectory)/LocalEvalRunner' # Configure: merge replacements, prepare agent config - template: .pipelines/templates/steps/BicEval-ConfigureTemplate.yml@LocalEvalRunner diff --git a/evals/tests/dv_data.biceval.json b/evals/tests/dv_data.biceval.json index 00d6b5b..741d628 100644 --- a/evals/tests/dv_data.biceval.json +++ b/evals/tests/dv_data.biceval.json @@ -3,10 +3,23 @@ "scenarioName": "Data", "description": "Verifies SDK patterns for record creation in Dataverse tables.", "enabled_evaluators": [ + { + "name": "CortexConfigurations:Common/DeterministicAssertionEvaluator", + "passing_score": 3, + "priority": 1, + "settings": { + "supported_verbs": "CONTAINS,NOT_CONTAINS,SKILL_LOADED" + } + }, { "name": "CortexConfigurations:Common/Skills/correctness.prompty", "passing_score": 3, "priority": 1 + }, + { + "name": "CortexConfigurations:Common/SEVAL/LMChecklist.prompty", + "passing_score": 1, + "priority": 1 } ], "tests": [