From 0b194896668fcbeba77d87477bb46b638b080e59 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 14 Apr 2026 12:10:30 +0000 Subject: [PATCH 01/24] feat: add eval foundation types and schemas --- evals/lib/schemas.ts | 977 +++++++++++++++++++++++++++++++++++++++++++ evals/lib/types.ts | 504 ++++++++++++++++++++++ tsconfig.json | 7 +- 3 files changed, 1487 insertions(+), 1 deletion(-) create mode 100644 evals/lib/schemas.ts create mode 100644 evals/lib/types.ts diff --git a/evals/lib/schemas.ts b/evals/lib/schemas.ts new file mode 100644 index 00000000..9e663f0d --- /dev/null +++ b/evals/lib/schemas.ts @@ -0,0 +1,977 @@ +import { z } from 'zod'; + +import { EventRecordSchema } from '../../src/protocol/schemas.js'; +import type { ArtifactKind } from '../../src/tools/review-bundle.js'; +import type { BundleValidationProfile } from '../../src/tools/validate-bundle.js'; +import type { + AggregateMetrics, + AntiPatternFinding, + BundleCompletenessScore, + ComparisonMetrics, + DogfoodEvalCase, + EvalCase, + EvalCliOptions, + EvalCliResult, + EvalEventRecord, + EvalResult, + ExecutionEvalCase, + JsonReport, + MatrixEntry, + NormalizedProviderOutput, + PatternMatchResult, + PromptCaseScore, + PromptEvalCase, + ProviderAgentRequest, + ProviderAgentResult, + ProviderCapabilities, + ProviderComparisonReport, + ProviderConfig, + ProviderPromptRequest, + ProviderPromptResult, + ProviderRuntimeInfo, + ReportCompletenessScore, +} from './types.js'; + +const NonEmptyStringSchema = z.string().min(1); +const PositiveIntSchema = z.number().int().positive(); +const NonNegativeIntSchema = z.number().int().nonnegative(); +const NonNegativeNumberSchema = z.number().nonnegative(); +const FiniteNumberSchema = z.number(); +const IsoTimestampSchema = z.iso.datetime(); +const RegexPatternSchema = z.string().min(1).max(500); +const PathStringSchema = NonEmptyStringSchema; +const StringListSchema = z.array(NonEmptyStringSchema); +const UnitIntervalSchema = z.number().min(0).max(1); +const StringRecordSchema = z.record(NonEmptyStringSchema, z.string()); +const UnknownRecordSchema = z.record(NonEmptyStringSchema, z.unknown()); + +function addTimestampOrderIssue( + startedAt: string, + completedAt: string, + ctx: z.RefinementCtx, +): void { + if (Date.parse(completedAt) < Date.parse(startedAt)) { + ctx.addIssue({ + code: 'custom', + message: 'completedAt must be on or after startedAt', + path: ['completedAt'], + }); + } +} + +const PromptCategorySchema = z.enum([ + 'trigger', + 'selection', + 'workflow', + 'anti-pattern', +]); +const ExecutionCategorySchema = z.enum([ + 'session', + 'tui', + 'artifact', + 'recovery', +]); +const DogfoodCategorySchema = z.enum([ + 'qa', + 'release-readiness', + 'bug-repro', + 'reporting', +]); + +export const ExpectedSkillSchema = z.enum(['none', 'agent-tty', 'dogfood-tui']); +export const SkillConditionSchema = z.enum([ + 'none', + 'self-load', + 'preloaded', + 'stale', +]); +export const EvalLaneSchema = z.enum(['prompt', 'execution', 'dogfood']); +export const VerifierKindSchema = z.enum([ + 'snapshot', + 'screenshot', + 'event-log', + 'json', + 'bundle', + 'command', + 'custom', +]); +export const AntiPatternSeveritySchema = z.enum(['info', 'warning', 'error']); +export const ProviderModeSchema = z.enum(['stub', 'plan-only', 'agent-run']); +export const ArtifactKindSchema = z.enum([ + 'screenshot', + 'video', + 'recording', + 'json', + 'notes', + 'script', + 'support', + 'other', +]); +export const BundleValidationProfileSchema = z.enum([ + 'contract-reporting', + 'interactive-renderer', +]); + +const PromptBudgetSchema = z + .object({ + timeoutMs: PositiveIntSchema, + }) + .strict(); + +const ExecutionBudgetSchema = z + .object({ + timeoutMs: PositiveIntSchema, + maxAgentSteps: PositiveIntSchema.optional(), + maxWallClockMs: PositiveIntSchema.optional(), + }) + .strict(); + +const DogfoodBudgetSchema = z + .object({ + timeoutMs: PositiveIntSchema, + maxWallClockMs: PositiveIntSchema.optional(), + }) + .strict(); + +const BundleValidationCheckSchema = z + .object({ + name: NonEmptyStringSchema, + ok: z.boolean(), + message: NonEmptyStringSchema, + }) + .strict(); + +export const ScoreComponentSchema = z + .object({ + name: NonEmptyStringSchema, + score: NonNegativeNumberSchema, + maxScore: NonNegativeNumberSchema, + reason: NonEmptyStringSchema.optional(), + }) + .strict() + .superRefine((obj, ctx) => { + if (obj.score > obj.maxScore) { + ctx.addIssue({ + code: 'custom', + message: 'score must be less than or equal to maxScore', + path: ['score'], + }); + } + }); + +export const SetupStepSchema = z + .object({ + id: NonEmptyStringSchema, + description: NonEmptyStringSchema, + command: NonEmptyStringSchema, + argv: z.array(NonEmptyStringSchema), + cwd: PathStringSchema.optional(), + env: StringRecordSchema.optional(), + timeoutMs: PositiveIntSchema.optional(), + }) + .strict(); + +export const VerifierSpecSchema = z + .object({ + id: NonEmptyStringSchema, + kind: VerifierKindSchema, + description: NonEmptyStringSchema, + required: z.boolean(), + config: UnknownRecordSchema, + }) + .strict(); + +export const WorkflowCheckSchema = z + .object({ + id: NonEmptyStringSchema, + description: NonEmptyStringSchema, + required: z.boolean(), + requiredPatterns: z.array(RegexPatternSchema), + forbiddenPatterns: z.array(RegexPatternSchema), + dependsOn: z.array(NonEmptyStringSchema), + weight: NonNegativeNumberSchema.optional(), + }) + .strict(); + +export const AntiPatternRuleSchema = z + .object({ + id: NonEmptyStringSchema, + description: NonEmptyStringSchema, + severity: AntiPatternSeveritySchema, + patterns: z.array(RegexPatternSchema), + suggestedFix: NonEmptyStringSchema, + lanes: z.array(EvalLaneSchema).optional(), + }) + .strict(); + +export const ArtifactRequirementSchema = z + .object({ + kind: ArtifactKindSchema, + required: z.boolean(), + description: NonEmptyStringSchema, + minCount: PositiveIntSchema.optional(), + pathPatterns: z.array(RegexPatternSchema), + }) + .strict() + .superRefine((obj, ctx) => { + if (obj.required && obj.minCount === undefined) { + ctx.addIssue({ + code: 'custom', + message: 'minCount is required when required is true', + path: ['minCount'], + }); + } + }); + +export const ReportRequirementSchema = z + .object({ + id: NonEmptyStringSchema, + description: NonEmptyStringSchema, + required: z.boolean(), + section: NonEmptyStringSchema.optional(), + requiredPatterns: z.array(RegexPatternSchema), + forbiddenPatterns: z.array(RegexPatternSchema), + }) + .strict(); + +export const PatternMatchResultSchema = z + .object({ + pattern: RegexPatternSchema, + matched: z.boolean(), + matchedTexts: z.array(z.string()), + lineNumbers: z.array(NonNegativeIntSchema), + matchCount: NonNegativeIntSchema, + }) + .strict(); + +export const ForbiddenPatternResultSchema = z + .object({ + pattern: RegexPatternSchema, + violated: z.boolean(), + matchedTexts: z.array(z.string()), + lineNumbers: z.array(NonNegativeIntSchema), + matchCount: NonNegativeIntSchema, + }) + .strict(); + +export const AntiPatternFindingSchema = z + .object({ + ruleId: NonEmptyStringSchema, + severity: AntiPatternSeveritySchema, + message: NonEmptyStringSchema, + matchedText: z.string().optional(), + lineNumber: NonNegativeIntSchema.optional(), + suggestedFix: NonEmptyStringSchema.optional(), + }) + .strict(); + +export const WorkflowCheckResultSchema = z + .object({ + checkId: NonEmptyStringSchema, + passed: z.boolean(), + message: NonEmptyStringSchema.optional(), + matches: z.array(PatternMatchResultSchema), + forbiddenMatches: z.array(ForbiddenPatternResultSchema), + }) + .strict(); + +export const ScoreBreakdownSchema = z + .object({ + total: NonNegativeNumberSchema, + maxPossible: NonNegativeNumberSchema, + items: z.array(ScoreComponentSchema), + }) + .strict() + .superRefine((obj, ctx) => { + if (obj.maxPossible < obj.total) { + ctx.addIssue({ + code: 'custom', + message: 'maxPossible must be greater than or equal to total', + path: ['maxPossible'], + }); + } + }); + +export const BundleCompletenessScoreSchema = z + .object({ + profile: BundleValidationProfileSchema, + totalChecks: NonNegativeIntSchema, + passed: NonNegativeIntSchema, + failed: NonNegativeIntSchema, + score: UnitIntervalSchema, + details: z.array(BundleValidationCheckSchema), + }) + .strict() + .superRefine((obj, ctx) => { + if (obj.passed + obj.failed !== obj.totalChecks) { + ctx.addIssue({ + code: 'custom', + message: 'passed + failed must equal totalChecks', + path: ['totalChecks'], + }); + } + }); + +export const ReportCompletenessScoreSchema = z + .object({ + sectionsExpected: NonNegativeIntSchema, + sectionsFound: NonNegativeIntSchema, + score: UnitIntervalSchema, + missingSections: z.array(NonEmptyStringSchema), + matchedRequirements: z.array(PatternMatchResultSchema), + forbiddenFindings: z.array(ForbiddenPatternResultSchema), + }) + .strict() + .superRefine((obj, ctx) => { + if (obj.sectionsFound > obj.sectionsExpected) { + ctx.addIssue({ + code: 'custom', + message: 'sectionsFound must be less than or equal to sectionsExpected', + path: ['sectionsFound'], + }); + } + }); + +export const EvidenceQualityScoreSchema = z + .object({ + score: UnitIntervalSchema, + artifactCoverage: UnitIntervalSchema, + breakdown: ScoreBreakdownSchema, + bundleCompleteness: BundleCompletenessScoreSchema.optional(), + reportCompleteness: ReportCompletenessScoreSchema.optional(), + notes: z.array(NonEmptyStringSchema), + }) + .strict(); + +export const RunMetadataSchema = z + .object({ + runId: NonEmptyStringSchema, + createdAt: IsoTimestampSchema, + repoRoot: PathStringSchema, + providers: z.array(NonEmptyStringSchema), + models: z.array(NonEmptyStringSchema), + lanes: z.array(EvalLaneSchema), + conditions: z.array(SkillConditionSchema), + totalTrials: NonNegativeIntSchema, + notes: z.array(NonEmptyStringSchema), + }) + .strict(); + +export const MatrixEntrySchema = z + .object({ + providerId: NonEmptyStringSchema, + lane: EvalLaneSchema, + caseId: NonEmptyStringSchema, + category: NonEmptyStringSchema, + condition: SkillConditionSchema, + expectedSkill: ExpectedSkillSchema, + fixture: PathStringSchema.optional(), + target: PathStringSchema.optional(), + }) + .strict(); + +export const ComparisonMetricsSchema = z + .object({ + providerId: NonEmptyStringSchema, + lane: EvalLaneSchema, + groupKey: NonEmptyStringSchema, + caseIds: StringListSchema, + expectedSkill: ExpectedSkillSchema, + totalCompared: NonNegativeIntSchema, + category: NonEmptyStringSchema.optional(), + fixture: PathStringSchema.optional(), + target: PathStringSchema.optional(), + missingConditions: z.array(SkillConditionSchema), + realizedSkillLift: FiniteNumberSchema.optional(), + oracleSkillLift: FiniteNumberSchema.optional(), + routingGap: FiniteNumberSchema.optional(), + staleSkillHarm: FiniteNumberSchema.optional(), + regressionRate: FiniteNumberSchema.optional(), + unlockRate: FiniteNumberSchema.optional(), + routingEfficiency: FiniteNumberSchema.optional(), + }) + .strict(); + +export const PromptEvalCaseSchema = z + .object({ + id: NonEmptyStringSchema, + lane: z.literal('prompt'), + category: PromptCategorySchema, + prompt: NonEmptyStringSchema, + expectedSkill: ExpectedSkillSchema, + context: NonEmptyStringSchema.optional(), + expectedPatterns: z.array(RegexPatternSchema).min(1), + forbiddenPatterns: z.array(RegexPatternSchema), + rubric: z.array(NonEmptyStringSchema), + workflowChecks: z.array(WorkflowCheckSchema), + antiPatterns: z.array(AntiPatternRuleSchema), + budgets: PromptBudgetSchema, + }) + .strict(); + +export const ExecutionEvalCaseSchema = z + .object({ + id: NonEmptyStringSchema, + lane: z.literal('execution'), + category: ExecutionCategorySchema, + prompt: NonEmptyStringSchema, + expectedSkill: ExpectedSkillSchema, + fixture: PathStringSchema.optional(), + target: PathStringSchema.optional(), + conditions: z.array(SkillConditionSchema), + setup: z.array(SetupStepSchema), + verifiers: z.array(VerifierSpecSchema), + workflowChecks: z.array(WorkflowCheckSchema), + antiPatterns: z.array(AntiPatternRuleSchema), + artifactRequirements: z.array(ArtifactRequirementSchema), + budgets: ExecutionBudgetSchema, + }) + .strict() + .superRefine((obj, ctx) => { + if (obj.conditions.length === 0) { + ctx.addIssue({ + code: 'custom', + message: 'conditions must include at least one skill condition', + path: ['conditions'], + }); + } + + if (obj.verifiers.length === 0) { + ctx.addIssue({ + code: 'custom', + message: 'verifiers must include at least one verifier', + path: ['verifiers'], + }); + } + + if (obj.fixture === undefined && obj.target === undefined) { + ctx.addIssue({ + code: 'custom', + message: 'fixture or target is required', + path: ['fixture'], + }); + } + }); + +export const DogfoodEvalCaseSchema = z + .object({ + id: NonEmptyStringSchema, + lane: z.literal('dogfood'), + category: DogfoodCategorySchema, + prompt: NonEmptyStringSchema, + expectedSkill: ExpectedSkillSchema, + fixture: PathStringSchema.optional(), + target: PathStringSchema.optional(), + bundlePath: PathStringSchema, + bundleRequirements: z.array(NonEmptyStringSchema), + conditions: z.array(SkillConditionSchema), + validationProfile: BundleValidationProfileSchema, + artifactRequirements: z.array(ArtifactRequirementSchema), + reportRequirements: z.array(ReportRequirementSchema), + verifiers: z.array(VerifierSpecSchema), + workflowChecks: z.array(WorkflowCheckSchema), + antiPatterns: z.array(AntiPatternRuleSchema), + budgets: DogfoodBudgetSchema, + }) + .strict() + .superRefine((obj, ctx) => { + if (obj.conditions.length === 0) { + ctx.addIssue({ + code: 'custom', + message: 'conditions must include at least one skill condition', + path: ['conditions'], + }); + } + + if (obj.bundleRequirements.length === 0) { + ctx.addIssue({ + code: 'custom', + message: 'bundleRequirements must include at least one requirement', + path: ['bundleRequirements'], + }); + } + + if ( + obj.artifactRequirements.length === 0 && + obj.reportRequirements.length === 0 + ) { + ctx.addIssue({ + code: 'custom', + message: + 'artifactRequirements or reportRequirements must include at least one requirement', + path: ['artifactRequirements'], + }); + } + }); + +export const EvalCaseSchema = z.discriminatedUnion('lane', [ + PromptEvalCaseSchema, + ExecutionEvalCaseSchema, + DogfoodEvalCaseSchema, +]); + +const AgentEvalCaseSchema = z.discriminatedUnion('lane', [ + ExecutionEvalCaseSchema, + DogfoodEvalCaseSchema, +]); + +export const PromptCaseScoreSchema = z + .object({ + expectedSkillCorrect: z.boolean(), + patternMatches: z.array(PatternMatchResultSchema), + forbiddenPatternMatches: z.array(ForbiddenPatternResultSchema), + workflowChecks: z.array(WorkflowCheckResultSchema), + antiPatternFindings: z.array(AntiPatternFindingSchema), + breakdown: ScoreBreakdownSchema, + passed: z.boolean(), + }) + .strict(); + +export const ProviderCapabilitiesSchema = z + .object({ + supportsDetect: z.boolean(), + supportsPlanMode: z.boolean(), + supportsAgentMode: z.boolean(), + supportsStreaming: z.boolean(), + supportsToolCalls: z.boolean(), + supportsTranscriptCapture: z.boolean(), + }) + .strict(); + +export const ProviderRuntimeInfoSchema = z + .object({ + providerId: NonEmptyStringSchema, + available: z.boolean(), + detectedAt: IsoTimestampSchema, + version: NonEmptyStringSchema.optional(), + commandPath: PathStringSchema.optional(), + defaultModelId: NonEmptyStringSchema.optional(), + capabilities: ProviderCapabilitiesSchema, + notes: z.array(NonEmptyStringSchema), + }) + .strict(); + +export const NormalizedProviderOutputSchema = z + .object({ + finalText: z.string(), + reasoningText: z.string().optional(), + messages: z.array(z.string()), + referencedSkills: z.array(NonEmptyStringSchema), + selectedSkill: ExpectedSkillSchema.optional(), + toolCalls: z.array(UnknownRecordSchema), + }) + .strict(); + +export const EvalResultSchema = z + .object({ + runId: NonEmptyStringSchema, + providerId: NonEmptyStringSchema, + providerVersion: NonEmptyStringSchema.optional(), + modelId: NonEmptyStringSchema.optional(), + lane: EvalLaneSchema, + caseId: NonEmptyStringSchema, + category: NonEmptyStringSchema, + condition: SkillConditionSchema, + expectedSkill: ExpectedSkillSchema, + trial: NonNegativeIntSchema, + ok: z.boolean(), + score: ScoreBreakdownSchema, + promptScore: PromptCaseScoreSchema.optional(), + workflowChecks: z.array(WorkflowCheckResultSchema), + antiPatternFindings: z.array(AntiPatternFindingSchema), + bundleCompleteness: BundleCompletenessScoreSchema.optional(), + reportCompleteness: ReportCompletenessScoreSchema.optional(), + evidenceQuality: EvidenceQualityScoreSchema.optional(), + transcriptPath: PathStringSchema.optional(), + stdoutPath: PathStringSchema.optional(), + stderrPath: PathStringSchema.optional(), + artifactManifestPath: PathStringSchema.optional(), + bundlePath: PathStringSchema.optional(), + eventLogPath: PathStringSchema.optional(), + normalizedOutput: NormalizedProviderOutputSchema, + judgeScore: UnitIntervalSchema.optional(), + errorClass: NonEmptyStringSchema.optional(), + errorMessage: NonEmptyStringSchema.optional(), + startedAt: IsoTimestampSchema, + completedAt: IsoTimestampSchema, + durationMs: NonNegativeIntSchema, + }) + .strict() + .superRefine((obj, ctx) => { + addTimestampOrderIssue(obj.startedAt, obj.completedAt, ctx); + }); + +export const AggregateMetricsSchema = z + .object({ + totalCases: NonNegativeIntSchema, + passed: NonNegativeIntSchema, + failed: NonNegativeIntSchema, + passRate: UnitIntervalSchema, + averageScore: NonNegativeNumberSchema, + workflowComplianceRate: UnitIntervalSchema, + antiPatternIncidenceRate: UnitIntervalSchema, + bundleCompletenessRate: UnitIntervalSchema.optional(), + reportCompletenessRate: UnitIntervalSchema.optional(), + evidenceQualityRate: UnitIntervalSchema.optional(), + }) + .strict() + .superRefine((obj, ctx) => { + if (obj.passed + obj.failed !== obj.totalCases) { + ctx.addIssue({ + code: 'custom', + message: 'passed + failed must equal totalCases', + path: ['totalCases'], + }); + } + }); + +const ProviderAggregateSchema = z + .object({ + providerId: NonEmptyStringSchema, + aggregate: AggregateMetricsSchema, + comparisons: z.array(ComparisonMetricsSchema), + results: z.array(EvalResultSchema), + }) + .strict(); + +export const ProviderComparisonReportSchema = z + .object({ + metadata: RunMetadataSchema, + aggregate: AggregateMetricsSchema, + matrix: z.array(MatrixEntrySchema), + providers: z.array(ProviderAggregateSchema), + comparisons: z.array(ComparisonMetricsSchema), + }) + .strict(); + +export const JsonReportSchema = z + .object({ + metadata: RunMetadataSchema, + aggregate: AggregateMetricsSchema, + comparisons: z.array(ComparisonMetricsSchema), + results: z.array(EvalResultSchema), + providerComparison: ProviderComparisonReportSchema.optional(), + }) + .strict(); + +export const EvalCliOptionsSchema = z + .object({ + args: StringListSchema, + cwd: PathStringSchema.optional(), + env: StringRecordSchema.optional(), + home: PathStringSchema.optional(), + timeoutMs: NonNegativeIntSchema.optional(), + json: z.boolean(), + }) + .strict(); + +export const EvalCliResultSchema = z + .object({ + command: StringListSchema, + cwd: PathStringSchema, + exitCode: z.number().int().nullable(), + signal: z.string().nullable(), + ok: z.boolean(), + durationMs: NonNegativeIntSchema, + stdout: z.string(), + stderr: z.string(), + parsed: z.unknown().optional(), + }) + .strict(); + +export const EvalEventRecordSchema = EventRecordSchema; + +export const ProviderPromptRequestSchema = z + .object({ + runId: NonEmptyStringSchema, + providerId: NonEmptyStringSchema, + condition: SkillConditionSchema, + trial: NonNegativeIntSchema, + modelId: NonEmptyStringSchema.optional(), + cwd: PathStringSchema.optional(), + env: StringRecordSchema.optional(), + evalCase: PromptEvalCaseSchema, + }) + .strict(); + +export const ProviderPromptResultSchema = z + .object({ + request: ProviderPromptRequestSchema, + runtime: ProviderRuntimeInfoSchema, + ok: z.boolean(), + exitCode: z.number().int().nullable(), + signal: z.string().nullable(), + startedAt: IsoTimestampSchema, + completedAt: IsoTimestampSchema, + durationMs: NonNegativeIntSchema, + rawStdout: z.string(), + rawStderr: z.string(), + normalized: NormalizedProviderOutputSchema, + errorClass: NonEmptyStringSchema.optional(), + errorMessage: NonEmptyStringSchema.optional(), + }) + .strict() + .superRefine((obj, ctx) => { + addTimestampOrderIssue(obj.startedAt, obj.completedAt, ctx); + }); + +export const ProviderAgentRequestSchema = z + .object({ + runId: NonEmptyStringSchema, + providerId: NonEmptyStringSchema, + condition: SkillConditionSchema, + trial: NonNegativeIntSchema, + modelId: NonEmptyStringSchema.optional(), + cwd: PathStringSchema, + homeDir: PathStringSchema.optional(), + outputDir: PathStringSchema.optional(), + env: StringRecordSchema.optional(), + evalCase: AgentEvalCaseSchema, + }) + .strict(); + +export const ProviderAgentResultSchema = z + .object({ + request: ProviderAgentRequestSchema, + runtime: ProviderRuntimeInfoSchema, + ok: z.boolean(), + exitCode: z.number().int().nullable(), + signal: z.string().nullable(), + startedAt: IsoTimestampSchema, + completedAt: IsoTimestampSchema, + durationMs: NonNegativeIntSchema, + rawStdout: z.string(), + rawStderr: z.string(), + normalized: NormalizedProviderOutputSchema, + sessionId: NonEmptyStringSchema.optional(), + transcriptPath: PathStringSchema.optional(), + bundlePath: PathStringSchema.optional(), + eventLogPath: PathStringSchema.optional(), + errorClass: NonEmptyStringSchema.optional(), + errorMessage: NonEmptyStringSchema.optional(), + }) + .strict() + .superRefine((obj, ctx) => { + addTimestampOrderIssue(obj.startedAt, obj.completedAt, ctx); + }); + +export const ProviderConfigSchema = z + .object({ + providerId: NonEmptyStringSchema, + mode: ProviderModeSchema, + command: StringListSchema, + cwd: PathStringSchema.optional(), + env: StringRecordSchema.optional(), + defaultModelId: NonEmptyStringSchema.optional(), + timeoutMs: NonNegativeIntSchema.optional(), + capabilities: ProviderCapabilitiesSchema, + cannedOutputs: z + .record(NonEmptyStringSchema, NormalizedProviderOutputSchema) + .optional(), + }) + .strict(); + +export type ExpectedSkillSchemaType = z.infer; +export type SkillConditionSchemaType = z.infer; +export type EvalLaneSchemaType = z.infer; +export type VerifierKindSchemaType = z.infer; +export type AntiPatternSeveritySchemaType = z.infer< + typeof AntiPatternSeveritySchema +>; +export type ProviderModeSchemaType = z.infer; +export type ArtifactKindSchemaType = z.infer; +export type BundleValidationProfileSchemaType = z.infer< + typeof BundleValidationProfileSchema +>; +export type ScoreComponentSchemaType = z.infer; +export type SetupStepSchemaType = z.infer; +export type VerifierSpecSchemaType = z.infer; +export type WorkflowCheckSchemaType = z.infer; +export type AntiPatternRuleSchemaType = z.infer; +export type ArtifactRequirementSchemaType = z.infer< + typeof ArtifactRequirementSchema +>; +export type ReportRequirementSchemaType = z.infer< + typeof ReportRequirementSchema +>; +export type PatternMatchResultSchemaType = z.infer< + typeof PatternMatchResultSchema +>; +export type ForbiddenPatternResultSchemaType = z.infer< + typeof ForbiddenPatternResultSchema +>; +export type AntiPatternFindingSchemaType = z.infer< + typeof AntiPatternFindingSchema +>; +export type WorkflowCheckResultSchemaType = z.infer< + typeof WorkflowCheckResultSchema +>; +export type ScoreBreakdownSchemaType = z.infer; +export type BundleCompletenessScoreSchemaType = z.infer< + typeof BundleCompletenessScoreSchema +>; +export type ReportCompletenessScoreSchemaType = z.infer< + typeof ReportCompletenessScoreSchema +>; +export type EvidenceQualityScoreSchemaType = z.infer< + typeof EvidenceQualityScoreSchema +>; +export type RunMetadataSchemaType = z.infer; +export type MatrixEntrySchemaType = z.infer; +export type ComparisonMetricsSchemaType = z.infer< + typeof ComparisonMetricsSchema +>; +export type PromptEvalCaseSchemaType = z.infer; +export type ExecutionEvalCaseSchemaType = z.infer< + typeof ExecutionEvalCaseSchema +>; +export type DogfoodEvalCaseSchemaType = z.infer; +export type EvalCaseSchemaType = z.infer; +export type PromptCaseScoreSchemaType = z.infer; +export type ProviderCapabilitiesSchemaType = z.infer< + typeof ProviderCapabilitiesSchema +>; +export type ProviderRuntimeInfoSchemaType = z.infer< + typeof ProviderRuntimeInfoSchema +>; +export type NormalizedProviderOutputSchemaType = z.infer< + typeof NormalizedProviderOutputSchema +>; +export type EvalResultSchemaType = z.infer; +export type AggregateMetricsSchemaType = z.infer; +export type ProviderComparisonReportSchemaType = z.infer< + typeof ProviderComparisonReportSchema +>; +export type JsonReportSchemaType = z.infer; +export type EvalCliOptionsSchemaType = z.infer; +export type EvalCliResultSchemaType = z.infer; +export type EvalEventRecordSchemaType = z.infer; +export type ProviderPromptRequestSchemaType = z.infer< + typeof ProviderPromptRequestSchema +>; +export type ProviderPromptResultSchemaType = z.infer< + typeof ProviderPromptResultSchema +>; +export type ProviderAgentRequestSchemaType = z.infer< + typeof ProviderAgentRequestSchema +>; +export type ProviderAgentResultSchemaType = z.infer< + typeof ProviderAgentResultSchema +>; +export type ProviderConfigSchemaType = z.infer; + +type AssertExact = [T] extends [U] + ? [U] extends [T] + ? true + : never + : never; + +export type _ArtifactKindSchemaParity = AssertExact< + ArtifactKind, + ArtifactKindSchemaType +>; +export type _BundleValidationProfileSchemaParity = AssertExact< + BundleValidationProfile, + BundleValidationProfileSchemaType +>; +export type _PatternMatchResultSchemaParity = AssertExact< + PatternMatchResult, + PatternMatchResultSchemaType +>; +export type _AntiPatternFindingSchemaParity = AssertExact< + AntiPatternFinding, + AntiPatternFindingSchemaType +>; +export type _BundleCompletenessScoreSchemaParity = AssertExact< + BundleCompletenessScore, + BundleCompletenessScoreSchemaType +>; +export type _ReportCompletenessScoreSchemaParity = AssertExact< + ReportCompletenessScore, + ReportCompletenessScoreSchemaType +>; +export type _MatrixEntrySchemaParity = AssertExact< + MatrixEntry, + MatrixEntrySchemaType +>; +export type _ComparisonMetricsSchemaParity = AssertExact< + ComparisonMetrics, + ComparisonMetricsSchemaType +>; +export type _PromptEvalCaseSchemaParity = AssertExact< + PromptEvalCase, + PromptEvalCaseSchemaType +>; +export type _ExecutionEvalCaseSchemaParity = AssertExact< + ExecutionEvalCase, + ExecutionEvalCaseSchemaType +>; +export type _DogfoodEvalCaseSchemaParity = AssertExact< + DogfoodEvalCase, + DogfoodEvalCaseSchemaType +>; +export type _EvalCaseSchemaParity = AssertExact; +export type _PromptCaseScoreSchemaParity = AssertExact< + PromptCaseScore, + PromptCaseScoreSchemaType +>; +export type _EvalResultSchemaParity = AssertExact< + EvalResult, + EvalResultSchemaType +>; +export type _AggregateMetricsSchemaParity = AssertExact< + AggregateMetrics, + AggregateMetricsSchemaType +>; +export type _JsonReportSchemaParity = AssertExact< + JsonReport, + JsonReportSchemaType +>; +export type _ProviderComparisonReportSchemaParity = AssertExact< + ProviderComparisonReport, + ProviderComparisonReportSchemaType +>; +export type _EvalCliOptionsSchemaParity = AssertExact< + EvalCliOptions, + EvalCliOptionsSchemaType +>; +export type _EvalCliResultSchemaParity = AssertExact< + EvalCliResult, + EvalCliResultSchemaType +>; +export type _EvalEventRecordSchemaParity = AssertExact< + EvalEventRecord, + EvalEventRecordSchemaType +>; +export type _ProviderPromptRequestSchemaParity = AssertExact< + ProviderPromptRequest, + ProviderPromptRequestSchemaType +>; +export type _ProviderPromptResultSchemaParity = AssertExact< + ProviderPromptResult, + ProviderPromptResultSchemaType +>; +export type _ProviderAgentRequestSchemaParity = AssertExact< + ProviderAgentRequest, + ProviderAgentRequestSchemaType +>; +export type _ProviderAgentResultSchemaParity = AssertExact< + ProviderAgentResult, + ProviderAgentResultSchemaType +>; +export type _ProviderRuntimeInfoSchemaParity = AssertExact< + ProviderRuntimeInfo, + ProviderRuntimeInfoSchemaType +>; +export type _NormalizedProviderOutputSchemaParity = AssertExact< + NormalizedProviderOutput, + NormalizedProviderOutputSchemaType +>; +export type _ProviderCapabilitiesSchemaParity = AssertExact< + ProviderCapabilities, + ProviderCapabilitiesSchemaType +>; +export type _ProviderConfigSchemaParity = AssertExact< + ProviderConfig, + ProviderConfigSchemaType +>; diff --git a/evals/lib/types.ts b/evals/lib/types.ts new file mode 100644 index 00000000..26be766a --- /dev/null +++ b/evals/lib/types.ts @@ -0,0 +1,504 @@ +import type { EventRecord } from '../../src/protocol/schemas.js'; +import type { ArtifactKind } from '../../src/tools/review-bundle.js'; +import type { + BundleValidationCheck, + BundleValidationProfile, +} from '../../src/tools/validate-bundle.js'; + +/** Expected skill for an eval case. */ +export type ExpectedSkill = 'none' | 'agent-tty' | 'dogfood-tui'; + +/** Skill loading condition for an eval run. */ +export type SkillCondition = 'none' | 'self-load' | 'preloaded' | 'stale'; + +/** Eval lane identifier. */ +export type EvalLane = 'prompt' | 'execution' | 'dogfood'; + +/** Deterministic verifier kind. */ +export type VerifierKind = + | 'snapshot' + | 'screenshot' + | 'event-log' + | 'json' + | 'bundle' + | 'command' + | 'custom'; + +/** Severity for anti-pattern findings. */ +export type AntiPatternSeverity = 'info' | 'warning' | 'error'; + +/** Provider execution mode. */ +export type ProviderMode = 'stub' | 'plan-only' | 'agent-run'; + +/** Weighted score component for a breakdown. */ +export interface ScoreComponent { + name: string; + score: number; + maxScore: number; + reason?: string; +} + +/** Setup command that prepares an execution or dogfood case. */ +export interface SetupStep { + id: string; + description: string; + command: string; + argv: string[]; + cwd?: string; + env?: Record; + timeoutMs?: number; +} + +/** Deterministic verifier configuration for a case. */ +export interface VerifierSpec { + id: string; + kind: VerifierKind; + description: string; + required: boolean; + config: Record; +} + +/** Workflow rule used for compliance scoring. */ +export interface WorkflowCheck { + id: string; + description: string; + required: boolean; + requiredPatterns: string[]; + forbiddenPatterns: string[]; + dependsOn: string[]; + weight?: number; +} + +/** Anti-pattern rule used for transcript analysis. */ +export interface AntiPatternRule { + id: string; + description: string; + severity: AntiPatternSeverity; + patterns: string[]; + suggestedFix: string; + lanes?: EvalLane[]; +} + +/** Artifact requirement for a proof bundle or run output. */ +export interface ArtifactRequirement { + kind: ArtifactKind; + required: boolean; + description: string; + minCount?: number; + pathPatterns: string[]; +} + +/** Report requirement for dogfood notes or summaries. */ +export interface ReportRequirement { + id: string; + description: string; + required: boolean; + section?: string; + requiredPatterns: string[]; + forbiddenPatterns: string[]; +} + +/** Positive pattern match details for deterministic scoring. */ +export interface PatternMatchResult { + pattern: string; + matched: boolean; + matchedTexts: string[]; + lineNumbers: number[]; + matchCount: number; +} + +/** Forbidden pattern match details for deterministic scoring. */ +export interface ForbiddenPatternResult { + pattern: string; + violated: boolean; + matchedTexts: string[]; + lineNumbers: number[]; + matchCount: number; +} + +/** Anti-pattern detection finding emitted from a transcript scan. */ +export interface AntiPatternFinding { + ruleId: string; + severity: AntiPatternSeverity; + message: string; + matchedText?: string; + lineNumber?: number; + suggestedFix?: string; +} + +/** Workflow check result emitted during scoring. */ +export interface WorkflowCheckResult { + checkId: string; + passed: boolean; + message?: string; + matches: PatternMatchResult[]; + forbiddenMatches: ForbiddenPatternResult[]; +} + +/** Generic numeric breakdown for deterministic scoring. */ +export interface ScoreBreakdown { + total: number; + maxPossible: number; + items: ScoreComponent[]; +} + +/** Bundle completeness result derived from bundle validation checks. */ +export interface BundleCompletenessScore { + profile: BundleValidationProfile; + totalChecks: number; + passed: number; + failed: number; + score: number; + details: BundleValidationCheck[]; +} + +/** Report completeness result for dogfood notes and summaries. */ +export interface ReportCompletenessScore { + sectionsExpected: number; + sectionsFound: number; + score: number; + missingSections: string[]; + matchedRequirements: PatternMatchResult[]; + forbiddenFindings: ForbiddenPatternResult[]; +} + +/** Aggregate evidence-quality score for dogfood outputs. */ +export interface EvidenceQualityScore { + score: number; + artifactCoverage: number; + breakdown: ScoreBreakdown; + bundleCompleteness?: BundleCompletenessScore; + reportCompleteness?: ReportCompletenessScore; + notes: string[]; +} + +/** Metadata describing a whole eval report run. */ +export interface RunMetadata { + runId: string; + createdAt: string; + repoRoot: string; + providers: string[]; + models: string[]; + lanes: EvalLane[]; + conditions: SkillCondition[]; + totalTrials: number; + notes: string[]; +} + +/** Matrix entry representing one case/provider/condition combination. */ +export interface MatrixEntry { + providerId: string; + lane: EvalLane; + caseId: string; + category: string; + condition: SkillCondition; + expectedSkill: ExpectedSkill; + fixture?: string; + target?: string; +} + +/** Derived comparison metrics across skill conditions. */ +export interface ComparisonMetrics { + providerId: string; + lane: EvalLane; + groupKey: string; + caseIds: string[]; + expectedSkill: ExpectedSkill; + totalCompared: number; + category?: string; + fixture?: string; + target?: string; + missingConditions: SkillCondition[]; + realizedSkillLift?: number; + oracleSkillLift?: number; + routingGap?: number; + staleSkillHarm?: number; + regressionRate?: number; + unlockRate?: number; + routingEfficiency?: number; +} + +/** Prompt-only eval case for routing and planning checks. */ +export interface PromptEvalCase { + id: string; + lane: 'prompt'; + category: 'trigger' | 'selection' | 'workflow' | 'anti-pattern'; + prompt: string; + expectedSkill: ExpectedSkill; + context?: string; + expectedPatterns: string[]; + forbiddenPatterns: string[]; + rubric: string[]; + workflowChecks: WorkflowCheck[]; + antiPatterns: AntiPatternRule[]; + budgets: { + timeoutMs: number; + }; +} + +/** Closed-loop execution eval case for terminal workflows. */ +export interface ExecutionEvalCase { + id: string; + lane: 'execution'; + category: 'session' | 'tui' | 'artifact' | 'recovery'; + prompt: string; + expectedSkill: ExpectedSkill; + fixture?: string; + target?: string; + conditions: SkillCondition[]; + setup: SetupStep[]; + verifiers: VerifierSpec[]; + workflowChecks: WorkflowCheck[]; + antiPatterns: AntiPatternRule[]; + artifactRequirements: ArtifactRequirement[]; + budgets: { + timeoutMs: number; + maxAgentSteps?: number; + maxWallClockMs?: number; + }; +} + +/** Dogfood eval case for evidence capture and reporting quality. */ +export interface DogfoodEvalCase { + id: string; + lane: 'dogfood'; + category: 'qa' | 'release-readiness' | 'bug-repro' | 'reporting'; + prompt: string; + expectedSkill: ExpectedSkill; + fixture?: string; + target?: string; + bundlePath: string; + bundleRequirements: string[]; + conditions: SkillCondition[]; + validationProfile: BundleValidationProfile; + artifactRequirements: ArtifactRequirement[]; + reportRequirements: ReportRequirement[]; + verifiers: VerifierSpec[]; + workflowChecks: WorkflowCheck[]; + antiPatterns: AntiPatternRule[]; + budgets: { + timeoutMs: number; + maxWallClockMs?: number; + }; +} + +/** Any supported eval case. */ +export type EvalCase = PromptEvalCase | ExecutionEvalCase | DogfoodEvalCase; + +/** Detailed prompt-lane score for one case response. */ +export interface PromptCaseScore { + expectedSkillCorrect: boolean; + patternMatches: PatternMatchResult[]; + forbiddenPatternMatches: ForbiddenPatternResult[]; + workflowChecks: WorkflowCheckResult[]; + antiPatternFindings: AntiPatternFinding[]; + breakdown: ScoreBreakdown; + passed: boolean; +} + +/** One normalized eval result emitted by a lane runner. */ +export interface EvalResult { + runId: string; + providerId: string; + providerVersion?: string; + modelId?: string; + lane: EvalLane; + caseId: string; + category: string; + condition: SkillCondition; + expectedSkill: ExpectedSkill; + trial: number; + ok: boolean; + score: ScoreBreakdown; + promptScore?: PromptCaseScore; + workflowChecks: WorkflowCheckResult[]; + antiPatternFindings: AntiPatternFinding[]; + bundleCompleteness?: BundleCompletenessScore; + reportCompleteness?: ReportCompletenessScore; + evidenceQuality?: EvidenceQualityScore; + transcriptPath?: string; + stdoutPath?: string; + stderrPath?: string; + artifactManifestPath?: string; + bundlePath?: string; + eventLogPath?: string; + normalizedOutput: NormalizedProviderOutput; + judgeScore?: number; + errorClass?: string; + errorMessage?: string; + startedAt: string; + completedAt: string; + durationMs: number; +} + +/** Aggregate metrics for a set of eval results. */ +export interface AggregateMetrics { + totalCases: number; + passed: number; + failed: number; + passRate: number; + averageScore: number; + workflowComplianceRate: number; + antiPatternIncidenceRate: number; + bundleCompletenessRate?: number; + reportCompletenessRate?: number; + evidenceQualityRate?: number; +} + +/** JSON-serializable top-level eval report. */ +export interface JsonReport { + metadata: RunMetadata; + aggregate: AggregateMetrics; + comparisons: ComparisonMetrics[]; + results: EvalResult[]; + providerComparison?: ProviderComparisonReport; +} + +/** Cross-provider comparison view for an eval run. */ +export interface ProviderComparisonReport { + metadata: RunMetadata; + aggregate: AggregateMetrics; + matrix: MatrixEntry[]; + providers: Array<{ + providerId: string; + aggregate: AggregateMetrics; + comparisons: ComparisonMetrics[]; + results: EvalResult[]; + }>; + comparisons: ComparisonMetrics[]; +} + +/** Options for invoking the repo CLI from eval code. */ +export interface EvalCliOptions { + args: string[]; + cwd?: string; + env?: Record; + home?: string; + timeoutMs?: number; + json: boolean; +} + +/** Result envelope for a repo CLI invocation. */ +export interface EvalCliResult { + command: string[]; + cwd: string; + exitCode: number | null; + signal: string | null; + ok: boolean; + durationMs: number; + stdout: string; + stderr: string; + parsed?: unknown; +} + +/** Canonical event-log record used by eval helpers. */ +export type EvalEventRecord = EventRecord; + +/** Request payload for a provider plan-only eval invocation. */ +export interface ProviderPromptRequest { + runId: string; + providerId: string; + condition: SkillCondition; + trial: number; + modelId?: string; + cwd?: string; + env?: Record; + evalCase: PromptEvalCase; +} + +/** Normalized result from a provider plan-only eval invocation. */ +export interface ProviderPromptResult { + request: ProviderPromptRequest; + runtime: ProviderRuntimeInfo; + ok: boolean; + exitCode: number | null; + signal: string | null; + startedAt: string; + completedAt: string; + durationMs: number; + rawStdout: string; + rawStderr: string; + normalized: NormalizedProviderOutput; + errorClass?: string; + errorMessage?: string; +} + +/** Request payload for a provider agent-run eval invocation. */ +export interface ProviderAgentRequest { + runId: string; + providerId: string; + condition: SkillCondition; + trial: number; + modelId?: string; + cwd: string; + homeDir?: string; + outputDir?: string; + env?: Record; + evalCase: ExecutionEvalCase | DogfoodEvalCase; +} + +/** Normalized result from a provider agent-run eval invocation. */ +export interface ProviderAgentResult { + request: ProviderAgentRequest; + runtime: ProviderRuntimeInfo; + ok: boolean; + exitCode: number | null; + signal: string | null; + startedAt: string; + completedAt: string; + durationMs: number; + rawStdout: string; + rawStderr: string; + normalized: NormalizedProviderOutput; + sessionId?: string; + transcriptPath?: string; + bundlePath?: string; + eventLogPath?: string; + errorClass?: string; + errorMessage?: string; +} + +/** Detected runtime properties for an eval provider. */ +export interface ProviderRuntimeInfo { + providerId: string; + available: boolean; + detectedAt: string; + version?: string; + commandPath?: string; + defaultModelId?: string; + capabilities: ProviderCapabilities; + notes: string[]; +} + +/** Provider output normalized for downstream scoring and storage. */ +export interface NormalizedProviderOutput { + finalText: string; + reasoningText?: string; + messages: string[]; + referencedSkills: string[]; + selectedSkill?: ExpectedSkill; + toolCalls: Array>; +} + +/** Capability flags exposed by a provider adapter. */ +export interface ProviderCapabilities { + supportsDetect: boolean; + supportsPlanMode: boolean; + supportsAgentMode: boolean; + supportsStreaming: boolean; + supportsToolCalls: boolean; + supportsTranscriptCapture: boolean; +} + +/** Provider configuration used by the eval harness. */ +export interface ProviderConfig { + providerId: string; + mode: ProviderMode; + command: string[]; + cwd?: string; + env?: Record; + defaultModelId?: string; + timeoutMs?: number; + capabilities: ProviderCapabilities; + cannedOutputs?: Record; +} diff --git a/tsconfig.json b/tsconfig.json index ff007394..2c45212b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -20,6 +20,11 @@ "skipLibCheck": true, "forceConsistentCasingInFileNames": true }, - "include": ["src/**/*.ts", "test/**/*.ts", "vitest.config.ts"], + "include": [ + "src/**/*.ts", + "test/**/*.ts", + "evals/**/*.ts", + "vitest.config.ts" + ], "exclude": ["dist", "coverage", "node_modules"] } From b38010a605ac98a8a74bcce1d2f811484296426a Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 14 Apr 2026 12:22:06 +0000 Subject: [PATCH 02/24] feat(evals): add deterministic scoring utilities --- evals/lib/scoring.ts | 880 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 880 insertions(+) create mode 100644 evals/lib/scoring.ts diff --git a/evals/lib/scoring.ts b/evals/lib/scoring.ts new file mode 100644 index 00000000..0cebd4bd --- /dev/null +++ b/evals/lib/scoring.ts @@ -0,0 +1,880 @@ +import { assertString, invariant } from '../../src/util/assert.js'; +import type { + AggregateMetrics, + AntiPatternFinding, + AntiPatternRule, + EvalResult, + ExpectedSkill, + ForbiddenPatternResult, + PatternMatchResult, + PromptCaseScore, + PromptEvalCase, + ScoreComponent, + WorkflowCheck, + WorkflowCheckResult, +} from './types.js'; + +const REGEX_LITERAL_FLAGS_PATTERN = /^[dgimsuvy]*$/; +const COMPILED_PATTERN_CACHE = new Map(); +const FORBIDDEN_PATTERN_SEVERITY = 'error' as const; +const FORBIDDEN_PATTERN_PENALTY = 0.1; +const ANTI_PATTERN_PENALTY = 0.05; +const PROMPT_SCORE_WEIGHTS = Object.freeze({ + expectedPatterns: 0.4, + skillSelection: 0.4, + workflowCompliance: 0.2, +}); + +invariant( + Math.abs( + PROMPT_SCORE_WEIGHTS.expectedPatterns + + PROMPT_SCORE_WEIGHTS.skillSelection + + PROMPT_SCORE_WEIGHTS.workflowCompliance - + 1, + ) < 1e-9, + 'Prompt scoring weights must sum to 1', +); + +interface MatchOccurrence { + matchedText: string; + offset: number; + lineNumber: number; +} + +interface PatternMatchResultWithMetadata extends PatternMatchResult { + matchedText?: string; + offset?: number; + lineNumber?: number; + offsets: number[]; +} + +interface ForbiddenPatternResultWithMetadata extends ForbiddenPatternResult { + severity: typeof FORBIDDEN_PATTERN_SEVERITY; + matchedText?: string; + offset?: number; + lineNumber?: number; + offsets: number[]; +} + +interface AggregateMetricsWithStats extends AggregateMetrics { + medianScore: number; + minScore: number; + maxScore: number; +} + +interface WorkflowEvaluation { + check: WorkflowCheck; + matches: PatternMatchResult[]; + forbiddenMatches: ForbiddenPatternResult[]; +} + +interface ResolvedWorkflowStatus { + passed: boolean; + message?: string; +} + +/** + * Compile a cached regex from either raw source text or /pattern/flags form. + * + * Throws a descriptive error when the pattern cannot be compiled. + */ +export function compilePattern(source: string): RegExp { + assertString(source, 'Pattern source must be a string'); + invariant(source.length > 0, 'Pattern source must not be empty'); + + const cached = COMPILED_PATTERN_CACHE.get(source); + if (cached !== undefined) { + return cached; + } + + const { pattern, flags } = parsePatternSource(source); + + try { + const compiled = new RegExp(pattern, flags); + COMPILED_PATTERN_CACHE.set(source, compiled); + return compiled; + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`Invalid regex pattern "${source}": ${message}`, { + cause: error, + }); + } +} + +/** + * Match each expected pattern against text. + * + * Returned objects include aggregate arrays required by PatternMatchResult and + * also carry first-match debug metadata at runtime (`matchedText`, `offset`, + * `lineNumber`, and `offsets`). Offsets are 0-based; line numbers are 1-based. + */ +export function matchPatterns( + text: string, + patterns: string[], +): PatternMatchResult[] { + assertString(text, 'Text to match must be a string'); + invariant(Array.isArray(patterns), 'Patterns must be an array'); + + const lineStarts = computeLineStarts(text); + return patterns.map((pattern) => { + assertString(pattern, 'Each pattern must be a string'); + invariant(pattern.length > 0, 'Patterns must not be empty'); + const occurrences = collectOccurrences( + text, + compilePattern(pattern), + lineStarts, + ); + return buildPatternMatchResult(pattern, occurrences); + }); +} + +/** + * Match each forbidden pattern against text. + * + * Returned objects include aggregate arrays required by + * ForbiddenPatternResult and also carry runtime debug metadata + * (`severity`, `matchedText`, `offset`, `lineNumber`, and `offsets`). Offsets + * are 0-based; line numbers are 1-based. + */ +export function checkForbiddenPatterns( + text: string, + patterns: string[], +): ForbiddenPatternResult[] { + assertString(text, 'Text to scan must be a string'); + invariant(Array.isArray(patterns), 'Forbidden patterns must be an array'); + + const lineStarts = computeLineStarts(text); + return patterns.map((pattern) => { + assertString(pattern, 'Each forbidden pattern must be a string'); + invariant(pattern.length > 0, 'Forbidden patterns must not be empty'); + const occurrences = collectOccurrences( + text, + compilePattern(pattern), + lineStarts, + ); + return buildForbiddenPatternResult(pattern, occurrences); + }); +} + +/** + * Evaluate workflow checks against text, including required and forbidden + * pattern matching plus dependency handling. + */ +export function checkWorkflow( + text: string, + checks: WorkflowCheck[], +): WorkflowCheckResult[] { + assertString(text, 'Workflow text must be a string'); + invariant(Array.isArray(checks), 'Workflow checks must be an array'); + + const evaluations = new Map(); + for (const check of checks) { + assertString(check.id, 'Workflow check id must be a string'); + invariant( + !evaluations.has(check.id), + `Duplicate workflow check id: ${check.id}`, + ); + invariant( + Array.isArray(check.requiredPatterns), + `Workflow check ${check.id} requiredPatterns must be an array`, + ); + invariant( + Array.isArray(check.forbiddenPatterns), + `Workflow check ${check.id} forbiddenPatterns must be an array`, + ); + invariant( + Array.isArray(check.dependsOn), + `Workflow check ${check.id} dependsOn must be an array`, + ); + + evaluations.set(check.id, { + check, + matches: matchPatterns(text, check.requiredPatterns), + forbiddenMatches: checkForbiddenPatterns(text, check.forbiddenPatterns), + }); + } + + const resolvedStatuses = new Map(); + return checks.map((check) => { + const evaluation = evaluations.get(check.id); + invariant( + evaluation !== undefined, + `Missing workflow evaluation for ${check.id}`, + ); + const resolved = resolveWorkflowStatus( + check.id, + evaluations, + resolvedStatuses, + new Set(), + ); + + const baseResult = { + checkId: check.id, + passed: resolved.passed, + matches: evaluation.matches, + forbiddenMatches: evaluation.forbiddenMatches, + }; + + if (resolved.message === undefined) { + return baseResult; + } + + return { + ...baseResult, + message: resolved.message, + }; + }); +} + +/** + * Score one prompt-lane response deterministically from regex matches, + * inferred skill selection, workflow compliance, and anti-pattern findings. + */ +export function scorePromptCase( + response: string, + evalCase: PromptEvalCase, +): PromptCaseScore { + assertString(response, 'Prompt response must be a string'); + invariant( + Array.isArray(evalCase.expectedPatterns), + `Prompt eval case ${evalCase.id} expectedPatterns must be an array`, + ); + invariant( + Array.isArray(evalCase.forbiddenPatterns), + `Prompt eval case ${evalCase.id} forbiddenPatterns must be an array`, + ); + invariant( + Array.isArray(evalCase.workflowChecks), + `Prompt eval case ${evalCase.id} workflowChecks must be an array`, + ); + invariant( + Array.isArray(evalCase.antiPatterns), + `Prompt eval case ${evalCase.id} antiPatterns must be an array`, + ); + + const patternMatches = matchPatterns(response, evalCase.expectedPatterns); + const forbiddenPatternMatches = checkForbiddenPatterns( + response, + evalCase.forbiddenPatterns, + ); + const workflowChecks = checkWorkflow(response, evalCase.workflowChecks); + const antiPatternFindings = detectAntiPatternFindings( + response, + evalCase.antiPatterns, + ); + + const matchedExpectedPatterns = countMatchedPatterns(patternMatches); + const expectedPatternCoverage = safeDivide( + matchedExpectedPatterns, + patternMatches.length, + ); + const forbiddenViolationCount = countForbiddenViolations( + forbiddenPatternMatches, + ); + const forbiddenPenalty = FORBIDDEN_PATTERN_PENALTY * forbiddenViolationCount; + const inferredSkill = inferSelectedSkill(response); + const expectedSkillCorrect = inferredSkill === evalCase.expectedSkill; + const skillScore = expectedSkillCorrect ? 1 : 0; + const requiredChecks = evalCase.workflowChecks.filter( + (check) => check.required, + ); + const passedRequiredChecks = workflowChecks.filter((result) => { + const workflowCheck = requiredChecks.find( + (check) => check.id === result.checkId, + ); + return workflowCheck !== undefined && result.passed; + }).length; + const workflowScore = safeDivide(passedRequiredChecks, requiredChecks.length); + const antiPatternPenalty = ANTI_PATTERN_PENALTY * antiPatternFindings.length; + + const positiveScore = clampUnitInterval( + expectedPatternCoverage * PROMPT_SCORE_WEIGHTS.expectedPatterns + + skillScore * PROMPT_SCORE_WEIGHTS.skillSelection + + workflowScore * PROMPT_SCORE_WEIGHTS.workflowCompliance, + ); + const totalScore = clampUnitInterval( + positiveScore - forbiddenPenalty - antiPatternPenalty, + ); + + const breakdownItems: ScoreComponent[] = [ + { + name: 'expected-pattern-coverage', + score: expectedPatternCoverage * PROMPT_SCORE_WEIGHTS.expectedPatterns, + maxScore: PROMPT_SCORE_WEIGHTS.expectedPatterns, + reason: `${String(matchedExpectedPatterns)}/${String(patternMatches.length)} expected patterns matched`, + }, + { + name: 'skill-selection-correctness', + score: skillScore * PROMPT_SCORE_WEIGHTS.skillSelection, + maxScore: PROMPT_SCORE_WEIGHTS.skillSelection, + reason: expectedSkillCorrect + ? `Inferred expected skill "${evalCase.expectedSkill}"` + : `Expected skill "${evalCase.expectedSkill}", inferred "${inferredSkill}"`, + }, + { + name: 'workflow-compliance', + score: workflowScore * PROMPT_SCORE_WEIGHTS.workflowCompliance, + maxScore: PROMPT_SCORE_WEIGHTS.workflowCompliance, + reason: `${String(passedRequiredChecks)}/${String(requiredChecks.length)} required workflow checks passed`, + }, + { + name: 'forbidden-pattern-penalty', + score: 0, + maxScore: 0, + reason: + forbiddenViolationCount === 0 + ? 'No forbidden pattern violations' + : `Penalty -${forbiddenPenalty.toFixed(2)} for ${String(forbiddenViolationCount)} forbidden matches`, + }, + { + name: 'anti-pattern-penalty', + score: 0, + maxScore: 0, + reason: + antiPatternFindings.length === 0 + ? 'No anti-pattern findings' + : `Penalty -${antiPatternPenalty.toFixed(2)} for ${String(antiPatternFindings.length)} anti-pattern findings`, + }, + ]; + + const passed = + patternMatches.every((match) => match.matched) && + forbiddenViolationCount === 0 && + expectedSkillCorrect && + requiredChecks.every((check) => + workflowChecks.some( + (result) => result.checkId === check.id && result.passed, + ), + ) && + antiPatternFindings.length === 0; + + return { + expectedSkillCorrect, + patternMatches, + forbiddenPatternMatches, + workflowChecks, + antiPatternFindings, + breakdown: { + total: totalScore, + maxPossible: 1, + items: breakdownItems, + }, + passed, + }; +} + +/** + * Compute aggregate metrics for a batch of eval results. + * + * The returned object always uses 0 for empty-input and zero-denominator cases. + * In addition to the AggregateMetrics contract, runtime results also include + * `medianScore`, `minScore`, and `maxScore` for reporting convenience. + */ +export function computeAggregateMetrics( + results: EvalResult[], +): AggregateMetrics { + invariant(Array.isArray(results), 'Eval results must be an array'); + + const totalCases = results.length; + const passed = results.filter((result) => result.ok).length; + const failed = totalCases - passed; + const normalizedScores = results.map(normalizeScoreBreakdown); + const bundleScores = results + .map((result) => result.bundleCompleteness?.score) + .filter((score): score is number => score !== undefined); + const reportScores = results + .map((result) => result.reportCompleteness?.score) + .filter((score): score is number => score !== undefined); + const evidenceScores = results + .map((result) => result.evidenceQuality?.score) + .filter((score): score is number => score !== undefined); + const totalWorkflowChecks = results.reduce( + (count, result) => count + result.workflowChecks.length, + 0, + ); + const passedWorkflowChecks = results.reduce( + (count, result) => + count + + result.workflowChecks.filter((workflow) => workflow.passed).length, + 0, + ); + const resultsWithAntiPatterns = results.filter( + (result) => result.antiPatternFindings.length > 0, + ).length; + + const aggregate: AggregateMetricsWithStats = { + totalCases, + passed, + failed, + passRate: safeDivide(passed, totalCases), + averageScore: average(normalizedScores), + medianScore: median(normalizedScores), + minScore: min(normalizedScores), + maxScore: max(normalizedScores), + workflowComplianceRate: safeDivide( + passedWorkflowChecks, + totalWorkflowChecks, + ), + antiPatternIncidenceRate: safeDivide(resultsWithAntiPatterns, totalCases), + bundleCompletenessRate: average(bundleScores), + reportCompletenessRate: average(reportScores), + evidenceQualityRate: average(evidenceScores), + }; + + return aggregate; +} + +/** + * Compute precision, recall, and F1 deterministically from expected/actual + * boolean classifications. Zero-denominator cases return 0. + */ +export function computePrecisionRecall( + results: Array<{ expected: boolean; actual: boolean }>, +): { precision: number; recall: number; f1: number } { + invariant( + Array.isArray(results), + 'Precision/recall results must be an array', + ); + + let truePositives = 0; + let falsePositives = 0; + let falseNegatives = 0; + + for (const result of results) { + invariant( + typeof result.expected === 'boolean', + 'Precision/recall result.expected must be a boolean', + ); + invariant( + typeof result.actual === 'boolean', + 'Precision/recall result.actual must be a boolean', + ); + + if (result.expected && result.actual) { + truePositives += 1; + } else if (!result.expected && result.actual) { + falsePositives += 1; + } else if (result.expected && !result.actual) { + falseNegatives += 1; + } + } + + const precision = safeDivide(truePositives, truePositives + falsePositives); + const recall = safeDivide(truePositives, truePositives + falseNegatives); + const f1 = + precision === 0 || recall === 0 + ? 0 + : safeDivide(2 * precision * recall, precision + recall); + + return { + precision, + recall, + f1, + }; +} + +function parsePatternSource(source: string): { + pattern: string; + flags: string; +} { + const literalPattern = tryParseRegexLiteral(source); + if (literalPattern !== null) { + return literalPattern; + } + + return { + pattern: source, + flags: '', + }; +} + +function tryParseRegexLiteral( + source: string, +): { pattern: string; flags: string } | null { + if (!source.startsWith('/') || source.length < 2) { + return null; + } + + let escaped = false; + let inCharacterClass = false; + + for (let index = 1; index < source.length; index += 1) { + const current = source[index]; + invariant(current !== undefined, 'Regex literal parser exceeded bounds'); + + if (escaped) { + escaped = false; + continue; + } + + if (current === '\\') { + escaped = true; + continue; + } + + if (current === '[') { + inCharacterClass = true; + continue; + } + + if (current === ']' && inCharacterClass) { + inCharacterClass = false; + continue; + } + + if (current === '/' && !inCharacterClass) { + const flags = source.slice(index + 1); + if (flags.length === 0) { + return { + pattern: source.slice(1, index), + flags, + }; + } + + if (!REGEX_LITERAL_FLAGS_PATTERN.test(flags)) { + return null; + } + + return { + pattern: source.slice(1, index), + flags, + }; + } + } + + return null; +} + +function computeLineStarts(text: string): number[] { + const lineStarts = [0]; + for (let index = 0; index < text.length; index += 1) { + if (text.charCodeAt(index) === 10) { + lineStarts.push(index + 1); + } + } + return lineStarts; +} + +function lineNumberFromOffset(offset: number, lineStarts: number[]): number { + invariant(offset >= 0, 'Match offset must be non-negative'); + invariant(lineStarts.length > 0, 'Line starts must not be empty'); + + let low = 0; + let high = lineStarts.length - 1; + + while (low <= high) { + const mid = low + Math.floor((high - low) / 2); + const start = lineStarts[mid]; + invariant(start !== undefined, 'Line start index must be defined'); + if (start <= offset) { + low = mid + 1; + } else { + high = mid - 1; + } + } + + return high + 1; +} + +function collectOccurrences( + text: string, + pattern: RegExp, + lineStarts: number[], +): MatchOccurrence[] { + const matcher = createGlobalMatcher(pattern); + + return Array.from(text.matchAll(matcher), (match): MatchOccurrence => { + const matchedText = match[0]; + assertString(matchedText, 'Regex match must include matched text'); + const offset = match.index; + return { + matchedText, + offset, + lineNumber: lineNumberFromOffset(offset, lineStarts), + }; + }); +} + +function createGlobalMatcher(pattern: RegExp): RegExp { + const flagsWithoutSticky = pattern.flags.replaceAll('y', ''); + const globalFlags = flagsWithoutSticky.includes('g') + ? flagsWithoutSticky + : `${flagsWithoutSticky}g`; + return new RegExp(pattern.source, globalFlags); +} + +function buildPatternMatchResult( + pattern: string, + occurrences: MatchOccurrence[], +): PatternMatchResultWithMetadata { + const baseResult = { + pattern, + matched: occurrences.length > 0, + matchedTexts: occurrences.map((occurrence) => occurrence.matchedText), + lineNumbers: occurrences.map((occurrence) => occurrence.lineNumber), + offsets: occurrences.map((occurrence) => occurrence.offset), + matchCount: occurrences.length, + }; + const firstOccurrence = occurrences[0]; + if (firstOccurrence === undefined) { + return baseResult; + } + + return { + ...baseResult, + matchedText: firstOccurrence.matchedText, + offset: firstOccurrence.offset, + lineNumber: firstOccurrence.lineNumber, + }; +} + +function buildForbiddenPatternResult( + pattern: string, + occurrences: MatchOccurrence[], +): ForbiddenPatternResultWithMetadata { + const baseResult = { + pattern, + violated: occurrences.length > 0, + matchedTexts: occurrences.map((occurrence) => occurrence.matchedText), + lineNumbers: occurrences.map((occurrence) => occurrence.lineNumber), + offsets: occurrences.map((occurrence) => occurrence.offset), + matchCount: occurrences.length, + severity: FORBIDDEN_PATTERN_SEVERITY, + }; + const firstOccurrence = occurrences[0]; + if (firstOccurrence === undefined) { + return baseResult; + } + + return { + ...baseResult, + matchedText: firstOccurrence.matchedText, + offset: firstOccurrence.offset, + lineNumber: firstOccurrence.lineNumber, + }; +} + +function resolveWorkflowStatus( + checkId: string, + evaluations: Map, + cache: Map, + activeCheckIds: Set, +): ResolvedWorkflowStatus { + const cached = cache.get(checkId); + if (cached !== undefined) { + return cached; + } + + const evaluation = evaluations.get(checkId); + invariant(evaluation !== undefined, `Unknown workflow check id: ${checkId}`); + + if (activeCheckIds.has(checkId)) { + return { + passed: false, + message: `Cyclic workflow dependency detected at ${checkId}`, + }; + } + + activeCheckIds.add(checkId); + + const messageParts: string[] = []; + for (const dependencyId of evaluation.check.dependsOn) { + const dependency = evaluations.get(dependencyId); + if (dependency === undefined) { + messageParts.push(`Missing dependency "${dependencyId}"`); + continue; + } + + const dependencyStatus = resolveWorkflowStatus( + dependencyId, + evaluations, + cache, + activeCheckIds, + ); + if (!dependencyStatus.passed) { + messageParts.push(`Dependency "${dependencyId}" did not pass`); + } + } + + activeCheckIds.delete(checkId); + + const missingRequiredPatterns = evaluation.matches + .filter((result) => !result.matched) + .map((result) => result.pattern); + if (missingRequiredPatterns.length > 0) { + messageParts.push( + `Missing required patterns: ${missingRequiredPatterns.join(', ')}`, + ); + } + + const violatedForbiddenPatterns = evaluation.forbiddenMatches + .filter((result) => result.violated) + .map((result) => result.pattern); + if (violatedForbiddenPatterns.length > 0) { + messageParts.push( + `Matched forbidden patterns: ${violatedForbiddenPatterns.join(', ')}`, + ); + } + + const resolved: ResolvedWorkflowStatus = { + passed: messageParts.length === 0, + }; + if (messageParts.length > 0) { + resolved.message = messageParts.join('; '); + } + + cache.set(checkId, resolved); + return resolved; +} + +function detectAntiPatternFindings( + text: string, + rules: AntiPatternRule[], +): AntiPatternFinding[] { + invariant(Array.isArray(rules), 'Anti-pattern rules must be an array'); + + const lineStarts = computeLineStarts(text); + const findings: AntiPatternFinding[] = []; + + for (const rule of rules) { + assertString(rule.id, 'Anti-pattern rule id must be a string'); + invariant( + Array.isArray(rule.patterns), + `Anti-pattern rule ${rule.id} patterns must be an array`, + ); + + for (const pattern of rule.patterns) { + assertString(pattern, 'Anti-pattern pattern must be a string'); + const occurrences = collectOccurrences( + text, + compilePattern(pattern), + lineStarts, + ); + + for (const occurrence of occurrences) { + findings.push({ + ruleId: rule.id, + severity: rule.severity, + message: rule.description, + matchedText: occurrence.matchedText, + lineNumber: occurrence.lineNumber, + suggestedFix: rule.suggestedFix, + }); + } + } + } + + return findings; +} + +function inferSelectedSkill(response: string): ExpectedSkill { + if (matchesAny(response, ['\\bdogfood-tui\\b'])) { + return 'dogfood-tui'; + } + + if ( + matchesAny(response, [ + '\\bagent-tty\\s+skills\\s+get\\s+agent-tty\\b', + '\\bagent-tty\\s+--', + '\\bagent-tty\\b', + ]) + ) { + return 'agent-tty'; + } + + return 'none'; +} + +function matchesAny(text: string, patterns: string[]): boolean { + return matchPatterns(text, patterns).some((result) => result.matched); +} + +function countMatchedPatterns(results: PatternMatchResult[]): number { + return results.filter((result) => result.matched).length; +} + +function countForbiddenViolations(results: ForbiddenPatternResult[]): number { + return results.reduce((count, result) => count + result.matchCount, 0); +} + +function normalizeScoreBreakdown(result: EvalResult): number { + return clampUnitInterval( + safeDivide(result.score.total, result.score.maxPossible), + ); +} + +function average(values: number[]): number { + if (values.length === 0) { + return 0; + } + + return safeDivide( + values.reduce((sum, value) => sum + value, 0), + values.length, + ); +} + +function median(values: number[]): number { + if (values.length === 0) { + return 0; + } + + const sorted = [...values].sort((left, right) => left - right); + const middle = Math.floor(sorted.length / 2); + const middleValue = sorted[middle]; + invariant(middleValue !== undefined, 'Median index must be in bounds'); + + if (sorted.length % 2 === 1) { + return middleValue; + } + + const previousValue = sorted[middle - 1]; + invariant( + previousValue !== undefined, + 'Median lower index must be in bounds', + ); + return (previousValue + middleValue) / 2; +} + +function min(values: number[]): number { + if (values.length === 0) { + return 0; + } + + return values.reduce((currentMin, value) => + value < currentMin ? value : currentMin, + ); +} + +function max(values: number[]): number { + if (values.length === 0) { + return 0; + } + + return values.reduce((currentMax, value) => + value > currentMax ? value : currentMax, + ); +} + +function safeDivide(numerator: number, denominator: number): number { + invariant(Number.isFinite(numerator), 'Numerator must be finite'); + invariant(Number.isFinite(denominator), 'Denominator must be finite'); + if (denominator <= 0) { + return 0; + } + + return numerator / denominator; +} + +function clampUnitInterval(value: number): number { + invariant(Number.isFinite(value), 'Score values must be finite'); + if (value <= 0) { + return 0; + } + + if (value >= 1) { + return 1; + } + + return value; +} From da2030f2a8508a569f90e234d8820c0da848e54d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Tue, 14 Apr 2026 12:20:47 +0000 Subject: [PATCH 03/24] feat(evals): add terminal anti-pattern detection engine --- evals/lib/antiPatterns.ts | 646 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 646 insertions(+) create mode 100644 evals/lib/antiPatterns.ts diff --git a/evals/lib/antiPatterns.ts b/evals/lib/antiPatterns.ts new file mode 100644 index 00000000..817f9a0d --- /dev/null +++ b/evals/lib/antiPatterns.ts @@ -0,0 +1,646 @@ +import { assertString, invariant } from '../../src/util/assert.js'; +import type { + AntiPatternFinding, + AntiPatternRule, + AntiPatternSeverity, +} from './types.js'; + +const SEVERITY_ORDER: Record = { + info: 0, + warning: 1, + error: 2, +}; + +const COMMENT_ONLY_LINE_PATTERN = /^\s*(?:#|\/\/|\/\*|\*|\*\/|