From 8f214137427cb0a76058945b6336f834eda8f9c1 Mon Sep 17 00:00:00 2001 From: Wu Sheng Date: Fri, 14 Aug 2026 12:57:57 +0800 Subject: [PATCH] genai: typed condition fields and source-aware span addressing Follow-up to #163, from reviewing the OAP implementation in apache/skywalking#13943. No OAP submodule pointer references #163 yet, so these changes are free of client impact. - Remove `tags` and `GenAIEvaluationRecordTag`. A tag condition exists to filter key-values the protocol cannot enumerate; this record persists no user-supplied attribute. `Log` returns `tags` and `LogQueryCondition` filters them, whereas `GenAIEvaluationRecord` has no `tags` field at all, so the condition filtered something the API never returns. - Add `GenAITraceRef` / `GenAITraceRefType`. GenAI evaluation accepts native, OTLP and Zipkin traces. Native span ids are a segment-local int index; OTLP and Zipkin ids are 16 hex characters. `spanIndex` is named for what it is - an index, not an identifier. Scoped to GenAI: `spanId: Int` elsewhere is correctly scoped to native addressing and is unchanged. - Split the value slots into `scoreValue` / `booleanValue` / `stringValue` so `valueType` genuinely discriminates. BOOLEAN previously shared `scoreValue`, so score-range filters spanned both types. - Drop `spanType`: `SpanEvaluationType` has one value, so there is nothing to query and nothing to display. - Document entity scope and layer on the id fields - provider is a service and model is its instance, both VIRTUAL_GENAI, while `serviceId` is a normal agent-detected service. Scores remain `Long` on the ppm scale; SkyWalking does not carry Float/Double in stored or transported values. Co-Authored-By: Claude Opus 5 (1M context) --- gen-ai-evaluation-record.graphqls | 89 ++++++++++++++++++++++--------- 1 file changed, 63 insertions(+), 26 deletions(-) diff --git a/gen-ai-evaluation-record.graphqls b/gen-ai-evaluation-record.graphqls index 91ef0a2..56729d0 100644 --- a/gen-ai-evaluation-record.graphqls +++ b/gen-ai-evaluation-record.graphqls @@ -21,28 +21,59 @@ type GenAIEvaluationRecords { debuggingTrace: DebuggingTrace } +# The addressing scheme of the span an evaluation was produced from. +enum GenAITraceRefType { + # SkyWalking native: trace + segment + segment-local span index. + SKYWALKING_NATIVE + # OTLP / Zipkin: trace + globally unique hex span id. No segment concept. + OTLP +} + +# Points at the span an evaluation was produced from, in whichever addressing scheme +# its source used. GenAI evaluation accepts SkyWalking native, OTLP and Zipkin traces, +# and a single span-id type cannot describe all of them. +type GenAITraceRef { + type: GenAITraceRefType! + traceId: String! + # SKYWALKING_NATIVE only. The segment carrying the span. + segmentId: String + # SKYWALKING_NATIVE only. Segment-local index (0, 1, 2 ...), NOT an identifier. + # Unique only in combination with segmentId. + spanIndex: Int + # OTLP / Zipkin only. 16 lowercase hex characters, globally unique within the trace. + spanId: String +} + type GenAIEvaluationRecord { - traceId: String + traceRef: GenAITraceRef! + + # The application service that issued the GenAI call. SERVICE scope, a normal + # (agent-detected) service - its layer is whatever the agent reports. serviceId: String serviceName: String + # The GenAI provider. SERVICE scope, Layer.VIRTUAL_GENAI (conjectured). + # The same entity as GenAIProviderAccess. providerId: String providerName: String + # The model. SERVICE_INSTANCE scope under the provider service, Layer.VIRTUAL_GENAI. + # The same entity as GenAIModelAccess. modelId: String modelName: String + operationName: String - # The value type determines which field contains the evaluation value. - # SCORE and BOOLEAN use scoreValue; STRING and JSON use value. - valueType: GenAIEvaluationValueType - # Numeric value stored in evaNumberValue using the 1,000,000 scale. It is populated for SCORE and BOOLEAN. - scoreValue: Long - # String or JSON value stored in evalStringValue. It is populated for STRING and JSON. - value: String - segmentId: String - spanId: String - spanType: String taskName: String - evaluationLevel: String + # Determines which of the three value fields below is populated. + valueType: GenAIEvaluationValueType! + # Populated when valueType = SCORE. Carried on the 1,000,000 ppm scale as stored; + # divide by 1,000,000 to display the original 0.0-1.0 score. + scoreValue: Long + # Populated when valueType = BOOLEAN. + booleanValue: Boolean + # Populated when valueType = STRING or JSON. Parse according to valueType. + stringValue: String + # The judge's explanation. Populated for every task, whatever the value type. reason: String + evaluationLevel: String judgeModel: String evaluationTime: Long } @@ -59,33 +90,39 @@ enum GenAIEvaluationValueType { JSON } -input GenAIEvaluationRecordTag { - key: String! - value: String -} - input GenAIEvaluationRecordQueryCondition { - serviceId: ID + # The GenAI provider, a VIRTUAL_GENAI service. providerId: ID + # The model, a VIRTUAL_GENAI service instance under the provider. modelId: ID - # Filter value_type first when selecting a specific evaluation kind. + # The application service that issued the GenAI call. + serviceId: ID + + taskName: String + # Filter valueType first when selecting a specific evaluation kind. valueType: GenAIEvaluationValueType - # Bounds for evaNumberValue, used only with the SCORE value type. Values use the 1,000,000 scale. + # Bounds for scoreValue on the 1,000,000 ppm scale, as stored. + # Only meaningful when valueType = SCORE. minScore: Long maxScore: Long - # Boolean value to match. This is only used when valueType is BOOLEAN. + # Boolean value to match. Only meaningful when valueType = BOOLEAN. booleanValue: Boolean - sortBy: GenAIEvaluationRecordSortBy - taskName: String evaluationLevel: String judgeModel: String - relatedTrace: TraceScopeCondition + + # [Required] queryDuration is required in most queries, the only exception + # is when relatedTrace is used. queryDuration: Duration + # Scopes to a SkyWalking native trace, segment and span. OTLP / Zipkin evaluations + # are scoped by traceId only - their span ids are not addressable through this + # native-oriented condition. + relatedTrace: TraceScopeCondition + paging: Pagination! - tags: [GenAIEvaluationRecordTag!] + sortBy: GenAIEvaluationRecordSortBy queryOrder: Order } extend type Query { queryGenAIEvaluationRecord(condition: GenAIEvaluationRecordQueryCondition, debug: Boolean): GenAIEvaluationRecords -} \ No newline at end of file +}