TRACY-87: Introduce Gradle Task for fixture collection (OpenAI)

.github/workflows/update-fixtures.yml

@@ -0,0 +1,170 @@
+name: Update Test Fixtures
+
+on:
+  # Manual trigger
+  workflow_dispatch:
+    inputs:
+      providers:
+        description: 'Providers to update (comma-separated: openai,anthropic,gemini or "all")'
+        required: true
+        default: 'openai'
+        type: string
+
+  # Scheduled weekly updates (every Monday at 2 AM UTC)
+  schedule:
+    - cron: '0 2 * * 1'
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  update-openai-fixtures:
+    if: |
+      (github.event_name == 'schedule') ||
+      (github.event_name == 'workflow_dispatch' && (github.event.inputs.providers == 'all' || contains(github.event.inputs.providers, 'openai')))
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+          cache: gradle
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+
+      - name: Record OpenAI Fixtures
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        run: |
+          ./gradlew :tracing:openai:recordFixtures --no-daemon
+
+      - name: Check for changes
+        id: check_changes
+        run: |
+          if git diff --quiet tracing/openai/src/jvmTest/resources/fixtures/; then
+            echo "has_changes=false" >> "$GITHUB_OUTPUT"
+            echo "No changes detected in OpenAI fixtures"
+          else
+            echo "has_changes=true" >> "$GITHUB_OUTPUT"
+            echo "Changes detected in OpenAI fixtures"
+          fi
+
+      - name: Create Pull Request
+        if: steps.check_changes.outputs.has_changes == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+
+          # Configure git
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+          # Create branch name with timestamp
+          BRANCH_NAME="fixtures/update-openai-$(date +%Y-%m-%d-%H%M%S)"
+
+          # Create and checkout new branch
+          git checkout -b "$BRANCH_NAME"
+
+          # Add and commit changes
+          git add tracing/openai/src/jvmTest/resources/fixtures/
+          git commit -m "chore: update OpenAI test fixtures
+
+          Automated update of OpenAI test fixtures from real API endpoints.
+
+          Changes:
+          - Refreshed API response schemas
+          - Sanitized non-deterministic data (timestamps, IDs, etc.)
+
+          Triggered by: ${{ github.event_name }}
+          Workflow run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
+
+          # Push branch
+          git push origin "$BRANCH_NAME"
+
+          # Create PR using gh CLI
+          gh pr create \
+            --title "chore: Update OpenAI test fixtures" \
+            --body "$(cat <<'EOF'
+          ## Summary
+          Automated update of OpenAI test fixtures from real API endpoints.
+
+          ## Changes
+          - Refreshed API response schemas to match current OpenAI API
+          - Sanitized non-deterministic data (timestamps, IDs, LLM outputs)
+          - All tests pass with updated fixtures
+
+          ## Testing
+          Tests were run in both RECORD mode (against real API) and MOCK mode (against fixtures) to ensure correctness.
+
+          ## Checklist
+          - [ ] Review sanitized responses for any leaked sensitive data
+          - [ ] Verify tests pass in CI with new fixtures
+          - [ ] Confirm fixture schemas match current API documentation
+
+          ---
+          🤖 Generated by [GitHub Actions](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
+          EOF
+          )" \
+            --base main \
+            --head "$BRANCH_NAME" \
+            --label "automated" \
+            --label "fixtures"
+
+  update-anthropic-fixtures:
+    if: |
+      (github.event_name == 'workflow_dispatch' && (github.event.inputs.providers == 'all' || contains(github.event.inputs.providers, 'anthropic')))
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+          cache: gradle
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+
+      - name: Record Anthropic Fixtures
+        env:
+          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+        run: |
+          echo "Anthropic fixture recording not yet implemented"
+          # ./gradlew :tracing:anthropic:recordFixtures --no-daemon
+
+  update-gemini-fixtures:
+    if: |
+      (github.event_name == 'workflow_dispatch' && (github.event.inputs.providers == 'all' || contains(github.event.inputs.providers, 'gemini')))
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '17'
+          cache: gradle
+
+      - name: Setup Gradle
+        uses: gradle/actions/setup-gradle@v4
+
+      - name: Record Gemini Fixtures
+        env:
+          GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
+        run: |
+          echo "Gemini fixture recording not yet implemented"
+          # ./gradlew :tracing:gemini:recordFixtures --no-daemon

AGENTS.md

@@ -76,13 +76,57 @@ tracy/
 4. **plugin/**: Kotlin compiler plugins for `@Trace` annotation processing
 5. **examples/**: Reference implementations (keep in sync with API changes)
 
+## Test Fixture Recording
+
+Tracy uses a dual-mode testing system to avoid calling real LLM endpoints in CI:
+
+**Mock Mode (Default)**
+- Tests run against a mock HTTP server using pre-recorded fixtures
+- Fast, offline, no API keys required
+- Fixtures stored in `tracing/{provider}/src/jvmTest/resources/fixtures/`
+
+**Record Mode**
+- Tests call real LLM endpoints and record sanitized responses as fixtures
+- Automatically sanitizes non-deterministic data (IDs, timestamps, AI outputs)
+- Used to update fixtures when API schemas change
+
+**Recording Fixtures:**
+
+```bash
+# Record OpenAI fixtures
+export OPENAI_API_KEY=sk-...
+./gradlew :tracing:openai:recordFixtures
+
+# Run tests in mock mode (default)
+./gradlew :tracing:openai:test
+
+# Run tests in record mode manually
+./gradlew :tracing:openai:test -Dtracy.test.mode=record
+```
+
+**Automated Updates:**
+- GitHub Actions workflow runs weekly to update fixtures automatically
+- Creates PR with updated fixtures for review
+- Manual trigger available via workflow_dispatch
+- See `.github/workflows/update-fixtures.yml`
+
+**Fixture Sanitization:**
+Each provider has a custom sanitizer that removes:
+- **IDs**: `id`, `request_id`, `organization_id`, etc. → `"sanitized-*"`
+- **Timestamps**: `created`, `created_at` → fixed timestamp
+- **AI Content**: Assistant messages, tool arguments → generic placeholders
+- **Rate Limit Headers**: Removed entirely
+
 ## Adding a New Provider
 
 Use existing providers as reference.
 
 **Steps:**
 1. Create `tracing/{provider}/` module, register it in `settings.gradle.kts`
-2. Extend `LLMTracingAdapter(genAISystem)` — override `getRequestBodyAttributes`, `getResponseBodyAttributes`, `getSpanName`, `isStreamingRequest`, `handleStreaming`
+2. Extend `LLMTracingAdapter(genAISystem)` — override `getRequestBodyAttributes`, `getResponseBodyAttributes`, `getSpanName`, and abstract `registerResponseStreamEvent`
 3. If multiple distinct API endpoints exist, implement `EndpointApiHandler` per endpoint and delegate from the adapter
 4. Add a public `instrument(client)` function — use `patchOpenAICompatibleClient()` for OpenAI-compatible SDKs, or reflection + `patchInterceptors()` for others (see `GeminiClient.kt`)
 5. Write tests extending `BaseAITracingTest`, tag with `@Tag("{provider}")`
+6. Create a `ResponseSanitizer` implementation for the provider in `test-utils`
+7. Add `recordFixtures` Gradle task (see `tracing/openai/build.gradle.kts`)
+8. Create fixtures directory: `src/jvmTest/resources/fixtures/`

examples/src/main/kotlin/org/jetbrains/ai/tracy/examples/clients/OkHttpClientAutotracingExample.kt

@@ -6,11 +6,11 @@
 package org.jetbrains.ai.tracy.examples.clients
 
 import org.jetbrains.ai.tracy.anthropic.adapters.AnthropicLLMTracingAdapter
-import org.jetbrains.ai.tracy.core.OpenTelemetryOkHttpInterceptor
+import org.jetbrains.ai.tracy.core.interceptors.OpenTelemetryOkHttpInterceptor
 import org.jetbrains.ai.tracy.core.TracingManager
 import org.jetbrains.ai.tracy.core.configureOpenTelemetrySdk
 import org.jetbrains.ai.tracy.core.exporters.ConsoleExporterConfig
-import org.jetbrains.ai.tracy.core.instrument
+import org.jetbrains.ai.tracy.core.interceptors.instrument
 import org.jetbrains.ai.tracy.gemini.adapters.GeminiLLMTracingAdapter
 import org.jetbrains.ai.tracy.openai.adapters.OpenAILLMTracingAdapter
 import com.openai.models.ChatModel

tracing/anthropic/src/jvmMain/kotlin/org/jetbrains/ai/tracy/anthropic/adapters/AnthropicLLMTracingAdapter.kt

@@ -5,9 +5,16 @@
 
 package org.jetbrains.ai.tracy.anthropic.adapters
 
+import io.opentelemetry.api.trace.Span
+import io.opentelemetry.sdk.trace.ReadableSpan
+import io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.*
+import kotlinx.serialization.json.*
+import mu.KotlinLogging
 import org.jetbrains.ai.tracy.core.adapters.LLMTracingAdapter
 import org.jetbrains.ai.tracy.core.adapters.LLMTracingAdapter.Companion.PayloadType
+import org.jetbrains.ai.tracy.core.adapters.handlers.sse.sseHandlingUnsupported
 import org.jetbrains.ai.tracy.core.adapters.media.*
+import org.jetbrains.ai.tracy.core.http.parsers.SseEvent
 import org.jetbrains.ai.tracy.core.http.protocol.TracyHttpRequest
 import org.jetbrains.ai.tracy.core.http.protocol.TracyHttpResponse
 import org.jetbrains.ai.tracy.core.http.protocol.TracyHttpUrl
@@ -16,10 +23,6 @@ import org.jetbrains.ai.tracy.core.policy.ContentKind
 import org.jetbrains.ai.tracy.core.policy.contentTracingAllowed
 import org.jetbrains.ai.tracy.core.policy.orRedactedInput
 import org.jetbrains.ai.tracy.core.policy.orRedactedOutput
-import io.opentelemetry.api.trace.Span
-import io.opentelemetry.semconv.incubating.GenAiIncubatingAttributes.*
-import kotlinx.serialization.json.*
-import mu.KotlinLogging
 
 /**
  * Tracing adapter for Anthropic Claude API.
@@ -52,8 +55,8 @@ class AnthropicLLMTracingAdapter : LLMTracingAdapter(genAISystem = GenAiSystemIn
         val body = request.body.asJson()?.jsonObject ?: return
 
         body["temperature"]?.jsonPrimitive?.doubleOrNull?.let { span.setAttribute(GEN_AI_REQUEST_TEMPERATURE, it) }
-        body["model"]?.jsonPrimitive?.let { span.setAttribute(GEN_AI_REQUEST_MODEL, it.content) }
-        body["max_tokens"]?.jsonPrimitive?.intOrNull?.let { span.setAttribute(GEN_AI_REQUEST_MAX_TOKENS, it.toLong()) }
+        body["model"]?.jsonPrimitive?.content?.let { span.setAttribute(GEN_AI_REQUEST_MODEL, it) }
+        body["max_tokens"]?.jsonPrimitive?.longOrNull?.let { span.setAttribute(GEN_AI_REQUEST_MAX_TOKENS, it) }
 
         // metadata
         body["metadata"]?.jsonObject?.let { metadata ->
@@ -127,10 +130,28 @@ class AnthropicLLMTracingAdapter : LLMTracingAdapter(genAISystem = GenAiSystemIn
         val body = response.body.asJson()?.jsonObject ?: return
 
         body["id"]?.let { span.setAttribute(GEN_AI_RESPONSE_ID, it.jsonPrimitive.content) }
-        body["type"]?.let { span.setAttribute(GEN_AI_OUTPUT_TYPE, it.jsonPrimitive.content) }
+        body["type"]?.jsonPrimitive?.content?.let {
+            span.setAttribute(GEN_AI_OUTPUT_TYPE, it)
+            span.setAttribute(GEN_AI_OPERATION_NAME, it)
+        }
         body["role"]?.let { span.setAttribute("gen_ai.response.role", it.jsonPrimitive.content) }
         body["model"]?.let { span.setAttribute(GEN_AI_RESPONSE_MODEL, it.jsonPrimitive.content) }
 
+        // update the span name to follow GenAI Anthropic Conventions
+        // convention: `{gen_ai.operation.name} {gen_ai.request.model}`
+        // see: https://opentelemetry.io/docs/specs/semconv/gen-ai/anthropic/#spans
+        val spanName = run {
+            val type = body["type"]?.jsonPrimitive?.contentOrNull
+            val model = (span as? ReadableSpan)
+                ?.attributes
+                ?.get(GEN_AI_REQUEST_MODEL)
+                ?: body["model"]
+            if (type != null && model != null) "$type $model" else null
+        }
+        if (spanName != null) {
+            span.updateName(spanName)
+        }
+
         // collecting response messages
         body["content"]?.let {
             for ((index, message) in it.jsonArray.withIndex()) {
@@ -204,11 +225,26 @@ class AnthropicLLMTracingAdapter : LLMTracingAdapter(genAISystem = GenAiSystemIn
         span.populateUnmappedAttributes(body, mappedAttributes, PayloadType.RESPONSE)
     }
 
-    override fun getSpanName(request: TracyHttpRequest) = "Anthropic-generation"
-
-    // streaming is not supported
-    override fun isStreamingRequest(request: TracyHttpRequest) = false
-    override fun handleStreaming(span: Span, url: TracyHttpUrl, events: String) = Unit
+    /**
+     * Sets a default span name to **"Anthropic-generation"**.
+     *
+     * This name will be overridden in [getResponseBodyAttributes] to follow GenAI Conventions for Anthropic:
+     * ```
+     * {gen_ai.operation.name} {gen_ai.request.model}
+     * ```
+     *
+     * See [GenAI Anthropic Spans](https://opentelemetry.io/docs/specs/semconv/gen-ai/anthropic/#spans)
+     */
+    override fun getSpanName() = "Anthropic-generation"
+
+    override fun registerResponseStreamEvent(
+        span: Span,
+        url: TracyHttpUrl,
+        event: SseEvent,
+        index: Long
+    ): Result<Unit> {
+        return sseHandlingUnsupported()
+    }
 
     /**
      * Parses content of the `messages` field when its type is

tracing/anthropic/src/jvmMain/kotlin/org/jetbrains/ai/tracy/anthropic/clients/AnthropicAIClient.kt

@@ -6,9 +6,9 @@
 package org.jetbrains.ai.tracy.anthropic.clients
 
 import org.jetbrains.ai.tracy.anthropic.adapters.AnthropicLLMTracingAdapter
-import org.jetbrains.ai.tracy.core.OpenTelemetryOkHttpInterceptor
+import org.jetbrains.ai.tracy.core.interceptors.OpenTelemetryOkHttpInterceptor
 import org.jetbrains.ai.tracy.core.TracingManager
-import org.jetbrains.ai.tracy.core.patchOpenAICompatibleClient
+import org.jetbrains.ai.tracy.core.interceptors.patchOpenAICompatibleClient
 import com.anthropic.client.AnthropicClient
 
 /**

tracing/anthropic/src/jvmTest/kotlin/org/jetbrains/ai/tracy/anthropic/AnthropicTracingTest.kt

@@ -7,7 +7,7 @@ package org.jetbrains.ai.tracy.anthropic
 
 import org.jetbrains.ai.tracy.anthropic.clients.instrument
 import org.jetbrains.ai.tracy.core.TracingManager
-import org.jetbrains.ai.tracy.core.patchOpenAICompatibleClient
+import org.jetbrains.ai.tracy.core.interceptors.patchOpenAICompatibleClient
 import org.jetbrains.ai.tracy.core.policy.ContentCapturePolicy
 import com.anthropic.core.JsonString
 import com.anthropic.core.JsonValue
@@ -419,10 +419,14 @@ class AnthropicTracingTest : BaseAnthropicTracingTest() {
                         }
                     """.trimIndent().toResponseBody("application/json".toMediaTypeOrNull())
 
-            response.newBuilder()
+            val newResponse = response.newBuilder()
                 .body(errorBody)
                 .code(529)
                 .build()
+            // close the original response body
+            response.body.close()
+
+            return@Interceptor newResponse
         }
 
         patchOpenAICompatibleClient(