diff --git a/.changeset/eve-instrumentation-events-and-caller.md b/.changeset/eve-instrumentation-events-and-caller.md new file mode 100644 index 00000000..7be251ee --- /dev/null +++ b/.changeset/eve-instrumentation-events-and-caller.md @@ -0,0 +1,28 @@ +--- +"evlog": minor +--- + +`evlog/eve` records the caller, and composes with another observability backend. + +An agent has exactly one `agent/instrumentation.ts`, and every observability item in eve's registry writes it. `defineEvlogInstrumentation()` owns that file, so it only fits an agent whose instrumentation is evlog's alone. The new `evlogRuntimeContext` contributes evlog's span attributes to instrumentation you already have, the way the other integrations do: + +```ts +import { defineInstrumentation } from 'eve/instrumentation' +import { evlogRuntimeContext } from 'evlog/eve' + +export default defineInstrumentation({ + setup: ({ agentName }) => registerOTel({ serviceName: agentName, spanProcessors: [...] }), + events: { + 'step.started': input => ({ + runtimeContext: { + ...evlogRuntimeContext(input), + posthog_distinct_id: input.session.auth.current?.principalId ?? '', + }, + }), + }, +}) +``` + +It returns `undefined` outside a tracked turn, so spreading it adds nothing. + +Turn and session events now carry `eve.caller` with the principal eve resolved at dispatch: `principalId`, `principalType` and `authenticator`. On a multi-user channel that is the dimension you group cost, volume and refusals by, and it was previously unreachable — the enrich hook is HTTP-shaped and exposes no path to the eve session. `subject` and `attributes` are deliberately excluded, since a channel may put a name or an email in them. diff --git a/apps/docs/content/5.use-cases/5.eve.md b/apps/docs/content/5.use-cases/5.eve.md index 65171515..afacd70d 100644 --- a/apps/docs/content/5.use-cases/5.eve.md +++ b/apps/docs/content/5.use-cases/5.eve.md @@ -187,6 +187,7 @@ Beyond the identifiers above, a turn records what eve reports about it. Every fi | Field | What it tells you | | --- | --- | | `eve.runtime` | eve version, agent id, model, and the deployed `gitSha` / `gitBranch` / `deployedAt` | +| `eve.caller` | Who triggered the turn: `principalId`, `principalType` and `authenticator`. On a multi-user channel this is what you group cost and volume by. `subject` and `attributes` are never recorded — a channel may put a name or an email in them | | `eve.parent` | Parent and root session ids for a subagent run — rebuild the delegation tree with `rootSessionId` | | `eve.authorizations` | Connection sign-ins with their `outcome`, `reason` and duration | | `eve.compaction` | How many compactions ran, on which model, and `inputTokensAtTrigger` — how full the context was when the first one fired | @@ -223,6 +224,50 @@ export default defineEvlogInstrumentation({ `functionId`, `recordInputs`, `recordOutputs` and `traceChannelRequests` pass straight through to eve's `defineInstrumentation`. +### Alongside another observability backend + +`defineEvlogInstrumentation()` is the shortcut for an agent whose instrumentation is evlog's alone. It owns the file, and an agent has exactly one `agent/instrumentation.ts` — every observability item in eve's registry writes that same path, which is why `eve add instrumentation/sentry` after `eve add instrumentation/posthog` refuses rather than clobbering the first. + +Once another backend is in play, write the file yourself with eve's own `defineInstrumentation` and spread `evlogRuntimeContext` into your runtime context. evlog contributes attributes to your instrumentation instead of wrapping it: + +```typescript [agent/instrumentation.ts] +import { BatchSpanProcessor, SimpleSpanProcessor } from '@opentelemetry/sdk-trace-base' +import { PostHogTraceExporter } from '@posthog/ai/otel' +import { OTLPHttpProtoTraceExporter, registerOTel } from '@vercel/otel' +import { defineInstrumentation } from 'eve/instrumentation' +import { evlogRuntimeContext } from 'evlog/eve' + +export default defineInstrumentation({ + // One provider, one span processor per backend. Items that generate + // `traceExporter` (Sentry, Datadog, Arize, Jaeger, Braintrust, Honeycomb) + // become an entry in this array. Call registerOTel once, not once per backend. + setup: ({ agentName }) => registerOTel({ + serviceName: agentName, + spanProcessors: [ + new SimpleSpanProcessor(new PostHogTraceExporter({ + projectToken: process.env.POSTHOG_PROJECT_TOKEN!, + })), + new BatchSpanProcessor(new OTLPHttpProtoTraceExporter({ + url: process.env.SENTRY_OTLP_TRACES_ENDPOINT!, + headers: { 'x-sentry-auth': `sentry sentry_key=${process.env.SENTRY_PUBLIC_KEY}` }, + })), + ], + }), + events: { + 'step.started': input => ({ + runtimeContext: { + ...evlogRuntimeContext(input), + posthog_distinct_id: input.session.auth.initiator?.principalId + ?? input.session.auth.current?.principalId + ?? '', + }, + }), + }, +}) +``` + +`evlogRuntimeContext` returns `undefined` outside a tracked turn, and spreading that adds nothing. PostHog is the only registry item that also uses `events`; the rest only need their span processor. Keep each generated file open while you merge — the env vars and exporter options are the parts worth copying exactly. + ## Production Long-running eve agents should disable terminal pretty-printing and use a non-blocking drain: diff --git a/packages/evlog/README.md b/packages/evlog/README.md index b5e1525b..398add9c 100644 --- a/packages/evlog/README.md +++ b/packages/evlog/README.md @@ -594,7 +594,9 @@ export default defineEvlogInstrumentation() `defineEvlogHook()` maps eve turn lifecycle events to one wide event per turn. Call `useLogger()` in tools — the logger is bound via AsyncLocalStorage on `turn.started`. Pass `ctx` only when ALS is unavailable (`useLogger(ctx)`). Pretty-printing follows `isDev()` by default (tree locally, JSON in production); set `init.pretty: false` explicitly if you need to override. -`defineEvlogInstrumentation()` is optional: it stamps `evlog.request_id` onto eve's AI SDK spans so a trace joins back to its wide event, and back. Requires eve 0.30 or later. Complements eve Agent Runs — see the [eve use case](https://evlog.dev/use-cases/eve). +`defineEvlogInstrumentation()` is optional: it stamps `evlog.request_id` onto eve's AI SDK spans so a trace joins back to its wide event, and back. It owns `agent/instrumentation.ts`, so when another observability backend needs that file, use eve's own `defineInstrumentation` and spread `evlogRuntimeContext(input)` into your runtime context instead. Requires eve 0.30 or later. Complements eve Agent Runs — see the [eve use case](https://evlog.dev/use-cases/eve). + +Every turn event carries `eve.caller` — `principalId`, `principalType` and `authenticator` — so cost and volume group by who triggered the turn. See the full [eve example](https://github.com/HugoRCD/evlog/tree/main/examples/eve) for a complete agent layout. diff --git a/packages/evlog/src/eve/index.ts b/packages/evlog/src/eve/index.ts index b953bf5c..ba7c03dc 100644 --- a/packages/evlog/src/eve/index.ts +++ b/packages/evlog/src/eve/index.ts @@ -724,6 +724,24 @@ function flushEveMetadata(state: TurnState): void { } /** Wide-event view of the eve instance and the parent session, when there is one. */ +/** + * Who triggered this turn, from the caller principal eve resolved at dispatch. + * + * Only the identifiers eve itself routes on: `principalId` is opaque on every + * channel eve ships (`github:`, a Slack user id), and `subject` and + * `attributes` are deliberately left out because a channel may put a name or an + * email in them. + */ +function buildCaller(ctx: HookContext): Record | null { + const auth = ctx.session.auth?.current + if (!auth) return null + return { + principalId: auth.principalId, + principalType: auth.principalType, + authenticator: auth.authenticator, + } +} + function buildLineage(sessionId: string, ctx: HookContext): Record { const eve: Record = {} const runtime = sessionRuntimes().get(sessionId) @@ -732,6 +750,9 @@ function buildLineage(sessionId: string, ctx: HookContext): Record 0) eve.runtime = identity } + const caller = buildCaller(ctx) + if (caller) eve.caller = caller + const { parent } = ctx.session if (parent) { eve.parent = { @@ -1413,24 +1434,56 @@ export interface EvlogEveInstrumentationOptions { } /** - * Per-model-call context linking an AI SDK span back to the evlog wide event - * for the same turn. Returns `undefined` outside a tracked turn, which - * contributes no context rather than a half-filled one. + * Per-model-call attributes linking an AI SDK span back to the evlog wide event + * for the same turn, ready to spread into your own runtime context. + * + * Use this when the agent already has an `agent/instrumentation.ts` — from + * `eve add instrumentation/...` or written by hand. Every observability + * integration writes that one file, so evlog contributes attributes to yours + * rather than asking you to nest it inside a wrapper: + * + * @example + * ```ts + * // agent/instrumentation.ts + * import { defineInstrumentation } from 'eve/instrumentation' + * import { evlogRuntimeContext } from 'evlog/eve' + * + * export default defineInstrumentation({ + * setup: ({ agentName }) => registerOTel({ serviceName: agentName }), + * events: { + * 'step.started': input => ({ + * runtimeContext: { + * ...evlogRuntimeContext(input), + * posthog_distinct_id: input.session.auth.current?.principalId ?? '', + * }, + * }), + * }, + * }) + * ``` + * + * Returns `undefined` outside a tracked turn — spreading that adds nothing, + * which is the point. {@link defineEvlogInstrumentation} is the shortcut for an + * agent with no other instrumentation to compose with. */ -function buildInstrumentationContext( +export function evlogRuntimeContext( input: InstrumentationStepStartedEventInput, -): InstrumentationStepStartedEventResult | undefined { +): Record | undefined { const state = getTurnState(input.session.id, input.turn.id) if (!state) return undefined return { - runtimeContext: { - 'evlog.request_id': state.turnId, - 'evlog.session_id': state.sessionId, - }, + 'evlog.request_id': state.turnId, + 'evlog.session_id': state.sessionId, } } +function buildInstrumentationContext( + input: InstrumentationStepStartedEventInput, +): InstrumentationStepStartedEventResult | undefined { + const runtimeContext = evlogRuntimeContext(input) + return runtimeContext ? { runtimeContext } : undefined +} + /** * Create an eve instrumentation definition that stamps evlog's turn identity * onto the AI SDK telemetry spans. @@ -1454,6 +1507,11 @@ function buildInstrumentationContext( * setup: ({ agentName }) => registerOTel({ serviceName: agentName }), * }) * ``` + * + * This is the shortcut for an agent whose instrumentation is evlog's alone. It + * owns the file's `events` slot, so once another integration needs it — PostHog + * links spans to the initiating user there — drop the wrapper and spread + * {@link evlogRuntimeContext} into your own `defineInstrumentation` instead. */ export function defineEvlogInstrumentation( options: EvlogEveInstrumentationOptions = {}, diff --git a/packages/evlog/test/eve.test.ts b/packages/evlog/test/eve.test.ts index 78f1a0ad..75198007 100644 --- a/packages/evlog/test/eve.test.ts +++ b/packages/evlog/test/eve.test.ts @@ -5,6 +5,7 @@ import { resetEvlogEveForTests, defineEvlogHook, defineEvlogInstrumentation, + evlogRuntimeContext, useLogger, detachActiveTurnLoggerForTests, } from '../src/eve/index' @@ -610,6 +611,82 @@ describe('evlog/eve', () => { }) }) + it('records the caller principal that triggered the turn', async () => { + const spies = createPipelineSpies() + const hook = defineEvlogHook({ drain: spies.drain }) + const ctx = { + ...hookContext(), + session: { + id: SESSION_ID, + auth: { + current: { + principalId: 'github:1234', + principalType: 'user', + authenticator: 'github', + subject: 'someone@example.com', + attributes: { login: 'someone' }, + }, + initiator: null, + }, + }, + } as unknown as HookContext + + await runTurn(hook, { ctx }) + + await waitForDrainCalls(spies.drain) + const event = findEventViaDrain(spies.drain, () => true) + expect(event?.eve).toMatchObject({ + caller: { + principalId: 'github:1234', + principalType: 'user', + authenticator: 'github', + }, + }) + }) + + it('keeps the caller subject and attributes off the event', async () => { + const spies = createPipelineSpies() + const hook = defineEvlogHook({ drain: spies.drain }) + const ctx = { + ...hookContext(), + session: { + id: SESSION_ID, + auth: { + current: { + principalId: 'github:1234', + principalType: 'user', + authenticator: 'github', + subject: 'someone@example.com', + attributes: { login: 'someone' }, + }, + initiator: null, + }, + }, + } as unknown as HookContext + + await runTurn(hook, { ctx }) + + await waitForDrainCalls(spies.drain) + const event = findEventViaDrain(spies.drain, () => true) + const { caller } = event?.eve as { caller?: Record } + expect(Object.keys(caller ?? {}).sort()).toEqual([ + 'authenticator', + 'principalId', + 'principalType', + ]) + }) + + it('omits the caller when the session carries no authenticated principal', async () => { + const spies = createPipelineSpies() + const hook = defineEvlogHook({ drain: spies.drain }) + + await runTurn(hook) + + await waitForDrainCalls(spies.drain) + const event = findEventViaDrain(spies.drain, () => true) + expect(event?.eve).not.toHaveProperty('caller') + }) + it('prefers the cost reported by eve over the configured pricing map', async () => { const spies = createPipelineSpies() const hook = defineEvlogHook({ @@ -1706,3 +1783,47 @@ describe('defineEvlogInstrumentation', () => { expect(Object.keys(defineEvlogInstrumentation())).toEqual(['events']) }) }) + +describe('evlogRuntimeContext', () => { + beforeEach(() => { + resetEvlogEveForTests() + initLogger({ env: { service: 'eve-test' } }) + }) + + afterEach(() => { + resetEvlogEveForTests() + }) + + function stepStartedInput(turnId = TURN_ID) { + return { + channel: { kind: 'http' }, + modelInput: { instructions: undefined, messages: [] }, + session: { id: SESSION_ID, auth: { current: null, initiator: null } }, + step: { index: 0 }, + turn: { id: turnId, sequence: 0 }, + } as Parameters[0] + } + + it('returns attributes that spread into an authored runtime context', () => { + const hook = defineEvlogHook({}) + + hook.events!['turn.started']!({ + type: 'turn.started', + data: { sequence: 0, turnId: TURN_ID }, + }, hookContext()) + + expect({ + ...evlogRuntimeContext(stepStartedInput()), + posthog_distinct_id: 'user_1', + }).toEqual({ + 'evlog.request_id': TURN_ID, + 'evlog.session_id': SESSION_ID, + posthog_distinct_id: 'user_1', + }) + }) + + it('spreads to nothing outside a tracked turn', () => { + expect(evlogRuntimeContext(stepStartedInput())).toBeUndefined() + expect({ ...evlogRuntimeContext(stepStartedInput()) }).toEqual({}) + }) +}) diff --git a/packages/evlog/test/toolkit/__snapshots__/api-surface.test.ts.snap b/packages/evlog/test/toolkit/__snapshots__/api-surface.test.ts.snap index 344b476d..352c6a4a 100644 --- a/packages/evlog/test/toolkit/__snapshots__/api-surface.test.ts.snap +++ b/packages/evlog/test/toolkit/__snapshots__/api-surface.test.ts.snap @@ -117,6 +117,7 @@ exports[`public API surface > matches snapshot for all subpath exports 1`] = ` "defineEvlogHook", "defineEvlogInstrumentation", "detachActiveTurnLoggerForTests", + "evlogRuntimeContext", "resetEvlogEveForTests", "useLogger", ],