From c4a390d560f8e3e0306347ebf781571908ed3227 Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Tue, 26 May 2026 17:50:31 -0400 Subject: [PATCH 1/7] chore(deps): bump @livekit/protocol to ^1.46.3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Picks up the new agent-session messages required for upcoming work: CustomEvent, AgentSessionEvent.custom_event (livekit/protocol#1588), AgentSessionEvent.FunctionToolsStarted, AgentSessionEvent.EotPrediction, and SessionRequest.UpdateIO. No runtime change in @livekit/agents itself — this only makes the new generated types resolvable. Type-check (tsc --noEmit) passes against the new version. Co-authored-by: Cursor --- .changeset/bump-protocol-1.46.3.md | 7 +++++++ agents/package.json | 2 +- pnpm-lock.yaml | 12 ++++++------ 3 files changed, 14 insertions(+), 7 deletions(-) create mode 100644 .changeset/bump-protocol-1.46.3.md diff --git a/.changeset/bump-protocol-1.46.3.md b/.changeset/bump-protocol-1.46.3.md new file mode 100644 index 000000000..bddb58463 --- /dev/null +++ b/.changeset/bump-protocol-1.46.3.md @@ -0,0 +1,7 @@ +--- +'@livekit/agents': patch +--- + +chore(deps): bump `@livekit/protocol` to `^1.46.3` + +Picks up new agent-session messages: `CustomEvent`, `AgentSessionEvent.custom_event` (livekit/protocol#1588), `AgentSessionEvent.FunctionToolsStarted`, `AgentSessionEvent.EotPrediction`, and `SessionRequest.UpdateIO`. No runtime behavior change in `@livekit/agents` itself — this only makes the new types available for downstream consumers. diff --git a/agents/package.json b/agents/package.json index e543404ff..93702f3b5 100644 --- a/agents/package.json +++ b/agents/package.json @@ -53,7 +53,7 @@ "@bufbuild/protobuf": "^1.10.0", "@ffmpeg-installer/ffmpeg": "^1.1.0", "@livekit/mutex": "^1.1.1", - "@livekit/protocol": "^1.45.7", + "@livekit/protocol": "^1.46.3", "@livekit/typed-emitter": "^3.0.0", "@livekit/throws-transformer": "0.1.8", "@opentelemetry/api": "^1.9.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 51c33a91e..c67ce6d8e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -116,8 +116,8 @@ importers: specifier: ^1.1.1 version: 1.1.1 '@livekit/protocol': - specifier: ^1.45.7 - version: 1.45.7 + specifier: ^1.46.3 + version: 1.46.3 '@livekit/throws-transformer': specifier: 0.1.8 version: 0.1.8(typescript@5.9.3) @@ -2250,8 +2250,8 @@ packages: cpu: [x64] os: [win32] - '@livekit/protocol@1.45.7': - resolution: {integrity: sha512-UVYtWQohAwowygFFglMKfgjVZMQncCEmHmsQX2yJDhgBf1nZQdfANgUJg+ifxZDTfVpNnQWQjikWMHViq5fh2Q==} + '@livekit/protocol@1.46.3': + resolution: {integrity: sha512-YvsE4UN5i+wY9vXfwhF6EUrRyUm/YhiFU1jBcsmsLd/xodUJxYTBcWS4OgL4IJffjzIoyxsrbKp1h9qC55mtcQ==} '@livekit/rtc-ffi-bindings-darwin-arm64@0.12.52-patch.0': resolution: {integrity: sha512-IKUir6goV8yVRR7E2qrAP0JtH7gUyMkO0TG8G+dopO/fkXAsPpSealgI9fLcBJl0zhKK+eGCr741r6xR+xxsVw==} @@ -6294,7 +6294,7 @@ snapshots: '@livekit/noise-cancellation-win32-x64@0.1.9': optional: true - '@livekit/protocol@1.45.7': + '@livekit/protocol@1.46.3': dependencies: '@bufbuild/protobuf': 1.10.1 @@ -8624,7 +8624,7 @@ snapshots: livekit-server-sdk@2.14.1: dependencies: '@bufbuild/protobuf': 1.10.1 - '@livekit/protocol': 1.45.7 + '@livekit/protocol': 1.46.3 camelcase-keys: 9.1.3 jose: 5.2.4 From 7812356cba26c7b7653119b92eab76725ba9c8c2 Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Wed, 27 May 2026 06:05:28 +0800 Subject: [PATCH 2/7] Add custom event remote session proto (#1610) --- agents/src/voice/agent_session.ts | 2 ++ agents/src/voice/events.ts | 22 ++++++++++++++++++ agents/src/voice/remote_session.ts | 37 +++++++++++++++++++++++++++++- examples/src/multi_agent.ts | 8 ++++++- 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index 33920015f..5f4717092 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -61,6 +61,7 @@ import { type CloseEvent, CloseReason, type ConversationItemAddedEvent, + type CustomEvent, type ErrorEvent, type FunctionToolsExecutedEvent, type MetricsCollectedEvent, @@ -157,6 +158,7 @@ export type AgentSessionCallbacks = { [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void; [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void; [AgentSessionEventTypes.OverlappingSpeech]: (ev: OverlappingSpeechEvent) => void; + [AgentSessionEventTypes.CustomEvent]: (ev: CustomEvent) => void; }; export type AgentSessionOptions = { diff --git a/agents/src/voice/events.ts b/agents/src/voice/events.ts index 55cd13e0f..65b610a46 100644 --- a/agents/src/voice/events.ts +++ b/agents/src/voice/events.ts @@ -33,6 +33,7 @@ export enum AgentSessionEventTypes { SpeechCreated = 'speech_created', AgentFalseInterruption = 'agent_false_interruption', OverlappingSpeech = 'overlapping_speech', + CustomEvent = 'custom_event', Error = 'error', Close = 'close', } @@ -340,6 +341,26 @@ export const createAgentFalseInterruptionEvent = ({ createdAt, }); +export type CustomEvent = { + type: 'custom_event'; + /** Maps to proto `CustomEvent.type`. */ + eventType: string; + /** Maps to proto `CustomEvent.payload` (arbitrary JSON object). */ + payload: Record; + createdAt: number; +}; + +export const createCustomEvent = ( + eventType: string, + payload: Record = {}, + createdAt: number = Date.now(), +): CustomEvent => ({ + type: 'custom_event', + eventType, + payload, + createdAt, +}); + export type AgentEvent = | UserInputTranscribedEvent | UserStateChangedEvent @@ -351,5 +372,6 @@ export type AgentEvent = | SpeechCreatedEvent | AgentFalseInterruptionEvent | OverlappingSpeechEvent + | CustomEvent | ErrorEvent | CloseEvent; diff --git a/agents/src/voice/remote_session.ts b/agents/src/voice/remote_session.ts index f970b064e..291e798a8 100644 --- a/agents/src/voice/remote_session.ts +++ b/agents/src/voice/remote_session.ts @@ -1,7 +1,7 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import { Duration, Timestamp } from '@bufbuild/protobuf'; +import { Duration, Struct, Timestamp } from '@bufbuild/protobuf'; import { AgentSession as pb } from '@livekit/protocol'; import type { ByteStreamReader, Room, TextStreamInfo } from '@livekit/rtc-node'; import { ThrowsPromise } from '@livekit/throws-transformer/throws'; @@ -32,6 +32,7 @@ import { type AgentState, type AgentStateChangedEvent, type ConversationItemAddedEvent, + type CustomEvent, type ErrorEvent, type FunctionToolsExecutedEvent, type MetricsCollectedEvent, @@ -62,6 +63,7 @@ export type RemoteSessionEventTypes = | 'function_tools_executed' | 'overlapping_speech' | 'amd_prediction' + | 'custom_event' | 'session_usage' | 'error'; @@ -74,10 +76,22 @@ export type RemoteSessionCallbacks = { function_tools_executed: (ev: pb.AgentSessionEvent_FunctionToolsExecuted) => void; overlapping_speech: (ev: pb.AgentSessionEvent_OverlappingSpeech) => void; amd_prediction: (ev: pb.AgentSessionEvent_AmdPrediction) => void; + custom_event: (ev: { eventType: string; payload: Record }) => void; session_usage: (ev: pb.AgentSessionEvent_SessionUsageUpdated) => void; error: (ev: pb.AgentSessionEvent_Error) => void; }; +function dictToStruct(data: Record): Struct { + return Struct.fromJson(data); +} + +function structToDict(st: Struct | undefined): Record { + if (!st) { + return {}; + } + return st.toJson() as Record; +} + // =========================================================================== // SessionTransport // =========================================================================== @@ -522,6 +536,7 @@ export class SessionHost { session.on(AgentSessionEventTypes.MetricsCollected, this.onMetricsCollected); session.on(AgentSessionEventTypes.OverlappingSpeech, this.onOverlappingSpeech); session.on(AgentSessionEventTypes.Error, this.onHostError); + session.on(AgentSessionEventTypes.CustomEvent, this.onCustomEvent); } } @@ -550,6 +565,7 @@ export class SessionHost { this.session.off(AgentSessionEventTypes.MetricsCollected, this.onMetricsCollected); this.session.off(AgentSessionEventTypes.OverlappingSpeech, this.onOverlappingSpeech); this.session.off(AgentSessionEventTypes.Error, this.onHostError); + this.session.off(AgentSessionEventTypes.CustomEvent, this.onCustomEvent); } if (this.recvTask) { @@ -713,6 +729,19 @@ export class SessionHost { ); }; + private onCustomEvent = (event: CustomEvent): void => { + this.emitEvent( + { + case: 'customEvent', + value: new pb.CustomEvent({ + type: event.eventType, + payload: dictToStruct(event.payload), + }), + }, + event.createdAt, + ); + }; + /** * @internal — forwards an AMD prediction to the connected * {@link RemoteSession} peer. Mirrors python @@ -1007,6 +1036,12 @@ export class RemoteSession extends (EventEmitter as new () => TypedEventEmitter< case 'error': this.emit('error', ev.value); break; + case 'customEvent': + this.emit('custom_event', { + eventType: ev.value.type, + payload: structToDict(ev.value.payload), + }); + break; } } diff --git a/examples/src/multi_agent.ts b/examples/src/multi_agent.ts index 7f4819bed..3fa4a6167 100644 --- a/examples/src/multi_agent.ts +++ b/examples/src/multi_agent.ts @@ -102,4 +102,10 @@ export default defineAgent({ }, }); -cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) })); +cli.runApp( + new ServerOptions({ + agent: fileURLToPath(import.meta.url), + // Enables explicit dispatch so agents-cli can summon this worker by name. + agentName: 'multi-agent-js', + }), +); From 62b343be9f6d1cbb08fc79fe86d7f0241eda6b2e Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Tue, 26 May 2026 18:06:54 -0400 Subject: [PATCH 3/7] Update multi_agent.ts --- examples/src/multi_agent.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/examples/src/multi_agent.ts b/examples/src/multi_agent.ts index 3fa4a6167..7f4819bed 100644 --- a/examples/src/multi_agent.ts +++ b/examples/src/multi_agent.ts @@ -102,10 +102,4 @@ export default defineAgent({ }, }); -cli.runApp( - new ServerOptions({ - agent: fileURLToPath(import.meta.url), - // Enables explicit dispatch so agents-cli can summon this worker by name. - agentName: 'multi-agent-js', - }), -); +cli.runApp(new ServerOptions({ agent: fileURLToPath(import.meta.url) })); From a9090150635c1bd4d335eaadde3ca57951908315 Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Tue, 26 May 2026 18:12:40 -0400 Subject: [PATCH 4/7] fix(voice): satisfy Struct.fromJson signature in dictToStruct MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Struct.fromJson` expects `JsonValue`, but the helper accepts `Record` which is structurally wider (unknown is broader than JsonValue). Cast at the trust boundary — the caller is contractually responsible for JSON-serializable values, and `Struct.fromJson` validates at runtime anyway. Fixes the `build:types` failure: src/voice/remote_session.ts:85:26 - error TS2345: Argument of type 'Record' is not assignable to parameter of type 'JsonValue'. Co-authored-by: Cursor --- agents/src/voice/remote_session.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/agents/src/voice/remote_session.ts b/agents/src/voice/remote_session.ts index 291e798a8..3539742ea 100644 --- a/agents/src/voice/remote_session.ts +++ b/agents/src/voice/remote_session.ts @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import type { JsonValue } from '@bufbuild/protobuf'; import { Duration, Struct, Timestamp } from '@bufbuild/protobuf'; import { AgentSession as pb } from '@livekit/protocol'; import type { ByteStreamReader, Room, TextStreamInfo } from '@livekit/rtc-node'; @@ -82,7 +83,10 @@ export type RemoteSessionCallbacks = { }; function dictToStruct(data: Record): Struct { - return Struct.fromJson(data); + // Caller is contractually responsible for JSON-serializable values; `Struct.fromJson` + // validates at runtime. The cast widens `unknown` -> `JsonValue` to satisfy the + // structural mismatch (`Record` is wider than `JsonObject`). + return Struct.fromJson(data as JsonValue); } function structToDict(st: Struct | undefined): Record { From 8e2f2c4641ea564efb3bf83274173c79aa9e887e Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Tue, 26 May 2026 18:47:05 -0400 Subject: [PATCH 5/7] refactor(voice): use pb.CustomEvent directly, drop wrapper Mirrors the Python-side decision (livekit/agents#5855): with CustomEvent already defined in livekit/protocol, the parallel TS wrapper + dict<->Struct conversion was pure ceremony with no semantic gain. Drop the wrapper and forward the proto unchanged. - events.ts: remove `CustomEvent` type + `createCustomEvent` helper + membership in the `AgentEvent` union. Keep the `AgentSessionEventTypes.CustomEvent` enum entry so the listener topic name stays stable. - agent_session.ts: callback signature is now `(ev: pb.CustomEvent) => void` (import-as-type). - remote_session.ts: - drop `dictToStruct` / `structToDict` and the `JsonValue` import - `RemoteSessionCallbacks.custom_event` typed as `pb.CustomEvent` - `onCustomEvent` forwards the proto as-is via `emitEvent({ case: 'customEvent', value: event })` - dispatch side re-emits `ev.value` directly (no Struct->dict round-trip) User-facing emit: session.emit( 'custom_event', new pb.CustomEvent({ type: 'anomaly_detected', payload: Struct.fromJson({ score: 0.92 }), }), ); `tsc --noEmit` and lint clean (no new warnings beyond the pre-existing test-file `any` noise). Co-authored-by: Cursor --- .changeset/bump-protocol-1.46.3.md | 27 +++++++++++++++++++-- agents/src/voice/agent_session.ts | 4 ++-- agents/src/voice/events.ts | 21 ----------------- agents/src/voice/remote_session.ts | 38 ++++-------------------------- 4 files changed, 32 insertions(+), 58 deletions(-) diff --git a/.changeset/bump-protocol-1.46.3.md b/.changeset/bump-protocol-1.46.3.md index bddb58463..3de5ae85f 100644 --- a/.changeset/bump-protocol-1.46.3.md +++ b/.changeset/bump-protocol-1.46.3.md @@ -2,6 +2,29 @@ '@livekit/agents': patch --- -chore(deps): bump `@livekit/protocol` to `^1.46.3` +feat(voice): add `CustomEvent` (proto-native) over remote-session wire -Picks up new agent-session messages: `CustomEvent`, `AgentSessionEvent.custom_event` (livekit/protocol#1588), `AgentSessionEvent.FunctionToolsStarted`, `AgentSessionEvent.EotPrediction`, and `SessionRequest.UpdateIO`. No runtime behavior change in `@livekit/agents` itself — this only makes the new types available for downstream consumers. +Bumps `@livekit/protocol` to `^1.46.3` and wires the new +`AgentSessionEvent.custom_event` (livekit/protocol#1588) through +`SessionHost` / `RemoteSession`. + +The event is forwarded **proto-native**: callers construct and emit a +`pb.CustomEvent` directly; the framework forwards it as-is. No wrapper +type, no dict↔`Struct` round-trip. + +```ts +import { AgentSession as pb } from '@livekit/protocol'; +import { Struct } from '@bufbuild/protobuf'; + +session.emit( + 'custom_event', + new pb.CustomEvent({ + type: 'anomaly_detected', + payload: Struct.fromJson({ score: 0.92 }), + }), +); +``` + +Also unlocks `AgentSessionEvent.FunctionToolsStarted`, +`AgentSessionEvent.EotPrediction`, and `SessionRequest.UpdateIO` for +downstream consumers. diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index 5f4717092..a52f047d6 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: Apache-2.0 import { Mutex } from '@livekit/mutex'; +import type { AgentSession as pb } from '@livekit/protocol'; import type { AudioFrame, Room } from '@livekit/rtc-node'; import { ThrowsPromise } from '@livekit/throws-transformer/throws'; import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter'; @@ -61,7 +62,6 @@ import { type CloseEvent, CloseReason, type ConversationItemAddedEvent, - type CustomEvent, type ErrorEvent, type FunctionToolsExecutedEvent, type MetricsCollectedEvent, @@ -158,7 +158,7 @@ export type AgentSessionCallbacks = { [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void; [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void; [AgentSessionEventTypes.OverlappingSpeech]: (ev: OverlappingSpeechEvent) => void; - [AgentSessionEventTypes.CustomEvent]: (ev: CustomEvent) => void; + [AgentSessionEventTypes.CustomEvent]: (ev: pb.CustomEvent) => void; }; export type AgentSessionOptions = { diff --git a/agents/src/voice/events.ts b/agents/src/voice/events.ts index 65b610a46..06001a07a 100644 --- a/agents/src/voice/events.ts +++ b/agents/src/voice/events.ts @@ -341,26 +341,6 @@ export const createAgentFalseInterruptionEvent = ({ createdAt, }); -export type CustomEvent = { - type: 'custom_event'; - /** Maps to proto `CustomEvent.type`. */ - eventType: string; - /** Maps to proto `CustomEvent.payload` (arbitrary JSON object). */ - payload: Record; - createdAt: number; -}; - -export const createCustomEvent = ( - eventType: string, - payload: Record = {}, - createdAt: number = Date.now(), -): CustomEvent => ({ - type: 'custom_event', - eventType, - payload, - createdAt, -}); - export type AgentEvent = | UserInputTranscribedEvent | UserStateChangedEvent @@ -372,6 +352,5 @@ export type AgentEvent = | SpeechCreatedEvent | AgentFalseInterruptionEvent | OverlappingSpeechEvent - | CustomEvent | ErrorEvent | CloseEvent; diff --git a/agents/src/voice/remote_session.ts b/agents/src/voice/remote_session.ts index 3539742ea..2b8e8ad25 100644 --- a/agents/src/voice/remote_session.ts +++ b/agents/src/voice/remote_session.ts @@ -1,8 +1,7 @@ // SPDX-FileCopyrightText: 2026 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 -import type { JsonValue } from '@bufbuild/protobuf'; -import { Duration, Struct, Timestamp } from '@bufbuild/protobuf'; +import { Duration, Timestamp } from '@bufbuild/protobuf'; import { AgentSession as pb } from '@livekit/protocol'; import type { ByteStreamReader, Room, TextStreamInfo } from '@livekit/rtc-node'; import { ThrowsPromise } from '@livekit/throws-transformer/throws'; @@ -33,7 +32,6 @@ import { type AgentState, type AgentStateChangedEvent, type ConversationItemAddedEvent, - type CustomEvent, type ErrorEvent, type FunctionToolsExecutedEvent, type MetricsCollectedEvent, @@ -77,25 +75,11 @@ export type RemoteSessionCallbacks = { function_tools_executed: (ev: pb.AgentSessionEvent_FunctionToolsExecuted) => void; overlapping_speech: (ev: pb.AgentSessionEvent_OverlappingSpeech) => void; amd_prediction: (ev: pb.AgentSessionEvent_AmdPrediction) => void; - custom_event: (ev: { eventType: string; payload: Record }) => void; + custom_event: (ev: pb.CustomEvent) => void; session_usage: (ev: pb.AgentSessionEvent_SessionUsageUpdated) => void; error: (ev: pb.AgentSessionEvent_Error) => void; }; -function dictToStruct(data: Record): Struct { - // Caller is contractually responsible for JSON-serializable values; `Struct.fromJson` - // validates at runtime. The cast widens `unknown` -> `JsonValue` to satisfy the - // structural mismatch (`Record` is wider than `JsonObject`). - return Struct.fromJson(data as JsonValue); -} - -function structToDict(st: Struct | undefined): Record { - if (!st) { - return {}; - } - return st.toJson() as Record; -} - // =========================================================================== // SessionTransport // =========================================================================== @@ -733,17 +717,8 @@ export class SessionHost { ); }; - private onCustomEvent = (event: CustomEvent): void => { - this.emitEvent( - { - case: 'customEvent', - value: new pb.CustomEvent({ - type: event.eventType, - payload: dictToStruct(event.payload), - }), - }, - event.createdAt, - ); + private onCustomEvent = (event: pb.CustomEvent): void => { + this.emitEvent({ case: 'customEvent', value: event }); }; /** @@ -1041,10 +1016,7 @@ export class RemoteSession extends (EventEmitter as new () => TypedEventEmitter< this.emit('error', ev.value); break; case 'customEvent': - this.emit('custom_event', { - eventType: ev.value.type, - payload: structToDict(ev.value.payload), - }); + this.emit('custom_event', ev.value); break; } } From f9213e77cd0a43979c72d01f5e2268972867b999 Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Tue, 26 May 2026 18:59:30 -0400 Subject: [PATCH 6/7] feat(voice): add AgentSession.emitCustomEvent(type, payload) Sugar for the common case: build pb.CustomEvent from a JSON-like payload and emit it, so callers don't have to import Struct + construct the proto at the call site. session.emitCustomEvent('anomaly_detected', { score: 0.92 }); is equivalent to: import { Struct } from '@bufbuild/protobuf'; import { AgentSession as pb } from '@livekit/protocol'; session.emit( AgentSessionEventTypes.CustomEvent, new pb.CustomEvent({ type: 'anomaly_detected', payload: Struct.fromJson({ score: 0.92 }), }), ); The framework layer stays proto-native (SessionHost / RemoteSession forward pb.CustomEvent as-is); this is purely a call-site convenience. Mirrors the python `AgentSession.emit_custom_event` helper. `tsc --noEmit` clean; lint clean (one pre-existing `any` warning elsewhere in the file is unchanged). Co-authored-by: Cursor --- agents/src/voice/agent_session.ts | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index a52f047d6..c6d52b11b 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -1,8 +1,10 @@ // SPDX-FileCopyrightText: 2024 LiveKit, Inc. // // SPDX-License-Identifier: Apache-2.0 +import type { JsonValue } from '@bufbuild/protobuf'; +import { Struct } from '@bufbuild/protobuf'; import { Mutex } from '@livekit/mutex'; -import type { AgentSession as pb } from '@livekit/protocol'; +import { AgentSession as pb } from '@livekit/protocol'; import type { AudioFrame, Room } from '@livekit/rtc-node'; import { ThrowsPromise } from '@livekit/throws-transformer/throws'; import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter'; @@ -708,6 +710,30 @@ export class AgentSession< return this.activity.interrupt(options); } + /** + * Emit an application-defined `custom_event` over the remote-session wire. + * + * Sugar over constructing `pb.CustomEvent` and emitting it: wraps `payload` + * (a JSON-compatible object) into a `google.protobuf.Struct` and emits the + * proto. Listeners registered via `session.on('custom_event', handler)` + * receive the same `pb.CustomEvent`. + * + * Mirrors python `AgentSession.emit_custom_event`. + */ + emitCustomEvent(eventType: string, payload: Record = {}): void { + // `Struct.fromJson` expects `JsonValue`; `Record` is + // structurally wider than `JsonObject`. Caller is contractually responsible + // for JSON-serializable values; the cast satisfies the type, `fromJson` + // validates at runtime. + this.emit( + AgentSessionEventTypes.CustomEvent, + new pb.CustomEvent({ + type: eventType, + payload: Struct.fromJson(payload as JsonValue), + }), + ); + } + /** * The currently bound `AMD` instance, or `null` if AMD is not in use. * Mirrors python `AgentSession.amd`. From a710d412ef0234fef538d84a42b668f1200f3746 Mon Sep 17 00:00:00 2001 From: Brian Yin Date: Wed, 27 May 2026 15:49:00 -0400 Subject: [PATCH 7/7] refactor(voice): rename CustomEvent -> DebugMessage; bump @livekit/protocol 1.46.4 Picks up livekit/protocol#1593 which renamed the agent-session event before any consumer shipped it. The message is repositioned as an internal debug/trace channel surfaced only to the debugger/recorder (e.g. agents-cli), not to user code. Proto change (mechanical rename): - AgentSessionEvent.custom_event -> AgentSessionEvent.debug_message (field 21) - pb.CustomEvent (string type, Struct payload) -> pb.DebugMessage (Struct payload) Surface change: - AgentSession.emitCustomEvent(type, payload) -> AgentSession._emitDebugMessage(payload) - underscore prefix + `/* internal */` JSDoc signal: not for user code - type discriminator dropped; callers just emit a JSON payload - AgentSessionEventTypes.CustomEvent -> AgentSessionEventTypes.DebugMessage - RemoteSessionCallbacks.custom_event -> debug_message - SessionHost.onCustomEvent -> onDebugMessage - Dispatch oneof case 'customEvent' -> 'debugMessage' Bumps `@livekit/protocol` floor `^1.46.3` -> `^1.46.4`. tsc --noEmit clean; lint clean (only pre-existing test-file `any` warnings). Co-authored-by: Cursor --- .changeset/bump-protocol-1.46.3.md | 30 ------------------- .../bump-protocol-1.46.4-debug-message.md | 16 ++++++++++ agents/package.json | 2 +- agents/src/voice/agent_session.ts | 26 ++++------------ agents/src/voice/events.ts | 2 +- agents/src/voice/remote_session.ts | 16 +++++----- pnpm-lock.yaml | 12 ++++---- 7 files changed, 37 insertions(+), 67 deletions(-) delete mode 100644 .changeset/bump-protocol-1.46.3.md create mode 100644 .changeset/bump-protocol-1.46.4-debug-message.md diff --git a/.changeset/bump-protocol-1.46.3.md b/.changeset/bump-protocol-1.46.3.md deleted file mode 100644 index 3de5ae85f..000000000 --- a/.changeset/bump-protocol-1.46.3.md +++ /dev/null @@ -1,30 +0,0 @@ ---- -'@livekit/agents': patch ---- - -feat(voice): add `CustomEvent` (proto-native) over remote-session wire - -Bumps `@livekit/protocol` to `^1.46.3` and wires the new -`AgentSessionEvent.custom_event` (livekit/protocol#1588) through -`SessionHost` / `RemoteSession`. - -The event is forwarded **proto-native**: callers construct and emit a -`pb.CustomEvent` directly; the framework forwards it as-is. No wrapper -type, no dict↔`Struct` round-trip. - -```ts -import { AgentSession as pb } from '@livekit/protocol'; -import { Struct } from '@bufbuild/protobuf'; - -session.emit( - 'custom_event', - new pb.CustomEvent({ - type: 'anomaly_detected', - payload: Struct.fromJson({ score: 0.92 }), - }), -); -``` - -Also unlocks `AgentSessionEvent.FunctionToolsStarted`, -`AgentSessionEvent.EotPrediction`, and `SessionRequest.UpdateIO` for -downstream consumers. diff --git a/.changeset/bump-protocol-1.46.4-debug-message.md b/.changeset/bump-protocol-1.46.4-debug-message.md new file mode 100644 index 000000000..7f53248f6 --- /dev/null +++ b/.changeset/bump-protocol-1.46.4-debug-message.md @@ -0,0 +1,16 @@ +--- +'@livekit/agents': patch +--- + +internal(voice): wire `DebugMessage` over the remote-session wire + +Bumps `@livekit/protocol` to `^1.46.4` and wires the new +`AgentSessionEvent.debug_message` (livekit/protocol#1593) through +`SessionHost` / `RemoteSession`. + +Internal-only — the framework exposes an unstable `AgentSession._emitDebugMessage(payload)` +for the debugger/recorder; not intended for user code. + +Also unlocks `AgentSessionEvent.FunctionToolsStarted`, +`AgentSessionEvent.EotPrediction`, and `SessionRequest.UpdateIO` for +downstream consumers. diff --git a/agents/package.json b/agents/package.json index 93702f3b5..5277a298f 100644 --- a/agents/package.json +++ b/agents/package.json @@ -53,7 +53,7 @@ "@bufbuild/protobuf": "^1.10.0", "@ffmpeg-installer/ffmpeg": "^1.1.0", "@livekit/mutex": "^1.1.1", - "@livekit/protocol": "^1.46.3", + "@livekit/protocol": "^1.46.4", "@livekit/typed-emitter": "^3.0.0", "@livekit/throws-transformer": "0.1.8", "@opentelemetry/api": "^1.9.0", diff --git a/agents/src/voice/agent_session.ts b/agents/src/voice/agent_session.ts index c6d52b11b..4406b267d 100644 --- a/agents/src/voice/agent_session.ts +++ b/agents/src/voice/agent_session.ts @@ -160,7 +160,7 @@ export type AgentSessionCallbacks = { [AgentSessionEventTypes.Error]: (ev: ErrorEvent) => void; [AgentSessionEventTypes.Close]: (ev: CloseEvent) => void; [AgentSessionEventTypes.OverlappingSpeech]: (ev: OverlappingSpeechEvent) => void; - [AgentSessionEventTypes.CustomEvent]: (ev: pb.CustomEvent) => void; + [AgentSessionEventTypes.DebugMessage]: (ev: pb.DebugMessage) => void; }; export type AgentSessionOptions = { @@ -710,27 +710,11 @@ export class AgentSession< return this.activity.interrupt(options); } - /** - * Emit an application-defined `custom_event` over the remote-session wire. - * - * Sugar over constructing `pb.CustomEvent` and emitting it: wraps `payload` - * (a JSON-compatible object) into a `google.protobuf.Struct` and emits the - * proto. Listeners registered via `session.on('custom_event', handler)` - * receive the same `pb.CustomEvent`. - * - * Mirrors python `AgentSession.emit_custom_event`. - */ - emitCustomEvent(eventType: string, payload: Record = {}): void { - // `Struct.fromJson` expects `JsonValue`; `Record` is - // structurally wider than `JsonObject`. Caller is contractually responsible - // for JSON-serializable values; the cast satisfies the type, `fromJson` - // validates at runtime. + /* internal */ + _emitDebugMessage(payload: Record): void { this.emit( - AgentSessionEventTypes.CustomEvent, - new pb.CustomEvent({ - type: eventType, - payload: Struct.fromJson(payload as JsonValue), - }), + AgentSessionEventTypes.DebugMessage, + new pb.DebugMessage({ payload: Struct.fromJson(payload as JsonValue) }), ); } diff --git a/agents/src/voice/events.ts b/agents/src/voice/events.ts index 06001a07a..481fe8308 100644 --- a/agents/src/voice/events.ts +++ b/agents/src/voice/events.ts @@ -33,7 +33,7 @@ export enum AgentSessionEventTypes { SpeechCreated = 'speech_created', AgentFalseInterruption = 'agent_false_interruption', OverlappingSpeech = 'overlapping_speech', - CustomEvent = 'custom_event', + DebugMessage = 'debug_message', Error = 'error', Close = 'close', } diff --git a/agents/src/voice/remote_session.ts b/agents/src/voice/remote_session.ts index 2b8e8ad25..8db13f456 100644 --- a/agents/src/voice/remote_session.ts +++ b/agents/src/voice/remote_session.ts @@ -62,7 +62,7 @@ export type RemoteSessionEventTypes = | 'function_tools_executed' | 'overlapping_speech' | 'amd_prediction' - | 'custom_event' + | 'debug_message' | 'session_usage' | 'error'; @@ -75,7 +75,7 @@ export type RemoteSessionCallbacks = { function_tools_executed: (ev: pb.AgentSessionEvent_FunctionToolsExecuted) => void; overlapping_speech: (ev: pb.AgentSessionEvent_OverlappingSpeech) => void; amd_prediction: (ev: pb.AgentSessionEvent_AmdPrediction) => void; - custom_event: (ev: pb.CustomEvent) => void; + debug_message: (ev: pb.DebugMessage) => void; session_usage: (ev: pb.AgentSessionEvent_SessionUsageUpdated) => void; error: (ev: pb.AgentSessionEvent_Error) => void; }; @@ -524,7 +524,7 @@ export class SessionHost { session.on(AgentSessionEventTypes.MetricsCollected, this.onMetricsCollected); session.on(AgentSessionEventTypes.OverlappingSpeech, this.onOverlappingSpeech); session.on(AgentSessionEventTypes.Error, this.onHostError); - session.on(AgentSessionEventTypes.CustomEvent, this.onCustomEvent); + session.on(AgentSessionEventTypes.DebugMessage, this.onDebugMessage); } } @@ -553,7 +553,7 @@ export class SessionHost { this.session.off(AgentSessionEventTypes.MetricsCollected, this.onMetricsCollected); this.session.off(AgentSessionEventTypes.OverlappingSpeech, this.onOverlappingSpeech); this.session.off(AgentSessionEventTypes.Error, this.onHostError); - this.session.off(AgentSessionEventTypes.CustomEvent, this.onCustomEvent); + this.session.off(AgentSessionEventTypes.DebugMessage, this.onDebugMessage); } if (this.recvTask) { @@ -717,8 +717,8 @@ export class SessionHost { ); }; - private onCustomEvent = (event: pb.CustomEvent): void => { - this.emitEvent({ case: 'customEvent', value: event }); + private onDebugMessage = (event: pb.DebugMessage): void => { + this.emitEvent({ case: 'debugMessage', value: event }); }; /** @@ -1015,8 +1015,8 @@ export class RemoteSession extends (EventEmitter as new () => TypedEventEmitter< case 'error': this.emit('error', ev.value); break; - case 'customEvent': - this.emit('custom_event', ev.value); + case 'debugMessage': + this.emit('debug_message', ev.value); break; } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index c67ce6d8e..60ae26883 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -116,8 +116,8 @@ importers: specifier: ^1.1.1 version: 1.1.1 '@livekit/protocol': - specifier: ^1.46.3 - version: 1.46.3 + specifier: ^1.46.4 + version: 1.46.4 '@livekit/throws-transformer': specifier: 0.1.8 version: 0.1.8(typescript@5.9.3) @@ -2250,8 +2250,8 @@ packages: cpu: [x64] os: [win32] - '@livekit/protocol@1.46.3': - resolution: {integrity: sha512-YvsE4UN5i+wY9vXfwhF6EUrRyUm/YhiFU1jBcsmsLd/xodUJxYTBcWS4OgL4IJffjzIoyxsrbKp1h9qC55mtcQ==} + '@livekit/protocol@1.46.4': + resolution: {integrity: sha512-yJZ8xvyVcs9CczK2V/EQQrSW0MA9VaZ1vL+FI6fd85KhIjfOg26HvrdUl2LZPT78Tu4R4opV4AW58eN5vgmzqg==} '@livekit/rtc-ffi-bindings-darwin-arm64@0.12.52-patch.0': resolution: {integrity: sha512-IKUir6goV8yVRR7E2qrAP0JtH7gUyMkO0TG8G+dopO/fkXAsPpSealgI9fLcBJl0zhKK+eGCr741r6xR+xxsVw==} @@ -6294,7 +6294,7 @@ snapshots: '@livekit/noise-cancellation-win32-x64@0.1.9': optional: true - '@livekit/protocol@1.46.3': + '@livekit/protocol@1.46.4': dependencies: '@bufbuild/protobuf': 1.10.1 @@ -8624,7 +8624,7 @@ snapshots: livekit-server-sdk@2.14.1: dependencies: '@bufbuild/protobuf': 1.10.1 - '@livekit/protocol': 1.46.3 + '@livekit/protocol': 1.46.4 camelcase-keys: 9.1.3 jose: 5.2.4