diff --git a/CHANGELOG.md b/CHANGELOG.md
index d8e51945..db194d41 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,38 @@
## Unreleased
+### Added
+
+- **xAI (Grok) voice provider suite** — all three xAI Voice APIs land in both
+ SDKs, wired end-to-end with real pricing:
+ - **`XaiRealtime` engine (Grok Voice Agent API)**: real-time speech-to-speech
+ over `wss://api.x.ai/v1/realtime` (default model `grok-voice-latest`, voice
+ `eve`). The protocol is OpenAI-Realtime-GA-compatible, so the adapter
+ subclasses the GA adapter and inherits audio streaming, barge-in, and the
+ full tool-calling bridge (`transfer_call`/`end_call` included). xAI-specific
+ session knobs are exposed opt-in: `reasoning_effort` (`"high"`/`"none"`),
+ VAD `threshold`/`prefix_padding_ms`/`idle_timeout_ms`, ASR
+ `language_hint`/`keyterms`, output `speed`, pronunciation `replace` map,
+ session `resumption`, and raw `server_tools` passthrough for xAI
+ server-side tools (`web_search`, `x_search`, `mcp`, `file_search`).
+ - **`XaiSTT`**: streaming speech-to-text over `wss://api.x.ai/v1/stt`
+ (binary frames, `transcript.created` handshake, chunk/utterance finals,
+ Smart Turn end-of-turn detection, keyterm biasing, diarization) plus a
+ batch `transcribe()` helper for `POST /v1/stt` (word timestamps, ITN
+ formatting, 25 languages).
+ - **`XaiTTS`**: one-shot streaming synthesis via `POST /v1/tts` with the
+ 26-voice Grok roster, `for_twilio()` (native G.711 µ-law 8 kHz) /
+ `for_telnyx()` (PCM16 16 kHz) carrier presets, and a
+ `create_custom_voice()` helper for `POST /v1/custom-voices` voice cloning.
+ - **Pricing** (official, docs.x.ai): Realtime $0.05/min, TTS $15.00/1M chars,
+ STT $0.20/hr streaming ($0.10/hr batch). `calculate_realtime_cost` /
+ `calculateRealtimeCost` gain an optional duration argument to support
+ per-minute realtime billing (token-based path unchanged).
+ `libraries/python/getpatter/providers/xai_{stt,tts,realtime}.py`,
+ `libraries/typescript/src/providers/xai-{stt,tts,realtime}.ts`,
+ `engines/xai.*`, `pricing.*`, docs pages under
+ `docs/{python,typescript}-sdk/providers/xai-*.mdx`. Beta: validated against
+ the xAI API spec and mocked protocol tests; not yet exercised on a live call.
+
### Fixed
- **`gemini-3.1-flash-live-preview` is now actually usable** (field-debugged on
diff --git a/docs/concepts.mdx b/docs/concepts.mdx
index 7b1e68b3..922b8f0c 100644
--- a/docs/concepts.mdx
+++ b/docs/concepts.mdx
@@ -81,6 +81,7 @@ agent = phone.agent(
|---|---|---|
| `OpenAIRealtime()` | Speech-to-speech via OpenAI's Realtime API | Lowest latency, general-purpose |
| `ElevenLabsConvAI()` | Managed conversational agent on ElevenLabs | Premium voice quality |
+| `XaiRealtime()` | Speech-to-speech via xAI's Grok Voice Agent | Reasoning + built-in web / X search |
**Pipeline mode (full control)** — you pick each stage:
@@ -100,9 +101,9 @@ Want fully custom LLM logic (multi-model routing, local models, an internal gate
Three independent stages — swap any of them for a different vendor or local model:
-- **STT (Speech-to-Text)** — transcribes caller audio in real time. Providers: `DeepgramSTT`, `WhisperSTT`, `CartesiaSTT`, `SonioxSTT`, `SpeechmaticsSTT` (Python-only), `AssemblyAISTT`. See [STT](/python-sdk/stt).
+- **STT (Speech-to-Text)** — transcribes caller audio in real time. Providers: `DeepgramSTT`, `WhisperSTT`, `CartesiaSTT`, `SonioxSTT`, `SpeechmaticsSTT` (Python-only), `AssemblyAISTT`, `XaiSTT`. See [STT](/python-sdk/stt).
- **LLM (Large Language Model)** — generates the reply. Pass a class instance via `llm=`: `OpenAILLM`, `AnthropicLLM`, `GroqLLM`, `CerebrasLLM`, `GoogleLLM`. Tool calling works across all five. For anything else, use `on_message`. See [LLM](/python-sdk/llm).
-- **TTS (Text-to-Speech)** — synthesizes the reply audio. Providers: `ElevenLabsTTS`, `OpenAITTS`, `CartesiaTTS`, `RimeTTS`, `LMNTTTS`. See [TTS](/python-sdk/tts).
+- **TTS (Text-to-Speech)** — synthesizes the reply audio. Providers: `ElevenLabsTTS`, `OpenAITTS`, `CartesiaTTS`, `RimeTTS`, `LMNTTTS`, `XaiTTS`. See [TTS](/python-sdk/tts).
Pick engine mode when you want minimum code. Pick pipeline mode when you need a specific LLM, a custom voice, or fine-grained control over latency / costs.
diff --git a/docs/docs.json b/docs/docs.json
index 99ce3976..565aee7a 100644
--- a/docs/docs.json
+++ b/docs/docs.json
@@ -112,7 +112,8 @@
"python-sdk/providers/openai-realtime-2",
"python-sdk/providers/gemini-live",
"python-sdk/providers/ultravox-realtime",
- "python-sdk/providers/elevenlabs-convai"
+ "python-sdk/providers/elevenlabs-convai",
+ "python-sdk/providers/xai-realtime"
]
},
{
@@ -126,7 +127,8 @@
"python-sdk/providers/whisper",
"python-sdk/providers/soniox",
"python-sdk/providers/speechmatics",
- "python-sdk/providers/telnyx-stt"
+ "python-sdk/providers/telnyx-stt",
+ "python-sdk/providers/xai-stt"
]
},
{
@@ -152,7 +154,8 @@
"python-sdk/providers/openai-tts",
"python-sdk/providers/cartesia-tts",
"python-sdk/providers/inworld",
- "python-sdk/providers/telnyx-tts"
+ "python-sdk/providers/telnyx-tts",
+ "python-sdk/providers/xai-tts"
]
},
{
@@ -247,7 +250,8 @@
"typescript-sdk/providers/openai-realtime-2",
"typescript-sdk/providers/gemini-live",
"typescript-sdk/providers/ultravox-realtime",
- "typescript-sdk/providers/elevenlabs-convai"
+ "typescript-sdk/providers/elevenlabs-convai",
+ "typescript-sdk/providers/xai-realtime"
]
},
{
@@ -261,7 +265,8 @@
"typescript-sdk/providers/whisper",
"typescript-sdk/providers/soniox",
"typescript-sdk/providers/speechmatics",
- "typescript-sdk/providers/telnyx-stt"
+ "typescript-sdk/providers/telnyx-stt",
+ "typescript-sdk/providers/xai-stt"
]
},
{
@@ -287,7 +292,8 @@
"typescript-sdk/providers/openai-tts",
"typescript-sdk/providers/cartesia-tts",
"typescript-sdk/providers/inworld",
- "typescript-sdk/providers/telnyx-tts"
+ "typescript-sdk/providers/telnyx-tts",
+ "typescript-sdk/providers/xai-tts"
]
},
{
diff --git a/docs/python-sdk/engines.mdx b/docs/python-sdk/engines.mdx
index 0d304b56..7a460599 100644
--- a/docs/python-sdk/engines.mdx
+++ b/docs/python-sdk/engines.mdx
@@ -8,11 +8,12 @@ icon: "bolt"
An **engine** is an end-to-end speech-to-speech runtime. Pass an engine instance to `phone.agent(engine=...)` and Patter wires the audio stream straight through to the provider — no separate STT or TTS is needed.
-Patter ships with three engine classes today:
+Patter ships several engine classes:
- [`OpenAIRealtime`](#openairealtime) — OpenAI's Realtime API (v1-beta family, `gpt-realtime-mini` / `gpt-realtime` / `gpt-4o-*-realtime-preview`)
- [`OpenAIRealtime2`](#openairealtime2) — OpenAI's GA Realtime API (`gpt-realtime-2`), separate marker because the GA endpoint speaks a different `session.update` wire shape
- [`ElevenLabsConvAI`](#elevenlabsconvai) — ElevenLabs Conversational AI
+- [`XaiRealtime`](#xairealtime) — xAI Grok Voice Agent (OpenAI-GA-compatible)
Each class ships as both a **flat alias** (`from getpatter import OpenAIRealtime`) and a **namespaced** class (`from getpatter.engines import openai` → `openai.Realtime()`). They are equivalent.
@@ -159,6 +160,50 @@ engine = elevenlabs_engine.ConvAI() # reads env
engine = elevenlabs_engine.ConvAI(agent_id="agent_abc123", voice="rachel")
```
+## XaiRealtime
+
+xAI's **Grok Voice Agent** — an OpenAI-Realtime-GA-compatible speech-to-speech engine with on-by-default reasoning and server-side tools (`web_search`, `x_search`, `mcp`, `file_search`).
+
+```python
+import asyncio
+from getpatter import Patter, Twilio, XaiRealtime
+
+phone = Patter(carrier=Twilio(), phone_number="+15550001234") # TWILIO_* from env
+
+agent = phone.agent(
+ engine=XaiRealtime(voice="eve"), # XAI_API_KEY from env
+ system_prompt="You are a friendly receptionist.",
+ first_message="Hello! How can I help?",
+)
+
+async def main():
+ await phone.serve(agent)
+
+asyncio.run(main())
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `api_key` | `str` | `""` | xAI API key. Reads from `XAI_API_KEY` when empty. |
+| `model` | `str` | `"grok-voice-latest"` | Grok voice model. `grok-voice-latest` tracks the flagship (`grok-voice-think-fast-1.0`). |
+| `voice` | `str` | `"eve"` | Built-in or custom voice ID. |
+| `reasoning_effort` | `"high" \| "none" \| None` | `None` | `"high"` (xAI server default) enables reasoning; `"none"` disables it for lower latency. |
+
+Namespaced form:
+
+```python
+from getpatter.engines import xai
+
+engine = xai.XaiRealtime() # reads XAI_API_KEY
+engine = xai.XaiRealtime(voice="eve", reasoning_effort="none")
+```
+
+For the full session-option surface — VAD tuning, language hint, keyterms, pronunciation replacements, session resumption, and server-side tools — see [xAI Realtime — full reference](/python-sdk/providers/xai-realtime).
+
+
+**Beta** — spec-validated, not yet live-call-validated.
+
+
## What's Next
diff --git a/docs/python-sdk/metrics.mdx b/docs/python-sdk/metrics.mdx
index e99ea48b..7b55c8af 100644
--- a/docs/python-sdk/metrics.mdx
+++ b/docs/python-sdk/metrics.mdx
@@ -245,13 +245,16 @@ Provider-level defaults are listed below. Per-model rates live under `DEFAULT_PR
| Cartesia STT (ink-whisper) | per minute | $0.0025 |
| Soniox | per minute | $0.002 |
| Speechmatics (Pro) | per minute | $0.004 |
+| xAI STT (streaming) | per minute | $0.003333 ($0.20/hr) |
| ElevenLabs (`eleven_flash_v2_5`) | per 1k chars | $0.06 |
| OpenAI TTS (`tts-1`) | per 1k chars | $0.015 |
| Cartesia TTS (`sonic-2`) | per 1k chars | $0.030 |
| Rime (`mistv2`) | per 1k chars | $0.030 |
| LMNT (`aurora`) | per 1k chars | $0.050 |
| Inworld (`inworld-tts-2`) | per 1k chars | $0.020 |
+| xAI TTS | per 1k chars | $0.015 |
| OpenAI Realtime (`gpt-realtime-mini` / `gpt-4o-mini-realtime-preview`) | per token | $10/M audio in · $20/M audio out · $0.60/M text in · $2.40/M text out (cached: $0.30/M audio · $0.06/M text) |
+| xAI Realtime (Grok Voice Agent) | per minute | $0.05 ($3.00/hr) |
| Twilio (US inbound local) | per minute | $0.0085 (rounded up to whole minute, per Twilio) |
| Telnyx | per minute | $0.007 |
@@ -271,6 +274,8 @@ Provider-level defaults are listed below. Per-model rates live under `DEFAULT_PR
| OpenAI Transcribe (`openai_transcribe`) | `gpt-4o-transcribe` *(default)* | $0.006/min |
| OpenAI Transcribe | `gpt-4o-mini-transcribe` | $0.003/min |
| OpenAI Transcribe | `whisper-1` | $0.006/min |
+| xAI (`xai`) | streaming *(default)* | $0.003333/min ($0.20/hr) |
+| xAI | batch / REST (`xai_transcribe`) | $0.10/hr (documented; not metered on the pipeline) |
#### TTS — per-model rates
@@ -289,6 +294,7 @@ Provider-level defaults are listed below. Per-model rates live under `DEFAULT_PR
| LMNT | `aurora` *(default)* / `blizzard` | $0.050/1k |
| Inworld | `inworld-tts-2` *(default)* | $0.020/1k |
| Inworld | `inworld-tts-1.5-max` / `inworld-tts-1.5` | $0.025/1k |
+| xAI (`xai_tts`) | default | $0.015/1k ($15.00/1M chars) |
#### OpenAI Realtime — per-model rates
@@ -303,6 +309,15 @@ Provider-level defaults are listed below. Per-model rates live under `DEFAULT_PR
`gpt-4o-realtime-preview` is roughly **10x** the cost of `gpt-realtime-mini` for audio. Switching realtime models has direct billing impact — confirm the model on `agent.realtime.model` matches the rate you expect.
+#### xAI Realtime — rates
+
+Unlike OpenAI Realtime (token-based), the xAI Grok Voice Agent is billed **per minute** of session audio, so Patter meters it from the call's `duration_seconds` rather than token counts.
+
+| Provider | Unit | Rate |
+|----------|------|------|
+| xAI Realtime (`xai_realtime`) | per minute | $0.05/min ($3.00/hr) |
+| xAI Realtime — text input | per message | $0.004 per `conversation.item.create` *(documented constant; not metered by Patter metrics v1)* |
+
Twilio defaults match US **inbound** local. Override `pricing.twilio.price` for US toll-free inbound (~$0.022/min) or US outbound local (~$0.014/min). Default pricing is based on publicly listed provider rates and may become stale — check the provider's pricing page or pass your own overrides for authoritative numbers.
diff --git a/docs/python-sdk/providers/xai-realtime.mdx b/docs/python-sdk/providers/xai-realtime.mdx
new file mode 100644
index 00000000..b1c0ec09
--- /dev/null
+++ b/docs/python-sdk/providers/xai-realtime.mdx
@@ -0,0 +1,216 @@
+---
+title: "xAI Realtime"
+description: "xAI Grok Voice Agent — OpenAI-Realtime-GA-compatible speech-to-speech engine with reasoning, server-side tools, and pronunciation control."
+icon: "bolt"
+---
+
+# xAI Realtime
+
+`XaiRealtime` selects xAI's [Grok Voice Agent API](https://docs.x.ai/) (`wss://api.x.ai/v1/realtime`) as an end-to-end speech-to-speech engine. Pass it as the `engine` on `phone.agent(...)` and Patter wires the audio stream straight through — no separate STT or TTS.
+
+xAI's Voice Agent API is OpenAI-Realtime-GA-compatible: the same `session.update` / `response.create` / streaming-event flow works after swapping the base URL, the API key, and the model. Under the hood `XaiRealtime` reuses Patter's GA realtime adapter and grafts on the xAI-specific session knobs (reasoning effort, language hint, keyterms, output speed, pronunciation replacements, session resumption, and server-side tools), so every feature gate in the call handler — barge-in, tool calling, heartbeat, transcode — fires for xAI with no per-provider branches.
+
+
+**Beta.** The engine is validated against the xAI Voice Agent API spec; it has
+not yet been exercised against a live phone call.
+
+
+## Install
+
+
+```bash Python
+pip install "getpatter[xai]"
+```
+
+```bash TypeScript
+npm install getpatter
+```
+
+
+Set `XAI_API_KEY` in your environment (or pass `api_key`).
+
+## Constructor
+
+
+```python Python
+from getpatter import XaiRealtime
+
+engine = XaiRealtime(
+ model="grok-voice-latest", # default — always the newest voice model
+ voice="eve", # default
+ reasoning_effort="none", # "high" (default) enables reasoning; "none" disables it
+ language_hint="en", # bias input transcription (BCP-47)
+ speed=1.0, # assistant playback speed [0.7, 1.5]
+)
+```
+
+```typescript TypeScript
+import { XaiRealtime } from "getpatter";
+
+const engine = new XaiRealtime({
+ model: "grok-voice-latest", // default — always the newest voice model
+ voice: "eve", // default
+ reasoningEffort: "none", // "high" (default) enables reasoning; "none" disables it
+ languageHint: "en", // bias input transcription (BCP-47)
+ speed: 1.0, // assistant playback speed [0.7, 1.5]
+});
+```
+
+
+## Usage
+
+Pass the engine as the `engine` on `phone.agent(...)`:
+
+
+```python Python
+import asyncio
+from getpatter import Patter, Twilio, XaiRealtime
+
+phone = Patter(carrier=Twilio(), phone_number="+15550001234")
+
+agent = phone.agent(
+ engine=XaiRealtime(voice="eve"), # XAI_API_KEY from env
+ system_prompt="You are a friendly receptionist.",
+ first_message="Hello! How can I help?",
+)
+
+asyncio.run(phone.serve(agent))
+```
+
+```typescript TypeScript
+// npx tsx example.ts
+import { Patter, Twilio, XaiRealtime } from "getpatter";
+
+const phone = new Patter({ carrier: new Twilio(), phoneNumber: "+15550001234" });
+
+const agent = phone.agent({
+ engine: new XaiRealtime({ voice: "eve" }), // XAI_API_KEY from env
+ systemPrompt: "You are a friendly receptionist.",
+ firstMessage: "Hello! How can I help?",
+});
+
+await phone.serve({ agent });
+```
+
+
+## Models
+
+Pass a versioned name to pin a specific release; `grok-voice-latest` always points at the newest voice model.
+
+| Model | Notes |
+|---|---|
+| `grok-voice-latest` (default) | Alias — currently resolves to `grok-voice-think-fast-1.0`. Use it so your app tracks the recommended model. |
+| `grok-voice-think-fast-1.0` | Flagship voice model. Reasoning is on by default (`reasoning_effort="high"`). |
+| `grok-voice-fast-1.0` | Legacy model — **deprecated**. Available for existing integrations; new apps should use `grok-voice-latest`. |
+
+## Session options
+
+All optional with safe defaults; unset knobs are omitted from the wire so xAI applies its own server defaults.
+
+| Python | TypeScript | Default | Notes |
+|---|---|---|---|
+| `api_key` | `apiKey` | — | Reads `XAI_API_KEY` when omitted. |
+| `model` | `model` | `"grok-voice-latest"` | See [Models](#models). |
+| `voice` | `voice` | `"eve"` | Built-in or custom voice ID. |
+| `reasoning_effort` | `reasoningEffort` | `None` | `"high"` enables reasoning (xAI server default); `"none"` disables it for lower latency. |
+| `vad_threshold` | `vadThreshold` | `None` | Server VAD activation threshold `[0.1, 0.9]` (xAI default `0.85`). Higher requires louder audio to trigger. |
+| `silence_duration_ms` | `silenceDurationMs` | `None` | Trailing silence (ms) the VAD waits for before ending the user's turn. |
+| `prefix_padding_ms` | `prefixPaddingMs` | `None` | Audio (ms) captured before detected speech start (xAI default `333`). |
+| `idle_timeout_ms` | `idleTimeoutMs` | `None` | When set, the server re-engages the user after this many ms of silence following an assistant turn. |
+| `language_hint` | `languageHint` | `None` | BCP-47 code biasing input transcription toward one language (e.g. `"es-MX"`). |
+| `keyterms` | `keyterms` | `()` / `[]` | Up to 100 bias terms (≤50 chars each) for transcription. |
+| `speed` | `speed` | `None` | Assistant playback speed multiplier `[0.7, 1.5]`. |
+| `replace` | `replace` | `None` | Pronunciation map applied before TTS, e.g. `{"Acme Mobile": "Acme Mobull"}`. Only the spoken audio changes; the transcript keeps the original. |
+| `resumption` | `resumption` | `False` | Opt in to session resumption — the server caches conversation turns and replays them on reconnect. |
+| `server_tools` | `serverTools` | `()` / `[]` | Raw xAI server-side tool objects — see [Server-side tools](#server-side-tools). |
+
+## Tool calling
+
+Function tools declared on `phone.agent(tools=[...])` work exactly as they do on OpenAI Realtime — Patter's tool bridge forwards the model's function calls and returns your results. The built-in `transfer_call` and `end_call` tools are auto-injected into every agent, so a Grok voice agent can hand off or hang up out of the box.
+
+### Server-side tools
+
+`server_tools` / `serverTools` passes raw xAI tool objects through to `session.tools` verbatim — `web_search`, `x_search`, `mcp`, and `file_search`. These execute **server-side at xAI**; the client never handles their results, and **xAI bills them separately** from Patter's per-minute metering.
+
+
+```python Python
+engine = XaiRealtime(
+ server_tools=(
+ {"type": "web_search"},
+ {"type": "x_search"},
+ {"type": "mcp", "server_url": "https://mcp.example.com/mcp", "server_label": "my-tools"},
+ ),
+)
+```
+
+```typescript TypeScript
+const engine = new XaiRealtime({
+ serverTools: [
+ { type: "web_search" },
+ { type: "x_search" },
+ { type: "mcp", server_url: "https://mcp.example.com/mcp", server_label: "my-tools" },
+ ],
+});
+```
+
+
+## Pronunciation replacements
+
+Fix how the model pronounces brand names or domain terms with `replace` — matched case-insensitively in the model's output and swapped **before** TTS, so only the spoken audio changes:
+
+
+```python Python
+engine = XaiRealtime(
+ replace={"Acme Mobile": "Acme Mobull"},
+)
+```
+
+```typescript TypeScript
+const engine = new XaiRealtime({
+ replace: { "Acme Mobile": "Acme Mobull" },
+});
+```
+
+
+## Telephony audio
+
+Over Twilio / Telnyx the engine inherits Patter's GA-compatible audio path: it negotiates PCM-16-LE @ 24 kHz with xAI and transcodes to / from the carrier's μ-law 8 kHz internally — you don't configure any of this.
+
+
+xAI also supports native G.711 (`audio/pcmu` @ 8 kHz), which would let telephony
+calls skip the transcode entirely. That is a flagged **future optimization**, not
+yet wired into this engine.
+
+
+## When to use xAI Realtime vs alternatives
+
+| Use xAI Realtime when… | Use [OpenAI Realtime](/python-sdk/providers/openai-realtime) when… | Use [Pipeline mode](/python-sdk/agents) when… |
+|---|---|---|
+| You want Grok voices, on-by-default reasoning, and built-in server-side `web_search` / `x_search`. | You need the broadest tool-calling ecosystem and OpenAI's GA reasoning tiers. | You need provider-by-provider control (e.g. `DeepgramSTT` + `AnthropicLLM` + `XaiTTS`). |
+
+## Rates
+
+xAI bills the Grok Voice Agent **per minute of session audio** (provider key `xai_realtime`), not per token:
+
+| Item | Rate |
+|---|---|
+| Voice Agent session audio | $0.05 / min ($3.00 / hr) |
+| Text input messages | $0.004 per `conversation.item.create` — **documented, not metered by Patter metrics v1** |
+
+Override the metered rate per-project via `Patter(pricing={"xai_realtime": {"price": ...}})`. See [Metrics](/python-sdk/metrics) for the full rate table.
+
+## Notes
+
+- **Beta** — the engine is spec-validated, not yet live-call-validated. Pin a versioned model (`grok-voice-think-fast-1.0`) in production for stability.
+- **Caller transcript display (known Beta limitation).** The adapter sets xAI's input-transcription model (`grok-transcribe`), and xAI emits `conversation.item.input_audio_transcription.updated` events rather than OpenAI's `.completed`. This first Beta release does not yet consume the `.updated` variant, so the **caller-side** transcript does not populate the transcript display. Audio, model responses, barge-in, and tool calling are unaffected — and the **assistant-side** transcript still flows; full caller-transcript support is a follow-up.
+- Reasoning is enabled by default. Set `reasoning_effort="none"` to disable it for lower latency on simple flows.
+- `server_tools` results are billed by xAI and never surface to your client code.
+
+## What's Next
+
+
+ All engines side by side.
+ The default engine.
+ System prompts, tools, first messages.
+ Function calling inside a realtime session.
+
diff --git a/docs/python-sdk/providers/xai-stt.mdx b/docs/python-sdk/providers/xai-stt.mdx
new file mode 100644
index 00000000..33d8c9d3
--- /dev/null
+++ b/docs/python-sdk/providers/xai-stt.mdx
@@ -0,0 +1,281 @@
+---
+title: "xAI STT"
+description: "xAI (Grok) real-time streaming speech-to-text plus a one-shot batch transcription helper for Patter pipeline mode."
+icon: "waveform"
+---
+
+# xAI STT
+
+`XaiSTT` streams raw audio to xAI's real-time [Speech-to-Text WebSocket](https://docs.x.ai/) (`wss://api.x.ai/v1/stt`) and yields Patter transcript events. Configuration is carried entirely in the URL query string — there is **no setup message** — and audio is sent as raw binary frames (no base64). The adapter waits for the server's `transcript.created` event before it declares the connection ready, then maps xAI's `is_final` / `speech_final` flags onto Patter's interim / chunk-final / utterance-final semantics, the same two-flag scheme Deepgram uses.
+
+The module also exports a one-shot batch helper — `xai_transcribe` (Python) / `xaiTranscribe` (TypeScript) — that wraps the REST endpoint (`POST https://api.x.ai/v1/stt`) for file or URL transcription with word timestamps.
+
+
+**Beta.** The adapter is validated against the xAI STT API spec; it has not yet
+been exercised against a live phone call.
+
+
+## Install
+
+
+```bash Python
+pip install "getpatter[xai]"
+```
+
+```bash TypeScript
+npm install getpatter
+```
+
+
+The Python adapter uses `aiohttp` (pulled in by the `[xai]` extra). The TypeScript adapter uses the `ws` package and the platform `fetch` — no vendor SDK.
+
+## Authentication
+
+```bash
+export XAI_API_KEY=""
+```
+
+Both the streaming and batch endpoints authenticate with an `Authorization: Bearer ` header. `XAI_API_KEY` is read automatically when `api_key` / `apiKey` is omitted.
+
+## Usage
+
+
+ Use the namespaced import (`getpatter.stt.xai`) or the flat re-export
+ (`XaiSTT`). Both auto-resolve `XAI_API_KEY` from the environment when
+ `api_key` is omitted.
+
+
+
+```python Python
+# Namespaced import (pipeline mode)
+from getpatter.stt import xai
+
+stt = xai.STT() # reads XAI_API_KEY
+stt = xai.STT(api_key="...", language="en", smart_turn=0.7)
+
+# Flat alias (equivalent)
+from getpatter import XaiSTT
+
+stt = XaiSTT()
+```
+
+```typescript TypeScript
+// Namespaced import (pipeline mode)
+import * as xai from "getpatter/stt/xai";
+
+const stt = new xai.STT(); // reads XAI_API_KEY
+const stt2 = new xai.STT({ apiKey: "...", language: "en", smartTurn: 0.7 });
+
+// Flat alias (equivalent)
+import { XaiSTT } from "getpatter";
+
+const stt3 = new XaiSTT();
+```
+
+
+Plug it into an agent:
+
+
+```python Python
+import asyncio
+from getpatter import Patter, Twilio, XaiSTT, XaiTTS
+
+phone = Patter(carrier=Twilio(), phone_number="+15550001234")
+
+agent = phone.agent(
+ stt=XaiSTT(smart_turn=0.7), # XAI_API_KEY from env
+ tts=XaiTTS.for_twilio(voice="eve"),
+ system_prompt="You are a helpful assistant.",
+)
+
+asyncio.run(phone.serve(agent))
+```
+
+```typescript TypeScript
+// npx tsx example.ts
+import { Patter, Twilio, XaiSTT, XaiTTS } from "getpatter";
+
+const phone = new Patter({ carrier: new Twilio(), phoneNumber: "+15550001234" });
+
+const agent = phone.agent({
+ stt: new XaiSTT({ smartTurn: 0.7 }), // XAI_API_KEY from env
+ tts: XaiTTS.forTwilio({ voice: "eve" }),
+ systemPrompt: "You are a helpful assistant.",
+});
+
+await phone.serve({ agent });
+```
+
+
+## Streaming transcript semantics
+
+xAI emits `transcript.partial` events whose `is_final` / `speech_final` flags map onto the transcript states the pipeline reacts to:
+
+| `is_final` | `speech_final` | Patter meaning | Emitted when |
+|---|---|---|---|
+| `false` | `false` | **Interim** — a live partial for barge-in | only with `interim_results` on (~every 500 ms) |
+| `true` | `false` | **Chunk final** — a locked segment (~3 s) | continuous long speech |
+| `true` | `true` | **Utterance final** — end of turn | silence / Smart Turn / `finalize()` |
+
+A trailing `transcript.done` (after `close()` sends `audio.done`) surfaces any remaining text as a final. `error` frames are logged and do **not** close the connection.
+
+The adapter exposes `finalize()`, which sends `{"type": "finalize"}` to force the current utterance to `speech_final` immediately. The pipeline's VAD speech-end fast-path duck-types this — without it every turn waits out the full endpointing delay.
+
+## Smart Turn end-of-turn detection
+
+Set `smart_turn` / `smartTurn` to a confidence threshold in `[0.0, 1.0]` to enable xAI's Smart Turn ML end-of-turn model instead of relying on silence-based endpointing alone. The threshold trades responsiveness against interruption safety:
+
+| Threshold | Behaviour |
+|---|---|
+| Low (~0.1–0.3) | Ends the turn eagerly on weak end-of-turn signals — snappiest, but more likely to cut a speaker off mid-thought. |
+| Medium (~0.4–0.6) | Balanced default for most conversational agents. |
+| High (~0.7–0.9) | Waits for a strong end-of-turn signal — safest against premature cut-offs, slightly higher latency. |
+
+Pair it with `smart_turn_timeout_ms` / `smartTurnTimeoutMs` (`[1, 5000]`) to force `speech_final` after that much silence even when the model is undecided.
+
+
+```python Python
+stt = XaiSTT(smart_turn=0.6, smart_turn_timeout_ms=2000)
+```
+
+```typescript TypeScript
+const stt = new XaiSTT({ smartTurn: 0.6, smartTurnTimeoutMs: 2000 });
+```
+
+
+## Keyterms and diarization
+
+Bias transcription toward domain vocabulary (product names, proper nouns) with up to 100 key terms (≤50 chars each), and turn on speaker diarization so each word carries a `speaker` index:
+
+
+```python Python
+stt = XaiSTT(
+ keyterms=["Patter", "getpatter", "Grok"],
+ diarize=True,
+)
+```
+
+```typescript TypeScript
+const stt = new XaiSTT({
+ keyterms: ["Patter", "getpatter", "Grok"],
+ diarize: true,
+});
+```
+
+
+## Languages
+
+Pass a language code to `language` to enable server-side text formatting (Inverse Text Normalization) for that language. xAI supports 25 codes for the streaming endpoint:
+
+| Code | Language | Code | Language | Code | Language |
+|---|---|---|---|---|---|
+| `ar` | Arabic | `id` | Indonesian | `ro` | Romanian |
+| `cs` | Czech | `it` | Italian | `ru` | Russian |
+| `da` | Danish | `ja` | Japanese | `es` | Spanish |
+| `nl` | Dutch | `ko` | Korean | `sv` | Swedish |
+| `en` | English | `mk` | Macedonian | `th` | Thai |
+| `fil` | Filipino | `ms` | Malay | `tr` | Turkish |
+| `fr` | French | `fa` | Persian | `vi` | Vietnamese |
+| `de` | German | `pl` | Polish | | |
+| `hi` | Hindi | `pt` | Portuguese | | |
+
+## Encodings and sample rate
+
+`encoding` accepts `pcm` (default), `mulaw`, or `alaw`. The default `pcm` @ `sample_rate=16000` matches what Patter's pipeline feeds every STT stage (the carrier bridge decodes inbound μ-law to linear PCM16 @ 16 kHz on both Twilio and Telnyx), so `XaiSTT` drops in without transcoding. Set `mulaw` / `alaw` @ `8000` only when feeding raw carrier audio directly.
+
+## Rates
+
+xAI bills STT by audio duration. Patter meters the streaming rate on the pipeline (provider key `xai`):
+
+| Path | Rate |
+|---|---|
+| Streaming (WebSocket) | $0.20 / hr ($0.003333 / min) |
+| Batch (REST / `xai_transcribe`) | $0.10 / hr |
+
+Override the metered rate per-project via `Patter(pricing={"xai": {"price": ...}})`. See [Metrics](/python-sdk/metrics) for the full rate table.
+
+## Options
+
+| Python | TypeScript | Default | Notes |
+|---|---|---|---|
+| `api_key` | `apiKey` | — | Reads `XAI_API_KEY` when omitted. |
+| `language` | `language` | — | BCP-47 code enabling server-side text formatting. Auto when unset. |
+| `encoding` | `encoding` | `"pcm"` | `pcm` / `mulaw` / `alaw`. |
+| `sample_rate` | `sampleRate` | `16000` | Input sample rate (Hz). |
+| `interim_results` | `interimResults` | `True` | Emit `is_final=false` partials (~every 500 ms). |
+| `endpointing_ms` | `endpointingMs` | `None` | Silence (ms, `[0, 5000]`) before an utterance is final. `None` keeps the xAI default (10); `0` = any VAD boundary. |
+| `smart_turn` | `smartTurn` | `None` | Smart Turn end-of-turn confidence threshold `[0.0, 1.0]`. `None` leaves it off. |
+| `smart_turn_timeout_ms` | `smartTurnTimeoutMs` | `None` | Max silence (ms, `[1, 5000]`) before forcing `speech_final` when Smart Turn is on. |
+| `keyterms` | `keyterms` | `()` | Up to 100 bias terms (≤50 chars each). |
+| `diarize` | `diarize` | `False` | Speaker diarization — words gain a `speaker` index. |
+| `filler_words` | `fillerWords` | `False` | Include "uh" / "um" in the transcript. |
+| `base_url` | `baseUrl` | xAI STT WS endpoint | Override for proxying or tests. |
+
+## Batch transcription
+
+For a one-shot file or URL (rather than a live stream), use the batch helper. Pass either raw `audio` bytes or a `url` for xAI to download server-side. Container formats (WAV/MP3/OGG/Opus/FLAC/AAC/MP4/M4A/MKV) are auto-detected; for raw headerless audio pass `audio_format` (`pcm` / `mulaw` / `alaw`) and `sample_rate`. Set `format=True` (with `language`) for Inverse Text Normalization ("one hundred dollars" → "$100").
+
+
+```python Python
+from getpatter import xai_transcribe
+
+# From bytes
+with open("call.wav", "rb") as f:
+ audio = f.read()
+result = await xai_transcribe(audio, language="en", format=True)
+
+# Or from a URL, with word timestamps + diarization
+from_url = await xai_transcribe(
+ url="https://example.com/recording.mp3",
+ diarize=True,
+)
+
+print(result.text, result.duration)
+for word in result.words:
+ print(word.text, word.start, word.end, word.speaker)
+```
+
+```typescript TypeScript
+import { readFile } from "node:fs/promises";
+import { xaiTranscribe } from "getpatter";
+
+// From bytes
+const audio = await readFile("call.wav");
+const result = await xaiTranscribe({ audio, language: "en", format: true });
+
+// Or from a URL, with word timestamps + diarization
+const fromUrl = await xaiTranscribe({
+ url: "https://example.com/recording.mp3",
+ diarize: true,
+});
+
+console.log(result.text, result.duration);
+for (const word of result.words) {
+ console.log(word.text, word.start, word.end, word.speaker);
+}
+```
+
+
+The helper returns a parsed result — `XaiTranscription(text, language, duration, words)` — where each `XaiWord` carries `text`, `start`, `end`, and (when `diarize` is on) `speaker`.
+
+## Low-level usage
+
+The pipeline-mode wrapper adds `XAI_API_KEY` resolution on top of the underlying provider. To use the provider directly, import it from `getpatter.providers.xai_stt`:
+
+```python
+from getpatter.providers.xai_stt import XaiSTT
+
+stt = XaiSTT("", language="en", smart_turn=0.7)
+await stt.connect()
+async for transcript in stt.receive_transcripts():
+ print(transcript.text, transcript.is_final, transcript.speech_final)
+```
+
+## What's Next
+
+
+ All STT providers side by side.
+ Grok text-to-speech.
+ Grok Voice Agent engine.
+ Cost tracking and rates.
+
diff --git a/docs/python-sdk/providers/xai-tts.mdx b/docs/python-sdk/providers/xai-tts.mdx
new file mode 100644
index 00000000..9554d215
--- /dev/null
+++ b/docs/python-sdk/providers/xai-tts.mdx
@@ -0,0 +1,289 @@
+---
+title: "xAI TTS"
+description: "xAI (Grok) text-to-speech — expressive voices, inline speech tags, and telephony-native output over the one-shot /tts bytes endpoint."
+icon: "waveform"
+---
+
+# xAI TTS
+
+`XaiTTS` targets xAI's one-shot [Text-to-Speech endpoint](https://docs.x.ai/) (`POST https://api.x.ai/v1/tts`). The response body is raw audio bytes in the requested codec, which maps cleanly onto Patter's streaming synthesis contract and keeps the provider dependency-free (just `aiohttp` / `fetch`).
+
+The default voice is **`eve`**, and the default output is **PCM-16-LE @ 16 kHz** so chunks drop straight into the Patter pipeline without transcoding. For phone calls the carrier factories emit telephony-native audio — see [Telephony](#telephony).
+
+
+**Beta.** The adapter is validated against the xAI TTS API spec; it has not yet
+been exercised against a live phone call.
+
+
+## Install
+
+
+```bash Python
+pip install "getpatter[xai]"
+```
+
+```bash TypeScript
+npm install getpatter
+```
+
+
+## Authentication
+
+```bash
+export XAI_API_KEY=""
+```
+
+The REST endpoint authenticates with an `Authorization: Bearer ` header. `XAI_API_KEY` is read automatically when `api_key` / `apiKey` is omitted.
+
+## Usage
+
+
+ Use the namespaced import (`getpatter.tts.xai`) or the flat re-export
+ (`XaiTTS`). Both auto-resolve `XAI_API_KEY` from the environment when
+ `api_key` is omitted.
+
+
+
+```python Python
+# Namespaced import (pipeline mode)
+from getpatter.tts import xai
+
+tts = xai.TTS() # reads XAI_API_KEY, voice "eve"
+tts = xai.TTS(api_key="...", voice="leo", language="en")
+
+# Flat alias (equivalent)
+from getpatter import XaiTTS
+
+tts = XaiTTS()
+```
+
+```typescript TypeScript
+// Namespaced import (pipeline mode)
+import * as xai from "getpatter/tts/xai";
+
+const tts = new xai.TTS(); // reads XAI_API_KEY, voice "eve"
+const tts2 = new xai.TTS({ apiKey: "...", voice: "leo", language: "en" });
+
+// Flat alias (equivalent)
+import { XaiTTS } from "getpatter";
+
+const tts3 = new XaiTTS();
+```
+
+
+Plug it into an agent:
+
+
+```python Python
+import asyncio
+from getpatter import Patter, Twilio, DeepgramSTT, XaiTTS
+
+phone = Patter(carrier=Twilio(), phone_number="+15550001234")
+
+agent = phone.agent(
+ stt=DeepgramSTT(), # DEEPGRAM_API_KEY from env
+ tts=XaiTTS.for_twilio(voice="eve"), # XAI_API_KEY from env
+ system_prompt="You are a helpful assistant.",
+)
+
+asyncio.run(phone.serve(agent))
+```
+
+```typescript TypeScript
+// npx tsx example.ts
+import { Patter, Twilio, DeepgramSTT, XaiTTS } from "getpatter";
+
+const phone = new Patter({ carrier: new Twilio(), phoneNumber: "+15550001234" });
+
+const agent = phone.agent({
+ stt: new DeepgramSTT(), // DEEPGRAM_API_KEY from env
+ tts: XaiTTS.forTwilio({ voice: "eve" }), // XAI_API_KEY from env
+ systemPrompt: "You are a helpful assistant.",
+});
+
+await phone.serve({ agent });
+```
+
+
+## Voices
+
+xAI ships a roster of 26 built-in voices, each with a distinct personality. `eve` is the default; voice IDs are **case-insensitive** (`eve`, `Eve`, and `EVE` all work). Tone and suggested use cases below are from the [xAI Voice Overview](https://docs.x.ai/developers/model-capabilities/audio/voice); preview samples for each voice in the [xAI console playground](https://console.x.ai/).
+
+| Voice | Tone | Suggested use cases |
+|---|---|---|
+| `eve` *(default)* | Energetic and upbeat | — |
+| `altair` | Elegant, refined, and effortlessly premium | Advertising, Narration |
+| `ara` | Warm and friendly | — |
+| `atlas` | Confident, commanding, and reassuring | Sales, Assistant |
+| `carina` | Soft, empathetic, and soothing | Wellness, Support |
+| `castor` | Charismatic, down-to-earth, and easygoing | Sales, Support |
+| `celeste` | Compassionate, confident, and reassuring | Support, Assistant |
+| `cosmo` | Bright, curious, and easy to follow | Education, Podcast |
+| `helios` | Upbeat, energetic, and endlessly versatile | Assistant, Wellness |
+| `helix` | Bold, dynamic, and adrenaline-fueled | Commentary, Podcast |
+| `iris` | Friendly, upbeat, and naturally charming | Sales, Support |
+| `kepler` | Inventive, forward-thinking, and charismatic | Advertising, Podcast |
+| `leo` | Authoritative and strong | — |
+| `lumen` | Warm, articulate, and engaging | Education, Advertising |
+| `luna` | Gentle, patient, and deeply nurturing | Education, Assistant |
+| `lux` | Grounded, calm, and quietly wise | Wellness, Narration |
+| `naksh` | Warm, thoughtful, and wise | Assistant, Support |
+| `orion` | Rich, cinematic, and resonant | Narration, Audiobooks |
+| `perseus` | Strong, confident, and trustworthy | Advertising, Narration |
+| `rex` | Confident and clear | — |
+| `rigel` | Precise, professional, and calmly confident | Assistant, Support |
+| `sal` | Smooth and balanced | — |
+| `sirius` | Quick-witted, clever, and playful | Commentary, Characters |
+| `ursa` | Friendly, warm, and steadfast | Assistant, Podcast |
+| `zagan` | Powerful, dramatic, and unmistakable | Characters, Narration |
+| `zenith` | Sharp, focused, and driven | Sales, Advertising |
+
+### Custom voices
+
+Clone a voice from a short reference clip (WAV, ≤120 s) with the custom-voices helper. It returns a `voice_id` usable as the `voice` on both `XaiTTS` and the [xAI Realtime engine](/python-sdk/providers/xai-realtime):
+
+
+```python Python
+from getpatter import xai_create_custom_voice, XaiTTS
+
+with open("reference.wav", "rb") as f: # ≤120 s
+ clip = f.read()
+voice_id = await xai_create_custom_voice(
+ name="Front Desk",
+ language="en",
+ audio=clip,
+)
+
+tts = XaiTTS(voice=voice_id)
+```
+
+```typescript TypeScript
+import { readFile } from "node:fs/promises";
+import { xaiCreateCustomVoice, XaiTTS } from "getpatter";
+
+const clip = await readFile("reference.wav"); // ≤120 s
+const voiceId = await xaiCreateCustomVoice({
+ name: "Front Desk",
+ language: "en",
+ audio: clip,
+});
+
+const tts = new XaiTTS({ voice: voiceId });
+```
+
+
+## Language
+
+`language` is required by xAI and defaults to `"auto"` (the model detects the language of the text). Pass a BCP-47 code — e.g. `"en"`, `"it"`, `"pt-BR"` — for consistent results. Language validation is case-insensitive.
+
+## Speech tags
+
+xAI supports inline speech tags embedded in the synthesized text for expressive delivery — for example `[pause]` and `[laugh]` for vocal expressions, plus wrapping tags to change delivery style (whispering, singing). Because Patter forwards the reply text verbatim to xAI, any tags your LLM (or system prompt) produces in the text are honored:
+
+
+```python Python
+# The tags travel inside the text your agent speaks.
+tts = XaiTTS(voice="eve", language="en")
+async for chunk in tts.synthesize(
+ "So I walked in and [pause] there it was. [laugh] Incredible!",
+):
+ ... # raw PCM-16-LE @ 16 kHz
+```
+
+```typescript TypeScript
+// The tags travel inside the text your agent speaks.
+const tts = new XaiTTS({ voice: "eve", language: "en" });
+for await (const chunk of tts.synthesizeStream(
+ "So I walked in and [pause] there it was. [laugh] Incredible!",
+)) {
+ // raw PCM-16-LE @ 16 kHz
+}
+```
+
+
+
+A single request accepts up to **15,000 characters**. The adapter warns and
+forwards the text as-is past that limit (the API rejects it); split long text
+upstream — the pipeline already synthesizes per utterance.
+
+
+## Telephony
+
+The constructor default `codec="pcm"` @ `sample_rate=16000` is correct for web playback and 16 kHz pipelines. For real phone calls use the carrier factories, which negotiate the carrier-native codec so the pipeline skips resampling:
+
+
+```python Python
+from getpatter import XaiTTS
+
+# Twilio Media Streams: μ-law @ 8 kHz native — bit-clean passthrough, no resample.
+tts = XaiTTS.for_twilio(voice="eve")
+
+# Telnyx: PCM @ 16 kHz — flows through the standard PCM16 path.
+tts2 = XaiTTS.for_telnyx(voice="eve")
+```
+
+```typescript TypeScript
+import { XaiTTS } from "getpatter";
+
+// Twilio Media Streams: μ-law @ 8 kHz native — bit-clean passthrough, no resample.
+const tts = XaiTTS.forTwilio({ voice: "eve" });
+
+// Telnyx: PCM @ 16 kHz — flows through the standard PCM16 path.
+const tts2 = XaiTTS.forTelnyx({ voice: "eve" });
+```
+
+
+`for_twilio()` emits G.711 μ-law @ 8 kHz — exactly Twilio's wire codec, which xAI supports natively — so the audio passes straight through with zero resampling and zero PCM → μ-law encoding.
+
+## Output formats
+
+Control the codec and sample rate directly when not using the carrier factories:
+
+| `codec` | Best for |
+|---|---|
+| `mp3` | General web use — wide compatibility. |
+| `wav` | Lossless — editing / post-production. |
+| `pcm` (default) | Raw audio — real-time pipelines. |
+| `mulaw` | Telephony (G.711 μ-law). |
+| `alaw` | Telephony (G.711 A-law). |
+
+Supported `sample_rate` values: `8000`, `16000`, `22050`, `24000` (xAI's own default), `44100`, `48000`.
+
+## Rates
+
+xAI bills TTS per character. Patter's default rate (provider key `xai_tts`) is **$0.015 / 1k characters** ($15.00 / 1M chars). Override per-project via `Patter(pricing={"xai_tts": {"price": ...}})`. See [Metrics](/python-sdk/metrics) for the full rate table.
+
+## Options
+
+| Python | TypeScript | Default | Notes |
+|---|---|---|---|
+| `api_key` | `apiKey` | — | Reads `XAI_API_KEY` when omitted. |
+| `voice` | `voice` | `"eve"` | Built-in or custom voice ID (case-insensitive). |
+| `language` | `language` | `"auto"` | BCP-47 tag or `"auto"`. Required by xAI. |
+| `codec` | `codec` | `"pcm"` | `pcm` / `mulaw` / `alaw` / `wav` / `mp3`. |
+| `sample_rate` | `sampleRate` | `16000` | `8000` / `16000` / `22050` / `24000` / `44100` / `48000`. |
+| `speed` | `speed` | `None` | Speaking-rate multiplier in `[0.7, 1.5]`. |
+| `optimize_streaming_latency` | `optimizeStreamingLatency` | `None` | `0` / `1` / `2` — higher trades quality for lower time-to-first-audio. |
+| `text_normalization` | `textNormalization` | `False` | Normalize numbers / abbreviations / symbols to spoken form. |
+| `base_url` | `baseUrl` | xAI `/tts` endpoint | Override for proxying or tests. |
+
+## Low-level usage
+
+The pipeline-mode wrapper adds `XAI_API_KEY` resolution on top of the underlying provider. To use it directly, import from `getpatter.providers.xai_tts`:
+
+```python
+from getpatter.providers.xai_tts import XaiTTS
+
+tts = XaiTTS("", voice="eve", codec="pcm", sample_rate=16000)
+async for chunk in tts.synthesize("Hello from the Patter pipeline."):
+ ... # raw PCM-16-LE @ 16 kHz
+```
+
+## What's Next
+
+
+ All TTS providers side by side.
+ Grok speech-to-text.
+ Grok Voice Agent engine.
+ Cost tracking and rates.
+
diff --git a/docs/python-sdk/stt.mdx b/docs/python-sdk/stt.mdx
index 386a63bf..2efd1684 100644
--- a/docs/python-sdk/stt.mdx
+++ b/docs/python-sdk/stt.mdx
@@ -55,6 +55,7 @@ agent = phone.agent(
| `AssemblyAISTT` | `getpatter.stt.assemblyai.STT` | `ASSEMBLYAI_API_KEY` | `getpatter[assemblyai]` |
| `SonioxSTT` | `getpatter.stt.soniox.STT` | `SONIOX_API_KEY` | `getpatter[soniox]` |
| `SpeechmaticsSTT` | `getpatter.stt.speechmatics.STT` | `SPEECHMATICS_API_KEY` | `getpatter[speechmatics]` |
+| `XaiSTT` | `getpatter.stt.xai.STT` | `XAI_API_KEY` | `getpatter[xai]` |
### Model enums
@@ -175,6 +176,17 @@ from getpatter.stt import speechmatics
stt = speechmatics.STT() # reads SPEECHMATICS_API_KEY
```
+## xAI
+
+Real-time streaming STT via xAI (Grok), plus a one-shot batch transcription helper. Supports Smart Turn end-of-turn detection, keyterm biasing, and speaker diarization. See [xAI STT setup](/python-sdk/providers/xai-stt).
+
+```python
+from getpatter import XaiSTT
+
+stt = XaiSTT() # reads XAI_API_KEY
+stt = XaiSTT(smart_turn=0.7, language="en")
+```
+
## Missing credentials
Each class raises `ValueError` at construction time if no API key is resolved from either `api_key=` or the matching env var:
diff --git a/docs/python-sdk/tts.mdx b/docs/python-sdk/tts.mdx
index 2bfa6bea..01da0772 100644
--- a/docs/python-sdk/tts.mdx
+++ b/docs/python-sdk/tts.mdx
@@ -53,6 +53,7 @@ agent = phone.agent(
| `CartesiaTTS` | `getpatter.tts.cartesia.TTS` | `CARTESIA_API_KEY` | `getpatter[cartesia]` |
| `RimeTTS` | `getpatter.tts.rime.TTS` | `RIME_API_KEY` | `getpatter[rime]` |
| `LMNTTTS` | `getpatter.tts.lmnt.TTS` | `LMNT_API_KEY` | `getpatter[lmnt]` |
+| `XaiTTS` | `getpatter.tts.xai.TTS` | `XAI_API_KEY` | `getpatter[xai]` |
### Model / voice / format enums
@@ -180,6 +181,18 @@ tts = LMNTTTS() # reads LMNT_API_KEY
tts = LMNTTTS(model="blizzard", voice="leah")
```
+## xAI
+
+Grok text-to-speech (default voice `eve`) with 26 built-in voices, inline speech tags, custom voice cloning, and telephony-native output. See [xAI TTS setup](/python-sdk/providers/xai-tts).
+
+```python
+from getpatter import XaiTTS
+
+tts = XaiTTS() # reads XAI_API_KEY
+tts = XaiTTS(voice="leo", language="en")
+tts = XaiTTS.for_twilio(voice="eve") # μ-law @ 8 kHz native
+```
+
## Missing credentials
Each class raises `ValueError` at construction time if no API key is resolved:
diff --git a/docs/typescript-sdk/engines.mdx b/docs/typescript-sdk/engines.mdx
index a7304955..f5ad08c1 100644
--- a/docs/typescript-sdk/engines.mdx
+++ b/docs/typescript-sdk/engines.mdx
@@ -8,13 +8,14 @@ icon: "bolt"
An **engine** is an end-to-end speech-to-speech runtime. Pass an engine instance to `phone.agent({ engine })` and Patter wires the audio stream straight through to the provider — no separate STT or TTS is needed.
-Patter ships with three engine classes today:
+Patter ships several engine classes:
- [`OpenAIRealtime`](#openairealtime) — OpenAI's Realtime API (beta endpoint)
- [`OpenAIRealtime2`](#openairealtime2) — OpenAI's GA Realtime API (targets `gpt-realtime-2`)
- [`ElevenLabsConvAI`](#elevenlabsconvai) — ElevenLabs Conversational AI
+- [`XaiRealtime`](#xairealtime) — xAI Grok Voice Agent (OpenAI-GA-compatible)
-All three classes are imported by name from the package barrel: `import { OpenAIRealtime, OpenAIRealtime2, ElevenLabsConvAI } from "getpatter"`.
+Each class is imported by name from the package barrel: `import { OpenAIRealtime, OpenAIRealtime2, ElevenLabsConvAI, XaiRealtime } from "getpatter"`.
If you need full control over STT, LLM, and TTS independently, use [pipeline mode](/typescript-sdk/llm#pipeline-mode) instead and omit `engine`.
@@ -123,6 +124,38 @@ await phone.serve({ agent });
| `agentId` | `string` | — | ElevenLabs agent ID (from the ConvAI dashboard). Reads from `ELEVENLABS_AGENT_ID` when omitted. |
| `voice` | `string` | — | Optional override for the agent's default voice ID. |
+## XaiRealtime
+
+xAI's **Grok Voice Agent** — an OpenAI-Realtime-GA-compatible speech-to-speech engine with on-by-default reasoning and server-side tools (`web_search`, `x_search`, `mcp`, `file_search`).
+
+```typescript
+// npx tsx example.ts
+import { Patter, Twilio, XaiRealtime } from "getpatter";
+
+const phone = new Patter({ carrier: new Twilio(), phoneNumber: "+15550001234" });
+
+const agent = phone.agent({
+ engine: new XaiRealtime({ voice: "eve" }), // XAI_API_KEY from env
+ systemPrompt: "You are a friendly receptionist.",
+ firstMessage: "Hello! How can I help?",
+});
+
+await phone.serve({ agent });
+```
+
+| Parameter | Type | Default | Description |
+|-----------|------|---------|-------------|
+| `apiKey` | `string` | — | xAI API key. Reads from `XAI_API_KEY` when omitted. |
+| `model` | `string` | `"grok-voice-latest"` | Grok voice model. `grok-voice-latest` tracks the flagship (`grok-voice-think-fast-1.0`). |
+| `voice` | `string` | `"eve"` | Built-in or custom voice ID. |
+| `reasoningEffort` | `"high" \| "none"` | — | `"high"` (xAI server default) enables reasoning; `"none"` disables it for lower latency. |
+
+For the full session-option surface — VAD tuning, language hint, keyterms, pronunciation replacements, session resumption, and server-side tools — see [xAI Realtime — full reference](/typescript-sdk/providers/xai-realtime).
+
+
+**Beta** — spec-validated, not yet live-call-validated.
+
+
## What's Next
diff --git a/docs/typescript-sdk/metrics.mdx b/docs/typescript-sdk/metrics.mdx
index adc6ca69..801d7827 100644
--- a/docs/typescript-sdk/metrics.mdx
+++ b/docs/typescript-sdk/metrics.mdx
@@ -256,13 +256,16 @@ Provider-level defaults are listed below. Per-model rates live under `DEFAULT_PR
| Cartesia STT (ink-whisper) | per minute | $0.0025 |
| Soniox | per minute | $0.002 |
| Speechmatics (Pro) | per minute | $0.004 |
+| xAI STT (streaming) | per minute | $0.003333 ($0.20/hr) |
| ElevenLabs (`eleven_flash_v2_5`) | per 1k chars | $0.06 |
| OpenAI TTS (`tts-1`) | per 1k chars | $0.015 |
| Cartesia TTS (`sonic-2`) | per 1k chars | $0.030 |
| Rime (`mistv2`) | per 1k chars | $0.030 |
| LMNT (`aurora`) | per 1k chars | $0.050 |
| Inworld (`inworld-tts-2`) | per 1k chars | $0.020 |
+| xAI TTS | per 1k chars | $0.015 |
| OpenAI Realtime (`gpt-realtime-mini` / `gpt-4o-mini-realtime-preview`) | per token | $10/M audio in · $20/M audio out · $0.60/M text in · $2.40/M text out (cached: $0.30/M audio · $0.06/M text) |
+| xAI Realtime (Grok Voice Agent) | per minute | $0.05 ($3.00/hr) |
| Twilio (US inbound local) | per minute | $0.0085 (rounded up to whole minute, per Twilio) |
| Telnyx | per minute | $0.007 |
@@ -282,6 +285,8 @@ Provider-level defaults are listed below. Per-model rates live under `DEFAULT_PR
| OpenAI Transcribe (`openai_transcribe`) | `gpt-4o-transcribe` *(default)* | $0.006/min |
| OpenAI Transcribe | `gpt-4o-mini-transcribe` | $0.003/min |
| OpenAI Transcribe | `whisper-1` | $0.006/min |
+| xAI (`xai`) | streaming *(default)* | $0.003333/min ($0.20/hr) |
+| xAI | batch / REST (`xaiTranscribe`) | $0.10/hr (documented; not metered on the pipeline) |
#### TTS — per-model rates
@@ -300,6 +305,7 @@ Provider-level defaults are listed below. Per-model rates live under `DEFAULT_PR
| LMNT | `aurora` *(default)* / `blizzard` | $0.050/1k |
| Inworld | `inworld-tts-2` *(default)* | $0.020/1k |
| Inworld | `inworld-tts-1.5-max` / `inworld-tts-1.5` | $0.025/1k |
+| xAI (`xai_tts`) | default | $0.015/1k ($15.00/1M chars) |
#### OpenAI Realtime — per-model rates
@@ -314,6 +320,15 @@ Provider-level defaults are listed below. Per-model rates live under `DEFAULT_PR
`gpt-4o-realtime-preview` is roughly **10x** the cost of `gpt-realtime-mini` for audio. Switching realtime models has direct billing impact — confirm the model on `agent.realtime.model` matches the rate you expect.
+#### xAI Realtime — rates
+
+Unlike OpenAI Realtime (token-based), the xAI Grok Voice Agent is billed **per minute** of session audio, so Patter meters it from the call's `durationSeconds` rather than token counts.
+
+| Provider | Unit | Rate |
+|----------|------|------|
+| xAI Realtime (`xai_realtime`) | per minute | $0.05/min ($3.00/hr) |
+| xAI Realtime — text input | per message | $0.004 per `conversation.item.create` *(documented constant; not metered by Patter metrics v1)* |
+
Twilio defaults match US **inbound** local. Override `pricing.twilio.price` for US toll-free inbound (~$0.022/min) or US outbound local (~$0.014/min). Default pricing is based on publicly listed provider rates and may become stale — check the provider's pricing page or pass your own overrides for authoritative numbers.
diff --git a/docs/typescript-sdk/providers/xai-realtime.mdx b/docs/typescript-sdk/providers/xai-realtime.mdx
new file mode 100644
index 00000000..57bc89e1
--- /dev/null
+++ b/docs/typescript-sdk/providers/xai-realtime.mdx
@@ -0,0 +1,216 @@
+---
+title: "xAI Realtime"
+description: "xAI Grok Voice Agent — OpenAI-Realtime-GA-compatible speech-to-speech engine with reasoning, server-side tools, and pronunciation control."
+icon: "bolt"
+---
+
+# xAI Realtime
+
+`XaiRealtime` selects xAI's [Grok Voice Agent API](https://docs.x.ai/) (`wss://api.x.ai/v1/realtime`) as an end-to-end speech-to-speech engine. Pass it as the `engine` on `phone.agent({...})` and Patter wires the audio stream straight through — no separate STT or TTS.
+
+xAI's Voice Agent API is OpenAI-Realtime-GA-compatible: the same `session.update` / `response.create` / streaming-event flow works after swapping the base URL, the API key, and the model. Under the hood `XaiRealtime` reuses Patter's GA realtime adapter and grafts on the xAI-specific session knobs (reasoning effort, language hint, keyterms, output speed, pronunciation replacements, session resumption, and server-side tools), so every feature gate in the call handler — barge-in, tool calling, heartbeat, transcode — fires for xAI with no per-provider branches.
+
+
+**Beta.** The engine is validated against the xAI Voice Agent API spec; it has
+not yet been exercised against a live phone call.
+
+
+## Install
+
+
+```bash TypeScript
+npm install getpatter
+```
+
+```bash Python
+pip install "getpatter[xai]"
+```
+
+
+Set `XAI_API_KEY` in your environment (or pass `apiKey`).
+
+## Constructor
+
+
+```typescript TypeScript
+import { XaiRealtime } from "getpatter";
+
+const engine = new XaiRealtime({
+ model: "grok-voice-latest", // default — always the newest voice model
+ voice: "eve", // default
+ reasoningEffort: "none", // "high" (default) enables reasoning; "none" disables it
+ languageHint: "en", // bias input transcription (BCP-47)
+ speed: 1.0, // assistant playback speed [0.7, 1.5]
+});
+```
+
+```python Python
+from getpatter import XaiRealtime
+
+engine = XaiRealtime(
+ model="grok-voice-latest", # default — always the newest voice model
+ voice="eve", # default
+ reasoning_effort="none", # "high" (default) enables reasoning; "none" disables it
+ language_hint="en", # bias input transcription (BCP-47)
+ speed=1.0, # assistant playback speed [0.7, 1.5]
+)
+```
+
+
+## Usage
+
+Pass the engine as the `engine` on `phone.agent({...})`:
+
+
+```typescript TypeScript
+// npx tsx example.ts
+import { Patter, Twilio, XaiRealtime } from "getpatter";
+
+const phone = new Patter({ carrier: new Twilio(), phoneNumber: "+15550001234" });
+
+const agent = phone.agent({
+ engine: new XaiRealtime({ voice: "eve" }), // XAI_API_KEY from env
+ systemPrompt: "You are a friendly receptionist.",
+ firstMessage: "Hello! How can I help?",
+});
+
+await phone.serve({ agent });
+```
+
+```python Python
+import asyncio
+from getpatter import Patter, Twilio, XaiRealtime
+
+phone = Patter(carrier=Twilio(), phone_number="+15550001234")
+
+agent = phone.agent(
+ engine=XaiRealtime(voice="eve"), # XAI_API_KEY from env
+ system_prompt="You are a friendly receptionist.",
+ first_message="Hello! How can I help?",
+)
+
+asyncio.run(phone.serve(agent))
+```
+
+
+## Models
+
+Pass a versioned name to pin a specific release; `grok-voice-latest` always points at the newest voice model.
+
+| Model | Notes |
+|---|---|
+| `grok-voice-latest` (default) | Alias — currently resolves to `grok-voice-think-fast-1.0`. Use it so your app tracks the recommended model. |
+| `grok-voice-think-fast-1.0` | Flagship voice model. Reasoning is on by default (`reasoningEffort: "high"`). |
+| `grok-voice-fast-1.0` | Legacy model — **deprecated**. Available for existing integrations; new apps should use `grok-voice-latest`. |
+
+## Session options
+
+All optional with safe defaults; unset knobs are omitted from the wire so xAI applies its own server defaults.
+
+| TypeScript | Python | Default | Notes |
+|---|---|---|---|
+| `apiKey` | `api_key` | — | Reads `XAI_API_KEY` when omitted. |
+| `model` | `model` | `"grok-voice-latest"` | See [Models](#models). |
+| `voice` | `voice` | `"eve"` | Built-in or custom voice ID. |
+| `reasoningEffort` | `reasoning_effort` | — | `"high"` enables reasoning (xAI server default); `"none"` disables it for lower latency. |
+| `vadThreshold` | `vad_threshold` | — | Server VAD activation threshold `[0.1, 0.9]` (xAI default `0.85`). Higher requires louder audio to trigger. |
+| `silenceDurationMs` | `silence_duration_ms` | — | Trailing silence (ms) the VAD waits for before ending the user's turn. |
+| `prefixPaddingMs` | `prefix_padding_ms` | — | Audio (ms) captured before detected speech start (xAI default `333`). |
+| `idleTimeoutMs` | `idle_timeout_ms` | — | When set, the server re-engages the user after this many ms of silence following an assistant turn. |
+| `languageHint` | `language_hint` | — | BCP-47 code biasing input transcription toward one language (e.g. `"es-MX"`). |
+| `keyterms` | `keyterms` | `[]` / `()` | Up to 100 bias terms (≤50 chars each) for transcription. |
+| `speed` | `speed` | — | Assistant playback speed multiplier `[0.7, 1.5]`. |
+| `replace` | `replace` | — | Pronunciation map applied before TTS, e.g. `{ "Acme Mobile": "Acme Mobull" }`. Only the spoken audio changes; the transcript keeps the original. |
+| `resumption` | `resumption` | `false` | Opt in to session resumption — the server caches conversation turns and replays them on reconnect. |
+| `serverTools` | `server_tools` | `[]` / `()` | Raw xAI server-side tool objects — see [Server-side tools](#server-side-tools). |
+
+## Tool calling
+
+Function tools declared on `phone.agent({ tools: [...] })` work exactly as they do on OpenAI Realtime — Patter's tool bridge forwards the model's function calls and returns your results. The built-in `transfer_call` and `end_call` tools are auto-injected into every agent, so a Grok voice agent can hand off or hang up out of the box.
+
+### Server-side tools
+
+`serverTools` / `server_tools` passes raw xAI tool objects through to `session.tools` verbatim — `web_search`, `x_search`, `mcp`, and `file_search`. These execute **server-side at xAI**; the client never handles their results, and **xAI bills them separately** from Patter's per-minute metering.
+
+
+```typescript TypeScript
+const engine = new XaiRealtime({
+ serverTools: [
+ { type: "web_search" },
+ { type: "x_search" },
+ { type: "mcp", server_url: "https://mcp.example.com/mcp", server_label: "my-tools" },
+ ],
+});
+```
+
+```python Python
+engine = XaiRealtime(
+ server_tools=(
+ {"type": "web_search"},
+ {"type": "x_search"},
+ {"type": "mcp", "server_url": "https://mcp.example.com/mcp", "server_label": "my-tools"},
+ ),
+)
+```
+
+
+## Pronunciation replacements
+
+Fix how the model pronounces brand names or domain terms with `replace` / `replace` — matched case-insensitively in the model's output and swapped **before** TTS, so only the spoken audio changes:
+
+
+```typescript TypeScript
+const engine = new XaiRealtime({
+ replace: { "Acme Mobile": "Acme Mobull" },
+});
+```
+
+```python Python
+engine = XaiRealtime(
+ replace={"Acme Mobile": "Acme Mobull"},
+)
+```
+
+
+## Telephony audio
+
+Over Twilio / Telnyx the engine inherits Patter's GA-compatible audio path: it negotiates PCM-16-LE @ 24 kHz with xAI and transcodes to / from the carrier's μ-law 8 kHz internally — you don't configure any of this.
+
+
+xAI also supports native G.711 (`audio/pcmu` @ 8 kHz), which would let telephony
+calls skip the transcode entirely. That is a flagged **future optimization**, not
+yet wired into this engine.
+
+
+## When to use xAI Realtime vs alternatives
+
+| Use xAI Realtime when… | Use [OpenAI Realtime](/typescript-sdk/providers/openai-realtime) when… | Use [Pipeline mode](/typescript-sdk/agents) when… |
+|---|---|---|
+| You want Grok voices, on-by-default reasoning, and built-in server-side `web_search` / `x_search`. | You need the broadest tool-calling ecosystem and OpenAI's GA reasoning tiers. | You need provider-by-provider control (e.g. `DeepgramSTT` + `AnthropicLLM` + `XaiTTS`). |
+
+## Rates
+
+xAI bills the Grok Voice Agent **per minute of session audio** (provider key `xai_realtime`), not per token:
+
+| Item | Rate |
+|---|---|
+| Voice Agent session audio | $0.05 / min ($3.00 / hr) |
+| Text input messages | $0.004 per `conversation.item.create` — **documented, not metered by Patter metrics v1** |
+
+Override the metered rate per-project via `new Patter({ pricing: { xai_realtime: { price: ... } } })`. See [Metrics](/typescript-sdk/metrics) for the full rate table.
+
+## Notes
+
+- **Beta** — the engine is spec-validated, not yet live-call-validated. Pin a versioned model (`grok-voice-think-fast-1.0`) in production for stability.
+- **Caller transcript display (known Beta limitation).** The adapter sets xAI's input-transcription model (`grok-transcribe`), and xAI emits `conversation.item.input_audio_transcription.updated` events rather than OpenAI's `.completed`. This first Beta release does not yet consume the `.updated` variant, so the **caller-side** transcript does not populate the transcript display. Audio, model responses, barge-in, and tool calling are unaffected — and the **assistant-side** transcript still flows; full caller-transcript support is a follow-up.
+- Reasoning is enabled by default. Set `reasoningEffort: "none"` to disable it for lower latency on simple flows.
+- `serverTools` results are billed by xAI and never surface to your client code.
+
+## What's Next
+
+
+ All engines side by side.
+ The default engine.
+ System prompts, tools, first messages.
+ Function calling inside a realtime session.
+
diff --git a/docs/typescript-sdk/providers/xai-stt.mdx b/docs/typescript-sdk/providers/xai-stt.mdx
new file mode 100644
index 00000000..6bdbd659
--- /dev/null
+++ b/docs/typescript-sdk/providers/xai-stt.mdx
@@ -0,0 +1,280 @@
+---
+title: "xAI STT"
+description: "xAI (Grok) real-time streaming speech-to-text plus a one-shot batch transcription helper for Patter pipeline mode."
+icon: "waveform"
+---
+
+# xAI STT
+
+`XaiSTT` streams raw audio to xAI's real-time [Speech-to-Text WebSocket](https://docs.x.ai/) (`wss://api.x.ai/v1/stt`) and yields Patter transcript events. Configuration is carried entirely in the URL query string — there is **no setup message** — and audio is sent as raw binary frames (no base64). The adapter waits for the server's `transcript.created` event before it declares the connection ready, then maps xAI's `is_final` / `speech_final` flags onto Patter's interim / chunk-final / utterance-final semantics, the same two-flag scheme Deepgram uses.
+
+The module also exports a one-shot batch helper — `xaiTranscribe` (TypeScript) / `xai_transcribe` (Python) — that wraps the REST endpoint (`POST https://api.x.ai/v1/stt`) for file or URL transcription with word timestamps.
+
+
+**Beta.** The adapter is validated against the xAI STT API spec; it has not yet
+been exercised against a live phone call.
+
+
+## Install
+
+
+```bash TypeScript
+npm install getpatter
+```
+
+```bash Python
+pip install "getpatter[xai]"
+```
+
+
+The TypeScript adapter uses the `ws` package and the platform `fetch` — no vendor SDK. The Python adapter uses `aiohttp` (pulled in by the `[xai]` extra).
+
+## Authentication
+
+```bash
+export XAI_API_KEY=""
+```
+
+Both the streaming and batch endpoints authenticate with an `Authorization: Bearer ` header. `XAI_API_KEY` is read automatically when `apiKey` / `api_key` is omitted.
+
+## Usage
+
+
+ Use the namespaced import (`getpatter/stt/xai`) or the flat re-export
+ (`XaiSTT`). Both auto-resolve `XAI_API_KEY` from the environment when
+ `apiKey` is omitted.
+
+
+
+```typescript TypeScript
+// Namespaced import (pipeline mode)
+import * as xai from "getpatter/stt/xai";
+
+const stt = new xai.STT(); // reads XAI_API_KEY
+const stt2 = new xai.STT({ apiKey: "...", language: "en", smartTurn: 0.7 });
+
+// Flat alias (equivalent)
+import { XaiSTT } from "getpatter";
+
+const stt3 = new XaiSTT();
+```
+
+```python Python
+# Namespaced import (pipeline mode)
+from getpatter.stt import xai
+
+stt = xai.STT() # reads XAI_API_KEY
+stt = xai.STT(api_key="...", language="en", smart_turn=0.7)
+
+# Flat alias (equivalent)
+from getpatter import XaiSTT
+
+stt = XaiSTT()
+```
+
+
+Plug it into an agent:
+
+
+```typescript TypeScript
+// npx tsx example.ts
+import { Patter, Twilio, XaiSTT, XaiTTS } from "getpatter";
+
+const phone = new Patter({ carrier: new Twilio(), phoneNumber: "+15550001234" });
+
+const agent = phone.agent({
+ stt: new XaiSTT({ smartTurn: 0.7 }), // XAI_API_KEY from env
+ tts: XaiTTS.forTwilio({ voice: "eve" }),
+ systemPrompt: "You are a helpful assistant.",
+});
+
+await phone.serve({ agent });
+```
+
+```python Python
+import asyncio
+from getpatter import Patter, Twilio, XaiSTT, XaiTTS
+
+phone = Patter(carrier=Twilio(), phone_number="+15550001234")
+
+agent = phone.agent(
+ stt=XaiSTT(smart_turn=0.7), # XAI_API_KEY from env
+ tts=XaiTTS.for_twilio(voice="eve"),
+ system_prompt="You are a helpful assistant.",
+)
+
+asyncio.run(phone.serve(agent))
+```
+
+
+## Streaming transcript semantics
+
+xAI emits `transcript.partial` events whose `is_final` / `speech_final` flags map onto the transcript states the pipeline reacts to:
+
+| `is_final` | `speech_final` | Patter meaning | Emitted when |
+|---|---|---|---|
+| `false` | `false` | **Interim** — a live partial for barge-in | only with `interimResults` on (~every 500 ms) |
+| `true` | `false` | **Chunk final** — a locked segment (~3 s) | continuous long speech |
+| `true` | `true` | **Utterance final** — end of turn | silence / Smart Turn / `finalize()` |
+
+A trailing `transcript.done` (after `close()` sends `audio.done`) surfaces any remaining text as a final. `error` frames are logged and do **not** close the connection.
+
+The adapter exposes `finalize()` (Python `finalize()`), which sends `{"type": "finalize"}` to force the current utterance to `speech_final` immediately. The pipeline's VAD speech-end fast-path duck-types this — without it every turn waits out the full endpointing delay.
+
+## Smart Turn end-of-turn detection
+
+Set `smartTurn` / `smart_turn` to a confidence threshold in `[0.0, 1.0]` to enable xAI's Smart Turn ML end-of-turn model instead of relying on silence-based endpointing alone. The threshold trades responsiveness against interruption safety:
+
+| Threshold | Behaviour |
+|---|---|
+| Low (~0.1–0.3) | Ends the turn eagerly on weak end-of-turn signals — snappiest, but more likely to cut a speaker off mid-thought. |
+| Medium (~0.4–0.6) | Balanced default for most conversational agents. |
+| High (~0.7–0.9) | Waits for a strong end-of-turn signal — safest against premature cut-offs, slightly higher latency. |
+
+Pair it with `smartTurnTimeoutMs` / `smart_turn_timeout_ms` (`[1, 5000]`) to force `speech_final` after that much silence even when the model is undecided.
+
+
+```typescript TypeScript
+const stt = new XaiSTT({ smartTurn: 0.6, smartTurnTimeoutMs: 2000 });
+```
+
+```python Python
+stt = XaiSTT(smart_turn=0.6, smart_turn_timeout_ms=2000)
+```
+
+
+## Keyterms and diarization
+
+Bias transcription toward domain vocabulary (product names, proper nouns) with up to 100 key terms (≤50 chars each), and turn on speaker diarization so each word carries a `speaker` index:
+
+
+```typescript TypeScript
+const stt = new XaiSTT({
+ keyterms: ["Patter", "getpatter", "Grok"],
+ diarize: true,
+});
+```
+
+```python Python
+stt = XaiSTT(
+ keyterms=["Patter", "getpatter", "Grok"],
+ diarize=True,
+)
+```
+
+
+## Languages
+
+Pass a language code to `language` to enable server-side text formatting (Inverse Text Normalization) for that language. xAI supports 25 codes for the streaming endpoint:
+
+| Code | Language | Code | Language | Code | Language |
+|---|---|---|---|---|---|
+| `ar` | Arabic | `id` | Indonesian | `ro` | Romanian |
+| `cs` | Czech | `it` | Italian | `ru` | Russian |
+| `da` | Danish | `ja` | Japanese | `es` | Spanish |
+| `nl` | Dutch | `ko` | Korean | `sv` | Swedish |
+| `en` | English | `mk` | Macedonian | `th` | Thai |
+| `fil` | Filipino | `ms` | Malay | `tr` | Turkish |
+| `fr` | French | `fa` | Persian | `vi` | Vietnamese |
+| `de` | German | `pl` | Polish | | |
+| `hi` | Hindi | `pt` | Portuguese | | |
+
+## Encodings and sample rate
+
+`encoding` accepts `pcm` (default), `mulaw`, or `alaw`. The default `pcm` @ `sampleRate=16000` matches what Patter's pipeline feeds every STT stage (the carrier bridge decodes inbound μ-law to linear PCM16 @ 16 kHz on both Twilio and Telnyx), so `XaiSTT` drops in without transcoding. Set `mulaw` / `alaw` @ `8000` only when feeding raw carrier audio directly.
+
+## Rates
+
+xAI bills STT by audio duration. Patter meters the streaming rate on the pipeline (provider key `xai`):
+
+| Path | Rate |
+|---|---|
+| Streaming (WebSocket) | $0.20 / hr ($0.003333 / min) |
+| Batch (REST / `xaiTranscribe`) | $0.10 / hr |
+
+Override the metered rate per-project via `new Patter({ pricing: { xai: { price: ... } } })`. See [Metrics](/typescript-sdk/metrics) for the full rate table.
+
+## Options
+
+| TypeScript | Python | Default | Notes |
+|---|---|---|---|
+| `apiKey` | `api_key` | — | Reads `XAI_API_KEY` when omitted. |
+| `language` | `language` | — | BCP-47 code enabling server-side text formatting. Auto when unset. |
+| `encoding` | `encoding` | `"pcm"` | `pcm` / `mulaw` / `alaw`. |
+| `sampleRate` | `sample_rate` | `16000` | Input sample rate (Hz). |
+| `interimResults` | `interim_results` | `true` | Emit `is_final=false` partials (~every 500 ms). |
+| `endpointingMs` | `endpointing_ms` | — | Silence (ms, `[0, 5000]`) before an utterance is final. Unset keeps the xAI default (10); `0` = any VAD boundary. |
+| `smartTurn` | `smart_turn` | — | Smart Turn end-of-turn confidence threshold `[0.0, 1.0]`. Unset leaves it off. |
+| `smartTurnTimeoutMs` | `smart_turn_timeout_ms` | — | Max silence (ms, `[1, 5000]`) before forcing `speech_final` when Smart Turn is on. |
+| `keyterms` | `keyterms` | `[]` | Up to 100 bias terms (≤50 chars each). |
+| `diarize` | `diarize` | `false` | Speaker diarization — words gain a `speaker` index. |
+| `fillerWords` | `filler_words` | `false` | Include "uh" / "um" in the transcript. |
+| `baseUrl` | `base_url` | xAI STT WS endpoint | Override for proxying or tests. |
+
+## Batch transcription
+
+For a one-shot file or URL (rather than a live stream), use the batch helper. Pass either raw `audio` bytes or a `url` for xAI to download server-side. Container formats (WAV/MP3/OGG/Opus/FLAC/AAC/MP4/M4A/MKV) are auto-detected; for raw headerless audio pass `audioFormat` (`pcm` / `mulaw` / `alaw`) and `sampleRate`. Set `format: true` (with `language`) for Inverse Text Normalization ("one hundred dollars" → "$100").
+
+
+```typescript TypeScript
+import { readFile } from "node:fs/promises";
+import { xaiTranscribe } from "getpatter";
+
+// From bytes
+const audio = await readFile("call.wav");
+const result = await xaiTranscribe({ audio, language: "en", format: true });
+
+// Or from a URL, with word timestamps + diarization
+const fromUrl = await xaiTranscribe({
+ url: "https://example.com/recording.mp3",
+ diarize: true,
+});
+
+console.log(result.text, result.duration);
+for (const word of result.words) {
+ console.log(word.text, word.start, word.end, word.speaker);
+}
+```
+
+```python Python
+from getpatter import xai_transcribe
+
+# From bytes
+with open("call.wav", "rb") as f:
+ audio = f.read()
+result = await xai_transcribe(audio, language="en", format=True)
+
+# Or from a URL, with word timestamps + diarization
+from_url = await xai_transcribe(
+ url="https://example.com/recording.mp3",
+ diarize=True,
+)
+
+print(result.text, result.duration)
+for word in result.words:
+ print(word.text, word.start, word.end, word.speaker)
+```
+
+
+The helper returns a parsed result — `XaiTranscription { text, language, duration, words }` — where each `XaiWord` carries `text`, `start`, `end`, and (when `diarize` is on) `speaker`.
+
+## Low-level usage
+
+The pipeline-mode wrapper adds `XAI_API_KEY` resolution on top of the underlying provider. To use the provider directly, import it and pass the API key as the first argument:
+
+```typescript
+import { XaiSTT as LowLevelSTT } from "getpatter/providers/xai-stt";
+
+const stt = new LowLevelSTT("", { language: "en", smartTurn: 0.7 });
+await stt.connect();
+stt.onTranscript((t) => console.log(t.text, t.isFinal, t.speechFinal));
+```
+
+## What's Next
+
+
+ All STT providers side by side.
+ Grok text-to-speech.
+ Grok Voice Agent engine.
+ Cost tracking and rates.
+
diff --git a/docs/typescript-sdk/providers/xai-tts.mdx b/docs/typescript-sdk/providers/xai-tts.mdx
new file mode 100644
index 00000000..33c83e61
--- /dev/null
+++ b/docs/typescript-sdk/providers/xai-tts.mdx
@@ -0,0 +1,293 @@
+---
+title: "xAI TTS"
+description: "xAI (Grok) text-to-speech — expressive voices, inline speech tags, and telephony-native output over the one-shot /tts bytes endpoint."
+icon: "waveform"
+---
+
+# xAI TTS
+
+`XaiTTS` targets xAI's one-shot [Text-to-Speech endpoint](https://docs.x.ai/) (`POST https://api.x.ai/v1/tts`). The response body is raw audio bytes in the requested codec, which maps cleanly onto Patter's streaming synthesis contract and keeps the provider dependency-free (just `fetch` / `aiohttp`).
+
+The default voice is **`eve`**, and the default output is **PCM-16-LE @ 16 kHz** so chunks drop straight into the Patter pipeline without transcoding. For phone calls the carrier factories emit telephony-native audio — see [Telephony](#telephony).
+
+
+**Beta.** The adapter is validated against the xAI TTS API spec; it has not yet
+been exercised against a live phone call.
+
+
+## Install
+
+
+```bash TypeScript
+npm install getpatter
+```
+
+```bash Python
+pip install "getpatter[xai]"
+```
+
+
+## Authentication
+
+```bash
+export XAI_API_KEY=""
+```
+
+The REST endpoint authenticates with an `Authorization: Bearer ` header. `XAI_API_KEY` is read automatically when `apiKey` / `api_key` is omitted.
+
+## Usage
+
+
+ Use the namespaced import (`getpatter/tts/xai`) or the flat re-export
+ (`XaiTTS`). Both auto-resolve `XAI_API_KEY` from the environment when
+ `apiKey` is omitted.
+
+
+
+```typescript TypeScript
+// Namespaced import (pipeline mode)
+import * as xai from "getpatter/tts/xai";
+
+const tts = new xai.TTS(); // reads XAI_API_KEY, voice "eve"
+const tts2 = new xai.TTS({ apiKey: "...", voice: "leo", language: "en" });
+
+// Flat alias (equivalent)
+import { XaiTTS } from "getpatter";
+
+const tts3 = new XaiTTS();
+```
+
+```python Python
+# Namespaced import (pipeline mode)
+from getpatter.tts import xai
+
+tts = xai.TTS() # reads XAI_API_KEY, voice "eve"
+tts = xai.TTS(api_key="...", voice="leo", language="en")
+
+# Flat alias (equivalent)
+from getpatter import XaiTTS
+
+tts = XaiTTS()
+```
+
+
+Plug it into an agent:
+
+
+```typescript TypeScript
+// npx tsx example.ts
+import { Patter, Twilio, DeepgramSTT, XaiTTS } from "getpatter";
+
+const phone = new Patter({ carrier: new Twilio(), phoneNumber: "+15550001234" });
+
+const agent = phone.agent({
+ stt: new DeepgramSTT(), // DEEPGRAM_API_KEY from env
+ tts: XaiTTS.forTwilio({ voice: "eve" }), // XAI_API_KEY from env
+ systemPrompt: "You are a helpful assistant.",
+});
+
+await phone.serve({ agent });
+```
+
+```python Python
+import asyncio
+from getpatter import Patter, Twilio, DeepgramSTT, XaiTTS
+
+phone = Patter(carrier=Twilio(), phone_number="+15550001234")
+
+agent = phone.agent(
+ stt=DeepgramSTT(), # DEEPGRAM_API_KEY from env
+ tts=XaiTTS.for_twilio(voice="eve"), # XAI_API_KEY from env
+ system_prompt="You are a helpful assistant.",
+)
+
+asyncio.run(phone.serve(agent))
+```
+
+
+## Voices
+
+xAI ships a roster of 26 built-in voices, each with a distinct personality. `eve` is the default; voice IDs are **case-insensitive** (`eve`, `Eve`, and `EVE` all work). Tone and suggested use cases below are from the [xAI Voice Overview](https://docs.x.ai/developers/model-capabilities/audio/voice); preview samples for each voice in the [xAI console playground](https://console.x.ai/).
+
+| Voice | Tone | Suggested use cases |
+|---|---|---|
+| `eve` *(default)* | Energetic and upbeat | — |
+| `altair` | Elegant, refined, and effortlessly premium | Advertising, Narration |
+| `ara` | Warm and friendly | — |
+| `atlas` | Confident, commanding, and reassuring | Sales, Assistant |
+| `carina` | Soft, empathetic, and soothing | Wellness, Support |
+| `castor` | Charismatic, down-to-earth, and easygoing | Sales, Support |
+| `celeste` | Compassionate, confident, and reassuring | Support, Assistant |
+| `cosmo` | Bright, curious, and easy to follow | Education, Podcast |
+| `helios` | Upbeat, energetic, and endlessly versatile | Assistant, Wellness |
+| `helix` | Bold, dynamic, and adrenaline-fueled | Commentary, Podcast |
+| `iris` | Friendly, upbeat, and naturally charming | Sales, Support |
+| `kepler` | Inventive, forward-thinking, and charismatic | Advertising, Podcast |
+| `leo` | Authoritative and strong | — |
+| `lumen` | Warm, articulate, and engaging | Education, Advertising |
+| `luna` | Gentle, patient, and deeply nurturing | Education, Assistant |
+| `lux` | Grounded, calm, and quietly wise | Wellness, Narration |
+| `naksh` | Warm, thoughtful, and wise | Assistant, Support |
+| `orion` | Rich, cinematic, and resonant | Narration, Audiobooks |
+| `perseus` | Strong, confident, and trustworthy | Advertising, Narration |
+| `rex` | Confident and clear | — |
+| `rigel` | Precise, professional, and calmly confident | Assistant, Support |
+| `sal` | Smooth and balanced | — |
+| `sirius` | Quick-witted, clever, and playful | Commentary, Characters |
+| `ursa` | Friendly, warm, and steadfast | Assistant, Podcast |
+| `zagan` | Powerful, dramatic, and unmistakable | Characters, Narration |
+| `zenith` | Sharp, focused, and driven | Sales, Advertising |
+
+### Custom voices
+
+Clone a voice from a short reference clip (WAV, ≤120 s) with the custom-voices helper. It returns a `voice_id` usable as the `voice` on both `XaiTTS` and the [xAI Realtime engine](/typescript-sdk/providers/xai-realtime):
+
+
+```typescript TypeScript
+import { readFile } from "node:fs/promises";
+import { xaiCreateCustomVoice, XaiTTS } from "getpatter";
+
+const clip = await readFile("reference.wav"); // ≤120 s
+const voiceId = await xaiCreateCustomVoice({
+ name: "Front Desk",
+ language: "en",
+ audio: clip,
+});
+
+const tts = new XaiTTS({ voice: voiceId });
+```
+
+```python Python
+from getpatter import xai_create_custom_voice, XaiTTS
+
+with open("reference.wav", "rb") as f: # ≤120 s
+ clip = f.read()
+voice_id = await xai_create_custom_voice(
+ name="Front Desk",
+ language="en",
+ audio=clip,
+)
+
+tts = XaiTTS(voice=voice_id)
+```
+
+
+## Language
+
+`language` is required by xAI and defaults to `"auto"` (the model detects the language of the text). Pass a BCP-47 code — e.g. `"en"`, `"it"`, `"pt-BR"` — for consistent results. Language validation is case-insensitive.
+
+## Speech tags
+
+xAI supports inline speech tags embedded in the synthesized text for expressive delivery — for example `[pause]` and `[laugh]` for vocal expressions, plus wrapping tags to change delivery style (whispering, singing). Because Patter forwards the reply text verbatim to xAI, any tags your LLM (or system prompt) produces in the text are honored:
+
+
+```typescript TypeScript
+// The tags travel inside the text your agent speaks.
+const tts = new XaiTTS({ voice: "eve", language: "en" });
+for await (const chunk of tts.synthesizeStream(
+ "So I walked in and [pause] there it was. [laugh] Incredible!",
+)) {
+ // raw PCM-16-LE @ 16 kHz
+}
+```
+
+```python Python
+# The tags travel inside the text your agent speaks.
+tts = XaiTTS(voice="eve", language="en")
+async for chunk in tts.synthesize(
+ "So I walked in and [pause] there it was. [laugh] Incredible!",
+):
+ ... # raw PCM-16-LE @ 16 kHz
+```
+
+
+
+A single request accepts up to **15,000 characters**. The adapter warns and
+forwards the text as-is past that limit (the API rejects it); split long text
+upstream — the pipeline already synthesizes per utterance.
+
+
+## Telephony
+
+The constructor default `codec="pcm"` @ `sampleRate=16000` is correct for web playback and 16 kHz pipelines. For real phone calls use the carrier factories, which negotiate the carrier-native codec so the pipeline skips resampling:
+
+
+```typescript TypeScript
+import { XaiTTS } from "getpatter";
+
+// Twilio Media Streams: μ-law @ 8 kHz native — bit-clean passthrough, no resample.
+const tts = XaiTTS.forTwilio({ voice: "eve" });
+
+// Telnyx: PCM @ 16 kHz — flows through the standard PCM16 path.
+const tts2 = XaiTTS.forTelnyx({ voice: "eve" });
+```
+
+```python Python
+from getpatter import XaiTTS
+
+# Twilio Media Streams: μ-law @ 8 kHz native — bit-clean passthrough, no resample.
+tts = XaiTTS.for_twilio(voice="eve")
+
+# Telnyx: PCM @ 16 kHz — flows through the standard PCM16 path.
+tts2 = XaiTTS.for_telnyx(voice="eve")
+```
+
+
+`forTwilio()` emits G.711 μ-law @ 8 kHz — exactly Twilio's wire codec, which xAI supports natively — so the audio passes straight through with zero resampling and zero PCM → μ-law encoding.
+
+## Output formats
+
+Control the codec and sample rate directly when not using the carrier factories:
+
+| `codec` | Best for |
+|---|---|
+| `mp3` | General web use — wide compatibility. |
+| `wav` | Lossless — editing / post-production. |
+| `pcm` (default) | Raw audio — real-time pipelines. |
+| `mulaw` | Telephony (G.711 μ-law). |
+| `alaw` | Telephony (G.711 A-law). |
+
+Supported `sampleRate` values: `8000`, `16000`, `22050`, `24000` (xAI's own default), `44100`, `48000`. `bitRate` applies to `mp3` only (`32000`–`192000`).
+
+## Rates
+
+xAI bills TTS per character. Patter's default rate (provider key `xai_tts`) is **$0.015 / 1k characters** ($15.00 / 1M chars). Override per-project via `new Patter({ pricing: { xai_tts: { price: ... } } })`. See [Metrics](/typescript-sdk/metrics) for the full rate table.
+
+## Options
+
+| TypeScript | Python | Default | Notes |
+|---|---|---|---|
+| `apiKey` | `api_key` | — | Reads `XAI_API_KEY` when omitted. |
+| `voice` | `voice` | `"eve"` | Built-in or custom voice ID (case-insensitive). |
+| `language` | `language` | `"auto"` | BCP-47 tag or `"auto"`. Required by xAI. |
+| `codec` | `codec` | `"pcm"` | `pcm` / `mulaw` / `alaw` / `wav` / `mp3`. |
+| `sampleRate` | `sample_rate` | `16000` | `8000` / `16000` / `22050` / `24000` / `44100` / `48000`. |
+| `bitRate` | — | — | Bit rate (bits/sec), MP3 only. |
+| `speed` | `speed` | — | Speaking-rate multiplier in `[0.7, 1.5]`. |
+| `optimizeStreamingLatency` | `optimize_streaming_latency` | — | `0` / `1` / `2` — higher trades quality for lower time-to-first-audio. |
+| `textNormalization` | `text_normalization` | `false` | Normalize numbers / abbreviations / symbols to spoken form. |
+| `baseUrl` | `base_url` | xAI `/tts` endpoint | Override for proxying or tests. |
+
+`bitRate` is a TypeScript-only constructor option; on the Python side pass it via the low-level provider if you need a non-default MP3 bit rate.
+
+## Low-level usage
+
+The pipeline-mode wrapper adds `XAI_API_KEY` resolution on top of the underlying provider. To use it directly, import from `getpatter/providers/xai-tts` and pass the API key first:
+
+```typescript
+import { XaiTTS as LowLevelTTS } from "getpatter/providers/xai-tts";
+
+const tts = new LowLevelTTS("", { voice: "eve", codec: "pcm", sampleRate: 16000 });
+for await (const chunk of tts.synthesizeStream("Hello from the Patter pipeline.")) {
+ // raw PCM-16-LE @ 16 kHz
+}
+```
+
+## What's Next
+
+
+ All TTS providers side by side.
+ Grok speech-to-text.
+ Grok Voice Agent engine.
+ Cost tracking and rates.
+
diff --git a/docs/typescript-sdk/stt.mdx b/docs/typescript-sdk/stt.mdx
index 41485c22..7d3a4ae3 100644
--- a/docs/typescript-sdk/stt.mdx
+++ b/docs/typescript-sdk/stt.mdx
@@ -39,6 +39,7 @@ await phone.serve({ agent });
| `AssemblyAISTT` | `getpatter/stt/assemblyai` | `ASSEMBLYAI_API_KEY` |
| `SonioxSTT` | `getpatter/stt/soniox` | `SONIOX_API_KEY` |
| `SpeechmaticsSTT` | `getpatter/stt/speechmatics` | `SPEECHMATICS_API_KEY` |
+| `XaiSTT` | `getpatter/stt/xai` | `XAI_API_KEY` |
`SpeechmaticsSTT` is being ported to TypeScript in the upcoming release — see the `## Unreleased` section in `CHANGELOG.md`. Use Python or wait for the next minor version.
@@ -149,6 +150,17 @@ import { SonioxSTT } from "getpatter";
const stt = new SonioxSTT(); // reads SONIOX_API_KEY
```
+## xAI
+
+Real-time streaming STT via xAI (Grok), plus a one-shot batch transcription helper. Supports Smart Turn end-of-turn detection, keyterm biasing, and speaker diarization. See [xAI STT setup](/typescript-sdk/providers/xai-stt).
+
+```typescript
+import { XaiSTT } from "getpatter";
+
+const stt = new XaiSTT(); // reads XAI_API_KEY
+const stt2 = new XaiSTT({ smartTurn: 0.7, language: "en" });
+```
+
## Missing credentials
Each class throws at construction time if no API key is resolved:
diff --git a/docs/typescript-sdk/tts.mdx b/docs/typescript-sdk/tts.mdx
index b9d84d43..afdeba5e 100644
--- a/docs/typescript-sdk/tts.mdx
+++ b/docs/typescript-sdk/tts.mdx
@@ -37,6 +37,7 @@ await phone.serve({ agent });
| `CartesiaTTS` | `CARTESIA_API_KEY` |
| `RimeTTS` | `RIME_API_KEY` |
| `LMNTTTS` | `LMNT_API_KEY` |
+| `XaiTTS` | `XAI_API_KEY` |
### Model / voice / format enums
@@ -152,6 +153,18 @@ const tts = new LMNTTTS(); // reads LMNT_
const tts = new LMNTTTS({ model: "blizzard", voice: "leah" });
```
+## xAI
+
+Grok text-to-speech (default voice `eve`) with 26 built-in voices, inline speech tags, custom voice cloning, and telephony-native output. See [xAI TTS setup](/typescript-sdk/providers/xai-tts).
+
+```typescript
+import { XaiTTS } from "getpatter";
+
+const tts = new XaiTTS(); // reads XAI_API_KEY
+const tts2 = new XaiTTS({ voice: "leo", language: "en" });
+const tts3 = XaiTTS.forTwilio({ voice: "eve" }); // μ-law @ 8 kHz native
+```
+
## Missing credentials
Each class throws at construction time if no API key is resolved:
diff --git a/libraries/python/getpatter/__init__.py b/libraries/python/getpatter/__init__.py
index 4ac47e3f..9af2436a 100644
--- a/libraries/python/getpatter/__init__.py
+++ b/libraries/python/getpatter/__init__.py
@@ -103,7 +103,9 @@
from getpatter.engines.openai import Realtime as OpenAIRealtime
from getpatter.engines.openai_realtime_2 import Realtime2 as OpenAIRealtime2
from getpatter.engines.elevenlabs import ConvAI as ElevenLabsConvAI
+from getpatter.engines.xai import XaiRealtime
from getpatter.providers.openai_realtime_2 import OpenAIRealtime2Adapter
+from getpatter.providers.xai_realtime import XaiRealtimeAdapter
# STT flat aliases — parity with libraries/typescript/src/index.ts.
from getpatter.stt.deepgram import STT as DeepgramSTT
@@ -113,6 +115,12 @@
from getpatter.stt.soniox import STT as SonioxSTT
from getpatter.stt.speechmatics import STT as SpeechmaticsSTT
from getpatter.stt.assemblyai import STT as AssemblyAISTT
+from getpatter.stt.xai import STT as XaiSTT
+from getpatter.providers.xai_stt import (
+ XaiTranscription,
+ XaiWord,
+ transcribe as xai_transcribe,
+)
# TTS flat aliases. As of 0.6.1, ``ElevenLabsTTS`` (the canonical facade)
# defaults to the WebSocket streaming transport. ``ElevenLabsWebSocketTTS``
@@ -129,6 +137,8 @@
from getpatter.tts.inworld import TTS as InworldTTS
from getpatter.tts.soniox import TTS as SonioxTTS
from getpatter.tts.sarvam import TTS as SarvamTTS
+from getpatter.tts.xai import TTS as XaiTTS
+from getpatter.providers.xai_tts import create_custom_voice as xai_create_custom_voice
# LLM flat aliases — parity with libraries/typescript/src/index.ts and mirror of STT/TTS layout.
from getpatter.llm.openai import LLM as OpenAILLM
@@ -512,6 +522,8 @@ def mix_pcm(agent: bytes, bg: bytes, ratio: float) -> bytes:
"OpenAIRealtime",
"OpenAIRealtime2",
"OpenAIRealtime2Adapter",
+ "XaiRealtime",
+ "XaiRealtimeAdapter",
"ElevenLabsConvAI",
"DeepgramSTT",
"WhisperSTT",
@@ -520,6 +532,10 @@ def mix_pcm(agent: bytes, bg: bytes, ratio: float) -> bytes:
"SonioxSTT",
"SpeechmaticsSTT",
"AssemblyAISTT",
+ "XaiSTT",
+ "XaiTranscription",
+ "XaiWord",
+ "xai_transcribe",
"ElevenLabsTTS",
"ElevenLabsWebSocketTTS",
"ElevenLabsRestTTS",
@@ -530,6 +546,8 @@ def mix_pcm(agent: bytes, bg: bytes, ratio: float) -> bytes:
"InworldTTS",
"SonioxTTS",
"SarvamTTS",
+ "XaiTTS",
+ "xai_create_custom_voice",
"OpenAILLM",
"AnthropicLLM",
"GroqLLM",
diff --git a/libraries/python/getpatter/client.py b/libraries/python/getpatter/client.py
index e88b7f1a..3c14a9f2 100644
--- a/libraries/python/getpatter/client.py
+++ b/libraries/python/getpatter/client.py
@@ -2015,6 +2015,10 @@ def agent(
# --- Engine dispatch ---
openai_engine_key: str = ""
elevenlabs_engine_key: str = ""
+ xai_engine_key: str = ""
+ # Engine-supplied xAI Grok Voice Agent session tuning (dict of only the
+ # set knobs), forwarded to ``XaiRealtimeAdapter`` by the stream-handler.
+ xai_realtime_config: dict | None = None
# Engine-supplied OpenAI Realtime extras propagated to Agent so the
# stream-handler can forward them to ``OpenAIRealtimeAdapter``.
openai_realtime_reasoning_effort: str | None = None
@@ -2057,6 +2061,15 @@ def agent(
realtime_gate_response_on_transcript = engine_fields.get(
"gate_response_on_transcript"
)
+ elif engine_kind == "xai_realtime":
+ xai_engine_key = engine_fields.get("api_key", "")
+ # Carry only the SET xAI session knobs (drop None) — voice/model
+ # are already applied to the top-level slots above.
+ xai_realtime_config = {
+ k: v
+ for k, v in engine_fields.items()
+ if k not in ("api_key", "voice", "model") and v is not None
+ }
elif engine_kind == "elevenlabs_convai":
elevenlabs_engine_key = engine_fields.get("api_key", "")
elif provider is not None:
@@ -2085,6 +2098,8 @@ def agent(
self._local_config = replace(
self._local_config, elevenlabs_key=elevenlabs_engine_key
)
+ if xai_engine_key and not self._local_config.xai_key:
+ self._local_config = replace(self._local_config, xai_key=xai_engine_key)
if (
provider in ("openai_realtime", "openai_realtime_2")
@@ -2322,6 +2337,7 @@ def agent(
openai_realtime_noise_reduction=openai_realtime_noise_reduction,
realtime_turn_detection=realtime_turn_detection,
realtime_gate_response_on_transcript=realtime_gate_response_on_transcript,
+ xai_realtime=xai_realtime_config,
tool_call_preambles=tool_call_preambles,
preemptive_generation=preemptive_generation,
preemptive_min_stable_ms=preemptive_min_stable_ms,
@@ -2336,7 +2352,25 @@ def _unpack_engine(engine: Any) -> tuple[str, dict]:
from getpatter.engines.elevenlabs import ConvAI as _ConvAI
from getpatter.engines.openai import Realtime as _Realtime
from getpatter.engines.openai_realtime_2 import Realtime2 as _Realtime2
+ from getpatter.engines.xai import XaiRealtime as _XaiRealtime
+ if isinstance(engine, _XaiRealtime):
+ return "xai_realtime", {
+ "api_key": engine.api_key,
+ "voice": engine.voice,
+ "model": engine.model,
+ "reasoning_effort": engine.reasoning_effort,
+ "vad_threshold": engine.vad_threshold,
+ "silence_duration_ms": engine.silence_duration_ms,
+ "prefix_padding_ms": engine.prefix_padding_ms,
+ "idle_timeout_ms": engine.idle_timeout_ms,
+ "language_hint": engine.language_hint,
+ "keyterms": engine.keyterms,
+ "speed": engine.speed,
+ "replace": engine.replace,
+ "resumption": engine.resumption,
+ "server_tools": engine.server_tools,
+ }
if isinstance(engine, _Realtime2):
return "openai_realtime_2", {
"api_key": engine.api_key,
@@ -2368,8 +2402,9 @@ def _unpack_engine(engine: Any) -> tuple[str, dict]:
"voice": engine.voice,
}
raise TypeError(
- "engine= must be an OpenAIRealtime(...), OpenAIRealtime2(...), or "
- f"ElevenLabsConvAI(...) instance, got {type(engine).__name__}"
+ "engine= must be an OpenAIRealtime(...), OpenAIRealtime2(...), "
+ "XaiRealtime(...), or ElevenLabsConvAI(...) instance, got "
+ f"{type(engine).__name__}"
)
@staticmethod
diff --git a/libraries/python/getpatter/engines/__init__.py b/libraries/python/getpatter/engines/__init__.py
index 21e588d3..4794847e 100644
--- a/libraries/python/getpatter/engines/__init__.py
+++ b/libraries/python/getpatter/engines/__init__.py
@@ -12,4 +12,4 @@
from __future__ import annotations
-__all__ = ["openai", "openai_realtime_2", "elevenlabs"]
+__all__ = ["openai", "openai_realtime_2", "elevenlabs", "xai"]
diff --git a/libraries/python/getpatter/engines/xai.py b/libraries/python/getpatter/engines/xai.py
new file mode 100644
index 00000000..08490de7
--- /dev/null
+++ b/libraries/python/getpatter/engines/xai.py
@@ -0,0 +1,89 @@
+"""xAI Grok Voice Agent engine marker for Patter (Beta).
+
+Wraps the xAI Voice Agent API (OpenAI-Realtime-GA-compatible). The client
+dispatches to
+:class:`getpatter.providers.xai_realtime.XaiRealtimeAdapter` when this marker is
+passed to ``Patter.agent(engine=...)``.
+
+Beta: validated against the xAI Voice Agent API spec; not yet exercised against
+a live phone call.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+__all__ = ["XaiRealtime"]
+
+
+@dataclass(frozen=True)
+class XaiRealtime:
+ """xAI Grok Voice Agent engine config — selects ``grok-voice-latest``.
+
+ Holds the settings the Patter server needs to instantiate
+ :class:`getpatter.providers.xai_realtime.XaiRealtimeAdapter` at call time.
+ All fields are optional with safe defaults; the xAI-specific tuning knobs
+ are omitted from ``session.update`` unless set.
+
+ Example::
+
+ from getpatter import Patter, Twilio, XaiRealtime
+
+ phone = Patter(carrier=Twilio(), phone_number="+1...")
+ agent = phone.agent(
+ engine=XaiRealtime(voice="eve", reasoning_effort="none"),
+ system_prompt="You are a friendly receptionist.",
+ first_message="Hello! How can I help?",
+ )
+ """
+
+ api_key: str = ""
+ voice: str = "eve"
+ model: str = "grok-voice-latest"
+ # Reasoning-effort tier. ``None`` leaves the field unset (server default
+ # ``"high"``). xAI accepts ``"high"`` or ``"none"`` (disable reasoning for
+ # lower latency on simple flows).
+ reasoning_effort: Literal["high", "none"] | None = None
+ # VAD activation threshold (0.1–0.9). ``None`` keeps the xAI default (0.85).
+ vad_threshold: float | None = None
+ # Silence (ms, 0–10000) required before the server ends the user's turn.
+ # ``None`` keeps the adapter default.
+ silence_duration_ms: int | None = None
+ # Audio (ms, 0–10000) captured before detected speech start. ``None`` keeps
+ # the xAI default (333).
+ prefix_padding_ms: int | None = None
+ # When set, the server re-engages the user after this many ms of silence
+ # following an assistant turn. ``None`` disables the idle timer.
+ idle_timeout_ms: int | None = None
+ # BCP-47 language hint biasing ASR transcription (e.g. ``"es-MX"``). ``None``
+ # keeps automatic language detection.
+ language_hint: str | None = None
+ # Bias terms (≤100, ≤50 chars each) for ASR transcription.
+ keyterms: tuple[str, ...] = ()
+ # Assistant audio playback speed multiplier (0.7–1.5). ``None`` keeps 1.0.
+ speed: float | None = None
+ # Pronunciation replacements applied before TTS (e.g.
+ # ``{"Acme Mobile": "Acme Mobull"}``). ``None`` disables them.
+ replace: dict[str, str] | None = None
+ # Opt in to Session Resumption (cache + replay conversation turns on
+ # reconnect). ``False`` (default) keeps the stateless behaviour.
+ resumption: bool = False
+ # xAI server-side tools appended verbatim to ``session.tools`` (e.g.
+ # ``({"type": "web_search"},)``); xAI executes these on its side.
+ server_tools: tuple[dict[str, Any], ...] = field(default_factory=tuple)
+
+ def __post_init__(self) -> None:
+ key = self.api_key or os.environ.get("XAI_API_KEY", "")
+ if not key:
+ raise ValueError(
+ "xAI Realtime engine requires an api_key. Pass "
+ "api_key='xai-...' or set XAI_API_KEY in the environment."
+ )
+ object.__setattr__(self, "api_key", key)
+
+ @property
+ def kind(self) -> str:
+ """Stable discriminator used for engine dispatch."""
+ return "xai_realtime"
diff --git a/libraries/python/getpatter/local_config.py b/libraries/python/getpatter/local_config.py
index a10a7ea7..7b97b2e7 100644
--- a/libraries/python/getpatter/local_config.py
+++ b/libraries/python/getpatter/local_config.py
@@ -20,6 +20,7 @@ class LocalConfig:
soniox_key: str = ""
speechmatics_key: str = ""
assemblyai_key: str = ""
+ xai_key: str = ""
phone_number: str = ""
webhook_url: str = ""
# SECURITY: require valid webhook signatures on Twilio, Telnyx and Plivo
diff --git a/libraries/python/getpatter/models.py b/libraries/python/getpatter/models.py
index 3b6d2ee6..a39c46d8 100644
--- a/libraries/python/getpatter/models.py
+++ b/libraries/python/getpatter/models.py
@@ -33,7 +33,9 @@
# string union ``'openai_realtime' | 'elevenlabs_convai' | 'pipeline'`` in
# ``types.ts``. Tightened from a free ``str`` so editors autocomplete and
# typos surface at type-check time instead of at call time.
-ProviderMode = Literal["openai_realtime", "elevenlabs_convai", "pipeline"]
+ProviderMode = Literal[
+ "openai_realtime", "xai_realtime", "elevenlabs_convai", "pipeline"
+]
@dataclass(frozen=True)
@@ -765,6 +767,14 @@ class Agent:
# default is the decoupled behavior — it reclaims the ~500 ms Whisper wait).
# Realtime modes only; pipeline mode uses its own dedicated STT.
realtime_gate_response_on_transcript: bool | None = None
+ # xAI Grok Voice Agent — engine-supplied session tuning threaded through
+ # from ``engines.xai.XaiRealtime(...)``. ``None`` (default) when the agent
+ # is not an xAI realtime engine; otherwise a dict of the xAI-specific knobs
+ # (reasoning_effort, vad_threshold, silence_duration_ms, prefix_padding_ms,
+ # idle_timeout_ms, language_hint, keyterms, speed, replace, resumption,
+ # server_tools) that the stream-handler forwards to
+ # ``providers.xai_realtime.XaiRealtimeAdapter``. Only set keys are present.
+ xai_realtime: dict | None = None
# Opt-in barge-in confirmation strategies (pipeline mode). With the
# default empty tuple the SDK falls back to the legacy "interrupt
# immediately on VAD speech_start" behaviour. When at least one
diff --git a/libraries/python/getpatter/pricing.py b/libraries/python/getpatter/pricing.py
index f1f40dce..40b05190 100644
--- a/libraries/python/getpatter/pricing.py
+++ b/libraries/python/getpatter/pricing.py
@@ -141,6 +141,10 @@ class PricingUnit(StrEnum):
"cartesia_stt": {"unit": PricingUnit.MINUTE, "price": 0.0025},
# Soniox real-time STT: $0.12/hr = $0.002/min
"soniox": {"unit": PricingUnit.MINUTE, "price": 0.002},
+ # xAI streaming STT: $0.20/hr = $0.20/60 per min. The batch REST path is
+ # cheaper ($0.10/hr = $0.10/60 per min), but the pipeline meters the
+ # streaming rate; batch callers reconcile via ``XAI_STT_BATCH_PRICE_PER_MINUTE``.
+ "xai": {"unit": PricingUnit.MINUTE, "price": 0.20 / 60},
# Speechmatics Pro tier: $0.24/hr = $0.0040/min (new users land here).
# Previous $0.0173 reflected a retired Standard tier; users were
# being over-billed ~4.3x.
@@ -238,6 +242,8 @@ class PricingUnit(StrEnum):
# chars/hr, $0.70/hr ÷ 54 ≈ $0.013 / 1k chars. Source:
# https://soniox.com/pricing (also https://soniox.com/text-to-speech).
"soniox_tts": {"unit": PricingUnit.THOUSAND_CHARS, "price": 0.013},
+ # xAI TTS: $15.00 / 1M chars = $0.015 / 1k chars.
+ "xai_tts": {"unit": PricingUnit.THOUSAND_CHARS, "price": 0.015},
# Sarvam AI Bulbul TTS — per 1,000 characters. INR is authoritative; USD is
# an indicative FX conversion (~Rs 83-84/USD). Bulbul v3 (default): Rs 30 /
# 10k chars ≈ $0.036/1k; Bulbul v2: Rs 15 / 10k ≈ $0.018/1k. Billed per
@@ -326,6 +332,17 @@ class PricingUnit(StrEnum):
},
},
},
+ # xAI Grok Voice Agent (realtime) — billed per MINUTE of session audio
+ # ($0.05/min = $3.00/hr), NOT per token, so it uses the MINUTE unit and the
+ # ``duration_seconds`` path in :func:`calculate_realtime_cost`.
+ # ``text_input_per_message`` ($0.004 per ``conversation.item.create``) is a
+ # documented constant, NOT metered in v1 — the metrics layer has no
+ # per-message counter for realtime engines.
+ "xai_realtime": {
+ "unit": PricingUnit.MINUTE,
+ "price": 0.05,
+ "text_input_per_message": 0.004,
+ },
# Telephony — per minute of call duration.
# twilio default = US inbound local (the 99% case for voice agents
# receiving calls on a local number). For US toll-free inbound ($0.022/min)
@@ -451,21 +468,46 @@ def calculate_realtime_cost(
usage: dict,
pricing: dict,
model: str | None = None,
+ *,
+ provider: str = "openai_realtime",
+ duration_seconds: float | None = None,
) -> float:
- """Calculate OpenAI Realtime cost from token usage in ``response.done``.
+ """Calculate realtime-engine cost.
+
+ Two billing shapes are supported:
+
+ * **Token-based** (``openai_realtime``, the default ``provider``) — cost is
+ derived from the ``response.usage`` token counts in a ``response.done``
+ event. This is the original behaviour; existing callers are unaffected.
+ * **Per-minute** (e.g. ``xai_realtime``) — when the resolved provider entry's
+ unit is ``minute``, cost is ``price * duration_seconds / 60`` (``0.0`` when
+ ``duration_seconds`` is ``None``); token usage is ignored.
Args:
- usage: The ``response.usage`` dict from an OpenAI ``response.done``
- event. Expected keys: ``input_token_details``,
- ``output_token_details``.
+ usage: The ``response.usage`` dict from a ``response.done`` event
+ (token-based providers only). Expected keys:
+ ``input_token_details``, ``output_token_details``.
pricing: Merged pricing dict.
+ model: Optional model id for per-model rate resolution.
+ provider: Pricing-table key for the realtime engine (default
+ ``"openai_realtime"``).
+ duration_seconds: Session audio duration — used by MINUTE-unit
+ providers; ignored by token-unit providers.
Returns:
- Total cost in USD for this response.
+ Total cost in USD.
"""
- config = pricing.get("openai_realtime", {})
+ config = pricing.get(provider, {})
rates = _resolve_provider_rates(config, model)
- if rates.get("unit") != "token":
+ unit = rates.get("unit")
+ if unit == "minute":
+ # Per-minute realtime billing (e.g. xAI Grok Voice Agent). The
+ # ``text_input_per_message`` constant is documented on the entry but not
+ # metered here (no per-message counter in the metrics layer).
+ if duration_seconds is None:
+ return 0.0
+ return rates.get("price", 0.0) * duration_seconds / 60.0
+ if unit != "token":
return 0.0
# Guard against OpenAI sending ``"input_token_details": null`` — dict.get
diff --git a/libraries/python/getpatter/providers/xai_realtime.py b/libraries/python/getpatter/providers/xai_realtime.py
new file mode 100644
index 00000000..201030a1
--- /dev/null
+++ b/libraries/python/getpatter/providers/xai_realtime.py
@@ -0,0 +1,142 @@
+"""xAI Grok Voice Agent adapter for the Patter SDK (Beta).
+
+Beta: validated against the xAI Voice Agent API spec (docs.x.ai); not yet
+exercised against a live phone call.
+
+The xAI Voice Agent API is OpenAI-Realtime-GA-compatible: same
+``wss://.../v1/realtime?model=`` handshake, same ``session.update`` /
+event-name shapes, same function-call flow. So this adapter subclasses the GA
+:class:`~getpatter.providers.openai_realtime_2.OpenAIRealtime2Adapter` and
+changes only:
+
+* the endpoint (``wss://api.x.ai/v1/realtime``, via the ``OPENAI_REALTIME_URL``
+ class attribute the base ``connect()`` reads),
+* the ``provider_key`` and the default model / voice (``grok-voice-latest`` /
+ ``eve``),
+* the ``session.update`` body — it grafts the xAI-only knobs (``reasoning``,
+ ``audio.input.transcription.language_hint`` / ``keyterms``,
+ ``audio.output.speed``, ``turn_detection.{threshold,prefix_padding_ms,
+ idle_timeout_ms}``, ``replace``, ``resumption``, and server-side ``tools``)
+ onto the GA config produced by the parent.
+
+Audio transport is INHERITED UNCHANGED from the GA adapter: inbound mu-law
+8 kHz is transcoded to PCM 24 kHz and outbound PCM 24 kHz back to mu-law 8 kHz.
+xAI natively supports ``audio/pcmu`` @ 8 kHz, so a future optimization can flip
+the session to native G.711 passthrough and skip the transcode — deferred to a
+live-call validation pass.
+"""
+
+from __future__ import annotations
+
+from typing import Any, ClassVar, Mapping, Sequence
+
+from getpatter.providers.openai_realtime_2 import OpenAIRealtime2Adapter
+
+# Default Grok voice model. ``grok-voice-latest`` tracks the current flagship
+# (``grok-voice-think-fast-1.0`` at time of writing).
+XAI_REALTIME_URL = "wss://api.x.ai/v1/realtime"
+XAI_DEFAULT_MODEL = "grok-voice-latest"
+XAI_DEFAULT_VOICE = "eve"
+# xAI's input-transcription model. Setting it enables the
+# ``conversation.item.input_audio_transcription.updated`` events (display-only);
+# omitting the OpenAI-only ``whisper-1`` default keeps the session xAI-valid.
+XAI_TRANSCRIPTION_MODEL = "grok-transcribe"
+
+
+class XaiRealtimeAdapter(OpenAIRealtime2Adapter):
+ """Realtime WebSocket adapter speaking xAI's Grok Voice Agent API (Beta).
+
+ Subclasses :class:`OpenAIRealtime2Adapter` (GA-compatible) and overrides the
+ endpoint, provider key, defaults, and ``session.update`` builder. All
+ runtime behaviour — audio transcode, barge-in, tool dispatch, prewarm — is
+ inherited unchanged.
+ """
+
+ #: xAI Grok Voice Agent WebSocket endpoint (base ``connect()`` reads this).
+ OPENAI_REALTIME_URL: ClassVar[str] = XAI_REALTIME_URL
+ #: Stable pricing/dashboard key — read by stream-handler/metrics.
+ provider_key: ClassVar[str] = "xai_realtime"
+
+ def __init__(
+ self,
+ *args: Any,
+ vad_threshold: float | None = None,
+ prefix_padding_ms: int | None = None,
+ idle_timeout_ms: int | None = None,
+ language_hint: str | None = None,
+ keyterms: Sequence[str] = (),
+ speed: float | None = None,
+ replace: Mapping[str, str] | None = None,
+ resumption: bool = False,
+ server_tools: Sequence[Mapping[str, Any]] = (),
+ **kwargs: Any,
+ ) -> None:
+ # xAI defaults; only apply when the caller left the slot unset so an
+ # explicit model/voice/transcription-model always wins.
+ kwargs.setdefault("model", XAI_DEFAULT_MODEL)
+ kwargs.setdefault("voice", XAI_DEFAULT_VOICE)
+ kwargs.setdefault("input_audio_transcription_model", XAI_TRANSCRIPTION_MODEL)
+ # xAI-specific session knobs (grafted in ``_build_ga_session_config``).
+ # ``reasoning_effort`` and ``silence_duration_ms`` are reused from the
+ # base adapter and flow through ``**kwargs``.
+ self.vad_threshold = vad_threshold
+ self.prefix_padding_ms = prefix_padding_ms
+ self.idle_timeout_ms = idle_timeout_ms
+ self.language_hint = language_hint
+ self.keyterms = tuple(keyterms)
+ self.speed = speed
+ self.replace = dict(replace) if replace else None
+ self.resumption = resumption
+ self.server_tools = tuple(server_tools)
+ super().__init__(*args, **kwargs)
+
+ def _build_ga_session_config(self) -> dict[str, Any]:
+ """Build the GA session.update body, then graft the xAI-only knobs.
+
+ Calls the parent GA builder (audio format, turn detection, instructions,
+ voice, tools, reasoning) and adds the xAI extensions only when the caller
+ set them, so an unconfigured session stays byte-compatible with the GA
+ shape the xAI endpoint accepts.
+ """
+ config = super()._build_ga_session_config()
+
+ input_cfg = config["audio"]["input"]
+ transcription = input_cfg.setdefault("transcription", {})
+ # xAI biases ASR via ``language_hint`` (BCP-47), NOT OpenAI's
+ # ``language``. Drop the OpenAI-only key if the parent emitted it.
+ transcription.pop("language", None)
+ if self.language_hint is not None:
+ transcription["language_hint"] = self.language_hint
+ if self.keyterms:
+ transcription["keyterms"] = list(self.keyterms)
+
+ # Assistant playback speed multiplier (0.7–1.5).
+ if self.speed is not None:
+ config["audio"]["output"]["speed"] = self.speed
+
+ # Turn-detection tuning — graft xAI knobs onto the GA turn_detection
+ # block the parent produced (it always emits ``server_vad``).
+ turn_detection = input_cfg.get("turn_detection")
+ if isinstance(turn_detection, dict):
+ if self.vad_threshold is not None:
+ turn_detection["threshold"] = self.vad_threshold
+ if self.prefix_padding_ms is not None:
+ turn_detection["prefix_padding_ms"] = self.prefix_padding_ms
+ if self.idle_timeout_ms is not None:
+ turn_detection["idle_timeout_ms"] = self.idle_timeout_ms
+
+ # Pronunciation replacements applied before TTS (transcript unchanged).
+ if self.replace:
+ config["replace"] = dict(self.replace)
+
+ # Session resumption — cache + replay conversation turns on reconnect.
+ if self.resumption:
+ config["resumption"] = {"enabled": True}
+
+ # xAI server-side tools (web_search, x_search, mcp, file_search) appended
+ # verbatim to the session tools; xAI executes these on its side.
+ if self.server_tools:
+ config.setdefault("tools", [])
+ config["tools"].extend(dict(t) for t in self.server_tools)
+
+ return config
diff --git a/libraries/python/getpatter/providers/xai_stt.py b/libraries/python/getpatter/providers/xai_stt.py
new file mode 100644
index 00000000..69f4945b
--- /dev/null
+++ b/libraries/python/getpatter/providers/xai_stt.py
@@ -0,0 +1,585 @@
+"""xAI Speech-to-Text for the Patter SDK — streaming adapter + batch helper.
+
+Beta: validated against the xAI STT API spec (docs.x.ai); not yet exercised
+against a live phone call.
+
+Two entry points live here:
+
+* :class:`XaiSTT` — the streaming :class:`~getpatter.providers.base.STTProvider`
+ used by Patter's pipeline mode. Connects to ``wss://api.x.ai/v1/stt`` (config
+ passed entirely via URL query params — there is NO setup message), waits for
+ the ``transcript.created`` handshake, then streams RAW BINARY audio frames in
+ and yields :class:`~getpatter.providers.base.Transcript` events out. The
+ ``is_final`` / ``speech_final`` two-flag scheme mirrors Deepgram's.
+* :func:`transcribe` — a one-shot batch helper wrapping the REST
+ ``POST https://api.x.ai/v1/stt`` endpoint (multipart, ``file`` sent LAST per
+ the xAI requirement), returning a parsed :class:`XaiTranscription`.
+
+Credential: ``XAI_API_KEY`` env var / ``api_key`` argument. Auth is an
+``Authorization: Bearer `` header on both endpoints.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+from dataclasses import dataclass
+from enum import StrEnum
+from typing import Any, AsyncIterator, ClassVar, Optional, Sequence
+from urllib.parse import urlencode
+
+from getpatter.providers.base import STTProvider, Transcript
+
+logger = logging.getLogger("getpatter")
+
+# Lazy import: aiohttp is declared as an optional dep for this provider
+# (shared ``[xai]`` extra with the xAI TTS adapter).
+try: # pragma: no cover - trivial import guard
+ import aiohttp
+except ImportError: # pragma: no cover
+ aiohttp = None # type: ignore
+
+# === Constants ===
+
+# Streaming (WebSocket) Speech-to-Text endpoint.
+XAI_STT_WS_URL = "wss://api.x.ai/v1/stt"
+# Batch (REST) Speech-to-Text endpoint.
+XAI_STT_REST_URL = "https://api.x.ai/v1/stt"
+
+# Streaming STT is billed at $0.20/hr; the batch REST path is cheaper at
+# $0.10/hr. The pipeline meters the streaming rate (see ``pricing.py``); the
+# batch constant below is documented for callers of :func:`transcribe`.
+XAI_STT_BATCH_PRICE_PER_MINUTE = 0.10 / 60
+
+
+class XaiSTTEncoding(StrEnum):
+ """Raw audio encodings accepted by the xAI streaming ``encoding`` param."""
+
+ PCM = "pcm"
+ MULAW = "mulaw"
+ ALAW = "alaw"
+
+
+class XaiClientFrame(StrEnum):
+ """xAI streaming STT client control-message ``type`` values."""
+
+ FINALIZE = "finalize"
+ AUDIO_DONE = "audio.done"
+
+
+class XaiServerEvent(StrEnum):
+ """xAI streaming STT server event ``type`` values."""
+
+ TRANSCRIPT_CREATED = "transcript.created"
+ TRANSCRIPT_PARTIAL = "transcript.partial"
+ TRANSCRIPT_DONE = "transcript.done"
+ ERROR = "error"
+
+
+@dataclass(frozen=True)
+class XaiWord:
+ """A single word from an xAI transcription with timing (and speaker)."""
+
+ text: str
+ start: float
+ end: float
+ # Populated only when diarization is enabled; ``None`` otherwise.
+ speaker: int | None = None
+
+
+@dataclass(frozen=True)
+class XaiTranscription:
+ """Parsed result of a batch :func:`transcribe` call."""
+
+ text: str
+ language: str
+ duration: float
+ words: tuple[XaiWord, ...] = ()
+
+
+def _parse_words(raw_words: Any) -> tuple[XaiWord, ...]:
+ """Convert the raw ``words`` array from an xAI response into ``XaiWord``s."""
+ if not isinstance(raw_words, list):
+ return ()
+ out: list[XaiWord] = []
+ for w in raw_words:
+ if not isinstance(w, dict):
+ continue
+ speaker = w.get("speaker")
+ out.append(
+ XaiWord(
+ text=w.get("text", ""),
+ start=float(w.get("start", 0.0)),
+ end=float(w.get("end", 0.0)),
+ speaker=int(speaker) if speaker is not None else None,
+ )
+ )
+ return tuple(out)
+
+
+class XaiSTT(STTProvider):
+ """xAI streaming STT adapter (Beta).
+
+ Connects to the xAI real-time Speech-to-Text WebSocket API and yields
+ :class:`~getpatter.providers.base.Transcript` objects.
+
+ The default ``encoding="pcm"`` / ``sample_rate=16000`` matches what
+ Patter's pipeline feeds every STT adapter (the carrier bridge decodes
+ inbound mu-law to linear16 @ 16 kHz before the STT, on Twilio and Telnyx
+ alike), so this drops in without transcoding — mirroring
+ :class:`~getpatter.providers.deepgram_stt.DeepgramSTT` /
+ :class:`~getpatter.providers.soniox_stt.SonioxSTT`.
+
+ Args:
+ api_key: xAI API key. Falls back to ``XAI_API_KEY`` when omitted.
+ language: Optional language code (e.g. ``"en"``, ``"it"``) — enables
+ server-side text formatting (ITN) for that language.
+ encoding: Raw audio encoding (``"pcm"`` | ``"mulaw"`` | ``"alaw"``).
+ sample_rate: PCM sample rate (Hz).
+ interim_results: Emit ``is_final=false`` partials (~every 500 ms).
+ endpointing_ms: Silence (ms, 0–5000) before an utterance is marked
+ final. ``None`` keeps the xAI default (10). ``0`` = any VAD
+ boundary.
+ smart_turn: End-of-turn confidence threshold (0.0–1.0) enabling xAI's
+ Smart Turn ML end-of-turn model. ``None`` leaves it off.
+ smart_turn_timeout_ms: Max silence (1–5000 ms) before forcing
+ ``speech_final`` when ``smart_turn`` is set.
+ keyterms: Bias terms (≤100, ≤50 chars each), sent as repeated
+ ``keyterm=`` query params.
+ diarize: Speaker diarization — words gain a ``speaker`` index.
+ filler_words: Include "uh" / "um" in the transcript.
+ base_url: Override the xAI STT WebSocket URL (used by tests).
+ """
+
+ #: Stable pricing/dashboard key — read by stream-handler/metrics.
+ provider_key: ClassVar[str] = "xai"
+
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ language: str | None = None,
+ *,
+ encoding: XaiSTTEncoding | str = XaiSTTEncoding.PCM,
+ sample_rate: int = 16000,
+ interim_results: bool = True,
+ endpointing_ms: int | None = None,
+ smart_turn: float | None = None,
+ smart_turn_timeout_ms: int | None = None,
+ keyterms: Sequence[str] = (),
+ diarize: bool = False,
+ filler_words: bool = False,
+ base_url: str = XAI_STT_WS_URL,
+ ) -> None:
+ if aiohttp is None:
+ raise ImportError(
+ "aiohttp is required for XaiSTT. "
+ "Install with: pip install getpatter[xai]"
+ )
+ resolved_key = api_key or os.environ.get("XAI_API_KEY")
+ if not resolved_key:
+ raise ValueError(
+ "xAI STT requires an api_key. Pass api_key='...' or set "
+ "XAI_API_KEY in the environment."
+ )
+ if smart_turn is not None and not (0.0 <= smart_turn <= 1.0):
+ raise ValueError("smart_turn must be between 0.0 and 1.0")
+
+ self.api_key = resolved_key
+ self.language = language
+ self.encoding = str(encoding)
+ self.sample_rate = sample_rate
+ self.interim_results = interim_results
+ self.endpointing_ms = endpointing_ms
+ self.smart_turn = smart_turn
+ self.smart_turn_timeout_ms = smart_turn_timeout_ms
+ self.keyterms = tuple(keyterms)
+ self.diarize = diarize
+ self.filler_words = filler_words
+ self.base_url = base_url
+
+ self._session: aiohttp.ClientSession | None = None
+ self._owns_session: bool = False
+ self._ws: aiohttp.ClientWebSocketResponse | None = None
+ self._audio_bytes_sent: int = 0
+
+ def __repr__(self) -> str:
+ return (
+ f"XaiSTT(encoding={self.encoding!r}, sample_rate={self.sample_rate}, "
+ f"smart_turn={self.smart_turn}, diarize={self.diarize})"
+ )
+
+ # ------------------------------------------------------------------
+ # URL / query construction
+ # ------------------------------------------------------------------
+
+ def _build_query(self) -> list[tuple[str, str]]:
+ """Build the ordered ``(key, value)`` query params for the WS URL.
+
+ ``keyterm`` is repeatable (one pair per term), so the params are a
+ list of tuples rather than a dict.
+ """
+ params: list[tuple[str, str]] = [
+ ("sample_rate", str(self.sample_rate)),
+ ("encoding", self.encoding),
+ ("interim_results", "true" if self.interim_results else "false"),
+ ]
+ if self.language:
+ params.append(("language", self.language))
+ if self.endpointing_ms is not None:
+ params.append(("endpointing", str(self.endpointing_ms)))
+ if self.diarize:
+ params.append(("diarize", "true"))
+ if self.filler_words:
+ params.append(("filler_words", "true"))
+ if self.smart_turn is not None:
+ params.append(("smart_turn", str(self.smart_turn)))
+ if self.smart_turn_timeout_ms is not None:
+ params.append(("smart_turn_timeout", str(self.smart_turn_timeout_ms)))
+ for term in self.keyterms:
+ params.append(("keyterm", term))
+ return params
+
+ def _build_url(self) -> str:
+ return f"{self.base_url}?{urlencode(self._build_query())}"
+
+ def _record_transcript_cost(self) -> None:
+ """Emit ``patter.cost.stt_seconds`` for buffered audio."""
+ try:
+ from getpatter.observability.attributes import record_patter_attrs
+
+ bytes_per_sample = 1 if self.encoding in ("mulaw", "alaw") else 2
+ seconds = self._audio_bytes_sent / float(
+ self.sample_rate * bytes_per_sample
+ )
+ record_patter_attrs(
+ {
+ "patter.cost.stt_seconds": seconds,
+ "patter.stt.provider": "xai",
+ }
+ )
+ self._audio_bytes_sent = 0
+ except Exception: # pragma: no cover — defense in depth
+ logger.debug("_record_transcript_cost failed", exc_info=True)
+
+ # ------------------------------------------------------------------
+ # Connection lifecycle
+ # ------------------------------------------------------------------
+
+ async def connect(self) -> None:
+ """Open the WebSocket and wait for the ``transcript.created`` handshake.
+
+ Config is carried entirely in the URL query string — no setup message
+ is sent. The server signals readiness with ``transcript.created``; we
+ block on it so ``send_audio`` never races ahead of a ready session.
+ """
+ if self._ws is not None:
+ return # Already connected — idempotent.
+
+ if self._session is None:
+ self._session = aiohttp.ClientSession()
+ self._owns_session = True
+
+ headers = {"Authorization": f"Bearer {self.api_key}"}
+ try:
+ self._ws = await self._session.ws_connect(
+ self._build_url(), headers=headers
+ )
+ await self._await_transcript_created()
+ except Exception:
+ if self._owns_session and self._session is not None:
+ await self._session.close()
+ self._session = None
+ self._owns_session = False
+ self._ws = None
+ raise
+
+ async def _await_transcript_created(self) -> None:
+ """Read frames until ``transcript.created`` (raise on ``error``)."""
+ assert self._ws is not None
+ async for msg in self._ws:
+ if msg.type != aiohttp.WSMsgType.TEXT:
+ continue
+ try:
+ data = json.loads(msg.data)
+ except json.JSONDecodeError:
+ continue
+ event_type = data.get("type")
+ if event_type == XaiServerEvent.TRANSCRIPT_CREATED.value:
+ return
+ if event_type == XaiServerEvent.ERROR.value:
+ raise RuntimeError(
+ f"xAI STT setup error: {data.get('message', 'unknown')}"
+ )
+ raise RuntimeError("xAI STT closed before transcript.created handshake")
+
+ async def send_audio(self, audio_chunk: bytes) -> None:
+ """Send a raw audio chunk as a binary WebSocket frame (no base64)."""
+ if self._ws is None:
+ raise RuntimeError("XaiSTT is not connected. Call connect() first.")
+ if not audio_chunk:
+ return
+ self._audio_bytes_sent += len(audio_chunk)
+ await self._ws.send_bytes(audio_chunk)
+
+ async def finalize(self) -> None:
+ """Force the current utterance to ``speech_final`` immediately.
+
+ The pipeline's VAD ``speech_end`` fast-path duck-types ``stt.finalize``
+ — without it every turn waits out the full endpointing delay. xAI's WS
+ API accepts ``{"type": "finalize"}`` for exactly this.
+ """
+ if self._ws is None or self._ws.closed:
+ return
+ try:
+ await self._ws.send_str(json.dumps({"type": XaiClientFrame.FINALIZE.value}))
+ except Exception as exc: # noqa: BLE001 - best-effort fast path
+ logger.debug("xAI STT finalize failed: %s", exc)
+
+ # ------------------------------------------------------------------
+ # Transcript stream
+ # ------------------------------------------------------------------
+
+ async def receive_transcripts(self) -> AsyncIterator[Transcript]:
+ """Yield :class:`Transcript` objects from the xAI stream.
+
+ Event mapping (mirrors Deepgram's is_final/speech_final scheme):
+ - ``transcript.partial`` ``is_final=false`` → interim result.
+ - ``transcript.partial`` ``is_final=true, speech_final=false`` → chunk
+ final (~3 s locked).
+ - ``transcript.partial`` ``is_final=true, speech_final=true`` →
+ utterance final.
+ - ``transcript.done`` → final flush, then the stream closes.
+ - ``error`` → logged; the connection stays open.
+ """
+ if self._ws is None:
+ raise RuntimeError("XaiSTT is not connected. Call connect() first.")
+
+ async for msg in self._ws:
+ if msg.type in (
+ aiohttp.WSMsgType.CLOSED,
+ aiohttp.WSMsgType.CLOSE,
+ aiohttp.WSMsgType.CLOSING,
+ ):
+ break
+ if msg.type == aiohttp.WSMsgType.ERROR:
+ logger.error("XaiSTT WebSocket error: %s", self._ws.exception())
+ break
+ if msg.type != aiohttp.WSMsgType.TEXT:
+ continue
+
+ try:
+ data = json.loads(msg.data)
+ except json.JSONDecodeError:
+ logger.warning("XaiSTT: received non-JSON message")
+ continue
+
+ event_type = data.get("type")
+
+ if event_type == XaiServerEvent.ERROR.value:
+ # Errors do NOT close the connection — surface and keep going.
+ logger.error("XaiSTT error: %s", data.get("message", "unknown"))
+ continue
+
+ if event_type == XaiServerEvent.TRANSCRIPT_PARTIAL.value:
+ text = (data.get("text") or "").strip()
+ if not text:
+ continue
+ speech_final = bool(data.get("speech_final", False))
+ is_final = bool(data.get("is_final", False) or speech_final)
+ if is_final:
+ self._record_transcript_cost()
+ yield Transcript(
+ text=text,
+ is_final=is_final,
+ speech_final=speech_final,
+ words=_forward_words(data.get("words")),
+ )
+
+ elif event_type == XaiServerEvent.TRANSCRIPT_DONE.value:
+ text = (data.get("text") or "").strip()
+ if text:
+ self._record_transcript_cost()
+ yield Transcript(
+ text=text,
+ is_final=True,
+ speech_final=True,
+ words=_forward_words(data.get("words")),
+ )
+ break
+
+ # ------------------------------------------------------------------
+ # Cleanup
+ # ------------------------------------------------------------------
+
+ async def close(self) -> None:
+ """Signal end-of-audio, then close the WebSocket and owned resources."""
+ if self._ws is not None:
+ try:
+ await self._ws.send_str(
+ json.dumps({"type": XaiClientFrame.AUDIO_DONE.value})
+ )
+ except Exception: # noqa: BLE001
+ pass
+ try:
+ await self._ws.close()
+ except Exception: # noqa: BLE001
+ pass
+ self._ws = None
+
+ if self._owns_session and self._session is not None:
+ await self._session.close()
+ self._session = None
+ self._owns_session = False
+
+
+def _forward_words(raw_words: Any) -> tuple[dict[str, Any], ...]:
+ """Pass through the provider ``words`` array as a tuple of dicts."""
+ if not isinstance(raw_words, list):
+ return ()
+ return tuple(w for w in raw_words if isinstance(w, dict))
+
+
+# ---------------------------------------------------------------------------
+# Batch transcription helper (REST)
+# ---------------------------------------------------------------------------
+
+
+def _build_transcribe_fields(
+ *,
+ audio: bytes | None,
+ url: str | None,
+ language: str | None,
+ format: bool,
+ diarize: bool,
+ keyterms: Sequence[str],
+ filler_words: bool,
+ audio_format: str | None,
+ sample_rate: int | None,
+ filename: str,
+) -> list[tuple[str, Any, dict[str, Any]]]:
+ """Return the ordered multipart fields for :func:`transcribe`.
+
+ xAI requires the ``file`` part to be LAST, so all scalar fields come first
+ and the file (or ``url``) is appended at the end. Each entry is
+ ``(name, value, extra_kwargs)`` where ``extra_kwargs`` feeds
+ ``FormData.add_field`` (e.g. ``filename`` / ``content_type`` for the file).
+ """
+ fields: list[tuple[str, Any, dict[str, Any]]] = []
+ if language is not None:
+ fields.append(("language", language, {}))
+ if format:
+ fields.append(("format", "true", {}))
+ if diarize:
+ fields.append(("diarize", "true", {}))
+ if filler_words:
+ fields.append(("filler_words", "true", {}))
+ if audio_format is not None:
+ fields.append(("audio_format", audio_format, {}))
+ if sample_rate is not None:
+ fields.append(("sample_rate", str(sample_rate), {}))
+ for term in keyterms:
+ fields.append(("keyterm", term, {}))
+ # ``file`` / ``url`` LAST — xAI requires the binary part to be the final
+ # multipart field.
+ if url is not None:
+ fields.append(("url", url, {}))
+ else:
+ assert audio is not None
+ fields.append(
+ (
+ "file",
+ audio,
+ {"filename": filename, "content_type": "application/octet-stream"},
+ )
+ )
+ return fields
+
+
+async def transcribe(
+ audio: bytes | None = None,
+ *,
+ url: str | None = None,
+ api_key: str | None = None,
+ language: str | None = None,
+ format: bool = False, # noqa: A002 - matches the xAI API field name
+ diarize: bool = False,
+ keyterms: Sequence[str] = (),
+ filler_words: bool = False,
+ audio_format: str | None = None,
+ sample_rate: int | None = None,
+ filename: str = "audio.wav",
+) -> XaiTranscription:
+ """Transcribe an audio clip via the xAI batch REST endpoint (Beta).
+
+ Exactly one of ``audio`` (raw bytes) or ``url`` (server-side download) is
+ required. Container formats (WAV/MP3/…) are auto-detected; for raw
+ headerless audio pass ``audio_format`` (``pcm`` | ``mulaw`` | ``alaw``) and
+ ``sample_rate``.
+
+ Args:
+ audio: Audio file bytes (max 500 MB). Mutually exclusive with ``url``.
+ url: URL for xAI to download and transcribe server-side.
+ api_key: xAI API key. Falls back to ``XAI_API_KEY``.
+ language: Language code; used with ``format=True`` for ITN formatting.
+ format: Enable Inverse Text Normalization (requires ``language``).
+ diarize: Speaker diarization — words gain a ``speaker`` index.
+ keyterms: Bias terms (≤100, ≤50 chars each).
+ filler_words: Include "uh" / "um".
+ audio_format: Only for raw audio (``pcm`` | ``mulaw`` | ``alaw``).
+ sample_rate: Only for raw audio (Hz).
+ filename: Multipart filename for the uploaded ``file`` part.
+
+ Returns:
+ A parsed :class:`XaiTranscription`.
+ """
+ if aiohttp is None:
+ raise ImportError(
+ "aiohttp is required for xAI transcribe(). "
+ "Install with: pip install getpatter[xai]"
+ )
+ if (audio is None) == (url is None):
+ raise ValueError("transcribe() requires exactly one of audio= or url=")
+ resolved_key = api_key or os.environ.get("XAI_API_KEY")
+ if not resolved_key:
+ raise ValueError(
+ "xAI transcribe() requires an api_key. Pass api_key='...' or set "
+ "XAI_API_KEY in the environment."
+ )
+
+ form = aiohttp.FormData()
+ for name, value, extra in _build_transcribe_fields(
+ audio=audio,
+ url=url,
+ language=language,
+ format=format,
+ diarize=diarize,
+ keyterms=keyterms,
+ filler_words=filler_words,
+ audio_format=audio_format,
+ sample_rate=sample_rate,
+ filename=filename,
+ ):
+ form.add_field(name, value, **extra)
+
+ headers = {"Authorization": f"Bearer {resolved_key}"}
+ async with aiohttp.ClientSession() as session:
+ async with session.post(
+ XAI_STT_REST_URL,
+ headers=headers,
+ data=form,
+ timeout=aiohttp.ClientTimeout(total=300),
+ ) as resp:
+ if resp.status != 200:
+ body = await resp.text()
+ raise RuntimeError(f"xAI STT error {resp.status}: {body[:500]}")
+ payload = await resp.json()
+
+ return XaiTranscription(
+ text=payload.get("text", ""),
+ language=payload.get("language", ""),
+ duration=float(payload.get("duration", 0.0)),
+ words=_parse_words(payload.get("words")),
+ )
+
+
diff --git a/libraries/python/getpatter/providers/xai_tts.py b/libraries/python/getpatter/providers/xai_tts.py
new file mode 100644
index 00000000..0643b469
--- /dev/null
+++ b/libraries/python/getpatter/providers/xai_tts.py
@@ -0,0 +1,318 @@
+"""xAI Text-to-Speech for the Patter SDK — REST one-shot bytes endpoint.
+
+Beta: validated against the xAI TTS API spec (docs.x.ai); not yet exercised
+against a live phone call.
+
+Targets ``POST https://api.x.ai/v1/tts`` which returns raw audio bytes in the
+requested codec. This maps cleanly onto Patter's
+``TTSProvider.synthesize(text) -> AsyncIterator[bytes]`` contract and needs no
+vendor SDK — only ``aiohttp`` (mirrors :class:`SonioxTTS`). xAI also exposes a
+bidirectional WebSocket streaming variant; that is out of scope for v1 (a future
+``xai_ws_tts`` provider, mirroring ``elevenlabs_ws``, could add it).
+
+Credential: ``XAI_API_KEY`` env var / ``api_key`` argument. REST auth is an
+``Authorization: Bearer `` header.
+
+Telephony: xAI natively supports G.711 mu-law @ 8 kHz — request ``codec="mulaw"``
++ ``sample_rate=8000`` (the :meth:`for_twilio` / :meth:`for_telnyx` factories do
+this) to emit carrier-native audio with NO resampling, exactly matching Patter's
+mu-law / 8 kHz wire format.
+"""
+
+from __future__ import annotations
+
+import logging
+import os
+from typing import Any, AsyncIterator, ClassVar, Optional
+
+from getpatter.providers.base import TTSProvider
+
+logger = logging.getLogger("getpatter.providers.xai_tts")
+
+# Lazy import: aiohttp is declared as an optional dep for this provider
+# (shared ``[xai]`` extra with the xAI STT adapter).
+try: # pragma: no cover - trivial import guard
+ import aiohttp
+except ImportError: # pragma: no cover
+ aiohttp = None # type: ignore
+
+# REST one-shot text-to-speech endpoint. Returns raw audio bytes.
+XAI_TTS_REST_URL = "https://api.x.ai/v1/tts"
+# Custom-voice creation endpoint (multipart upload of a reference clip).
+XAI_CUSTOM_VOICES_URL = "https://api.x.ai/v1/custom-voices"
+
+# xAI's default voice. The full 26-voice roster is shared with the Voice Agent
+# API; ``eve`` is the documented default.
+XAI_DEFAULT_VOICE = "eve"
+
+# Pipeline-friendly defaults when the caller leaves codec / sample_rate unset:
+# raw PCM-16-LE @ 16 kHz (NOT xAI's own mp3 default, which the pipeline can't
+# resample). Telephony uses the carrier factories below instead.
+XAI_DEFAULT_CODEC = "pcm"
+XAI_DEFAULT_SAMPLE_RATE = 16000
+
+# Maximum text length accepted by a single REST request (xAI limit).
+XAI_TTS_MAX_CHARS = 15000
+
+
+class XaiTTS(TTSProvider):
+ """xAI TTS over the HTTP one-shot ``/tts`` bytes endpoint (Beta).
+
+ Default output is raw PCM-16-LE @ 16 kHz so it drops into Patter's pipeline
+ without transcoding. Default voice is ``eve`` and default ``language`` is
+ ``"auto"`` (xAI auto-detects the language of the text).
+
+ Telephony optimization
+ ----------------------
+ The constructor default ``codec="pcm"`` / ``sample_rate=16000`` is correct
+ for web playback and 16 kHz pipelines. For real phone calls use the carrier
+ factories:
+
+ * :meth:`for_twilio` — emits ``mulaw`` @ 8 kHz, Twilio's exact wire codec,
+ so the pipeline skips resampling AND PCM -> mu-law encoding (bit-clean
+ passthrough). The sender reads the declared output format via
+ :meth:`source_audio_format`.
+ * :meth:`for_telnyx` — emits ``pcm`` @ 16 kHz for the Telnyx PCM16 pipeline.
+ """
+
+ #: Stable pricing/dashboard key — read by stream-handler/metrics.
+ provider_key: ClassVar[str] = "xai_tts"
+
+ def __init__(
+ self,
+ api_key: Optional[str] = None,
+ *,
+ voice: str = XAI_DEFAULT_VOICE,
+ language: str = "auto",
+ speed: Optional[float] = None,
+ optimize_streaming_latency: Optional[int] = None,
+ text_normalization: bool = False,
+ codec: Optional[str] = None,
+ sample_rate: Optional[int] = None,
+ bit_rate: Optional[int] = None,
+ base_url: str = XAI_TTS_REST_URL,
+ session: Optional["aiohttp.ClientSession"] = None,
+ ) -> None:
+ if aiohttp is None:
+ raise ImportError(
+ "aiohttp is required for XaiTTS. "
+ "Install with: pip install getpatter[xai]"
+ )
+ resolved_key = api_key or os.environ.get("XAI_API_KEY")
+ if not resolved_key:
+ raise ValueError(
+ "xAI TTS requires an api_key. Pass api_key='...' or set "
+ "XAI_API_KEY in the environment."
+ )
+
+ self.api_key = resolved_key
+ self.voice = voice
+ self.language = language
+ self.speed = speed
+ self.optimize_streaming_latency = optimize_streaming_latency
+ self.text_normalization = text_normalization
+ self.codec = codec or XAI_DEFAULT_CODEC
+ self.sample_rate = sample_rate or XAI_DEFAULT_SAMPLE_RATE
+ # MP3-only bit rate (bits/sec); nested under ``output_format`` when set.
+ self.bit_rate = bit_rate
+ self.base_url = base_url
+ self._owns_session = session is None
+ self._session = session
+
+ def __repr__(self) -> str:
+ # Never leak the API key in repr / logs.
+ return (
+ f"XaiTTS(voice={self.voice!r}, language={self.language!r}, "
+ f"codec={self.codec!r}, sample_rate={self.sample_rate})"
+ )
+
+ # ------------------------------------------------------------------
+ # Telephony factories
+ # ------------------------------------------------------------------
+
+ @classmethod
+ def for_twilio(
+ cls,
+ api_key: Optional[str] = None,
+ **kwargs: Any,
+ ) -> "XaiTTS":
+ """Build an instance pre-configured for Twilio Media Streams.
+
+ Emits ``mulaw`` @ 8 kHz — exactly Twilio's wire codec — so the pipeline
+ takes the passthrough path: zero resampling, zero PCM -> mu-law
+ encoding, bit-clean audio.
+ """
+ kwargs.pop("codec", None)
+ kwargs.pop("sample_rate", None)
+ return cls(api_key=api_key, codec="mulaw", sample_rate=8000, **kwargs)
+
+ @classmethod
+ def for_telnyx(
+ cls,
+ api_key: Optional[str] = None,
+ **kwargs: Any,
+ ) -> "XaiTTS":
+ """Build an instance pre-configured for Telnyx bidirectional media.
+
+ Emits ``pcm`` @ 16 kHz to match the Telnyx PCM16 pipeline; the SDK
+ resamples to the 8 kHz PCMU wire once, downstream.
+ """
+ kwargs.pop("codec", None)
+ kwargs.pop("sample_rate", None)
+ return cls(api_key=api_key, codec="pcm", sample_rate=16000, **kwargs)
+
+ def source_audio_format(self) -> "AudioFormat":
+ """Declare the audio format this adapter emits so the pipeline sender
+ derives the correct resample ratio (or skips it for mu-law passthrough)
+ instead of assuming a fixed 16 kHz source. See ``getpatter.audio.format``.
+ """
+ from getpatter.audio.format import AudioFormat
+
+ if self.codec == "mulaw":
+ return AudioFormat(encoding="mulaw", sample_rate=int(self.sample_rate))
+ if self.codec == "alaw":
+ return AudioFormat(encoding="alaw", sample_rate=int(self.sample_rate))
+ return AudioFormat(encoding="pcm_s16le", sample_rate=int(self.sample_rate))
+
+ def _ensure_session(self) -> "aiohttp.ClientSession":
+ if self._session is None:
+ self._session = aiohttp.ClientSession()
+ self._owns_session = True
+ return self._session
+
+ def _build_payload(self, text: str) -> dict[str, Any]:
+ output_format: dict[str, Any] = {
+ "codec": self.codec,
+ "sample_rate": int(self.sample_rate),
+ }
+ if self.bit_rate is not None:
+ output_format["bit_rate"] = self.bit_rate
+ payload: dict[str, Any] = {
+ "text": text,
+ "voice_id": self.voice,
+ "language": self.language,
+ "output_format": output_format,
+ }
+ if self.speed is not None:
+ payload["speed"] = self.speed
+ if self.optimize_streaming_latency is not None:
+ payload["optimize_streaming_latency"] = self.optimize_streaming_latency
+ if self.text_normalization:
+ payload["text_normalization"] = True
+ return payload
+
+ def _record_synthesis_cost(self, text: str) -> None:
+ """Emit ``patter.cost.tts_chars`` for the synthesised text."""
+ try:
+ from getpatter.observability.attributes import record_patter_attrs
+
+ record_patter_attrs(
+ {
+ "patter.cost.tts_chars": len(text),
+ "patter.tts.provider": "xai_tts",
+ }
+ )
+ except Exception: # pragma: no cover — defense in depth
+ logger.debug("_record_synthesis_cost failed", exc_info=True)
+
+ async def synthesize(self, text: str) -> AsyncIterator[bytes]:
+ """Stream raw audio bytes for ``text`` over HTTP.
+
+ With the default ``codec="pcm"`` these are raw PCM-16-LE chunks at the
+ configured ``sample_rate``; with ``mulaw`` (see :meth:`for_twilio`) they
+ are carrier-native G.711 mu-law bytes.
+ """
+ if len(text) > XAI_TTS_MAX_CHARS:
+ # xAI rejects a single request over 15,000 chars. Splitting text is
+ # not this adapter's job (the pipeline synthesises per-utterance),
+ # so warn and send as-is — the API error is surfaced below.
+ logger.warning(
+ "xAI TTS text is %d chars, over the %d-char per-request limit; "
+ "the request will likely be rejected.",
+ len(text),
+ XAI_TTS_MAX_CHARS,
+ )
+ self._record_synthesis_cost(text)
+ session = self._ensure_session()
+
+ headers = {
+ "Authorization": f"Bearer {self.api_key}",
+ "Content-Type": "application/json",
+ }
+
+ async with session.post(
+ self.base_url,
+ headers=headers,
+ json=self._build_payload(text),
+ timeout=aiohttp.ClientTimeout(total=30),
+ ) as resp:
+ if resp.status != 200:
+ body = await resp.text()
+ raise RuntimeError(f"xAI TTS error {resp.status}: {body[:500]}")
+ async for chunk in resp.content.iter_chunked(4096):
+ if chunk:
+ yield chunk
+
+ async def close(self) -> None:
+ """Close the underlying session (idempotent)."""
+ if self._session is not None and self._owns_session:
+ await self._session.close()
+ self._session = None
+
+
+async def create_custom_voice(
+ name: str,
+ language: str,
+ audio: bytes,
+ *,
+ api_key: str | None = None,
+ filename: str = "reference.wav",
+) -> str:
+ """Create an xAI custom voice from a short reference clip (Beta).
+
+ POSTs multipart to ``https://api.x.ai/v1/custom-voices`` with the scalar
+ ``name`` / ``language`` fields first and the ``file`` (audio/wav, clip
+ ≤120 s) LAST — the xAI multipart requirement. Returns the resulting
+ ``voice_id`` string, usable as ``voice`` on both :class:`XaiTTS` and the
+ xAI Voice Agent engine.
+
+ Args:
+ name: Display name for the custom voice.
+ language: BCP-47 language code of the reference clip.
+ audio: WAV audio bytes of the reference clip (≤120 s).
+ api_key: xAI API key. Falls back to ``XAI_API_KEY``.
+ filename: Multipart filename for the uploaded ``file`` part.
+ """
+ if aiohttp is None:
+ raise ImportError(
+ "aiohttp is required for xAI create_custom_voice(). "
+ "Install with: pip install getpatter[xai]"
+ )
+ resolved_key = api_key or os.environ.get("XAI_API_KEY")
+ if not resolved_key:
+ raise ValueError(
+ "xAI create_custom_voice() requires an api_key. Pass api_key='...' "
+ "or set XAI_API_KEY in the environment."
+ )
+
+ form = aiohttp.FormData()
+ form.add_field("name", name)
+ form.add_field("language", language)
+ # ``file`` LAST — xAI requires the binary part to be the final field.
+ form.add_field("file", audio, filename=filename, content_type="audio/wav")
+
+ headers = {"Authorization": f"Bearer {resolved_key}"}
+ async with aiohttp.ClientSession() as session:
+ async with session.post(
+ XAI_CUSTOM_VOICES_URL,
+ headers=headers,
+ data=form,
+ timeout=aiohttp.ClientTimeout(total=120),
+ ) as resp:
+ if resp.status != 200:
+ body = await resp.text()
+ raise RuntimeError(
+ f"xAI custom-voice error {resp.status}: {body[:500]}"
+ )
+ payload = await resp.json()
+ return payload.get("voice_id", "")
diff --git a/libraries/python/getpatter/server.py b/libraries/python/getpatter/server.py
index a6fcd4fd..ad6169b9 100644
--- a/libraries/python/getpatter/server.py
+++ b/libraries/python/getpatter/server.py
@@ -1597,6 +1597,7 @@ async def twilio_stream_handler(websocket: WebSocket, call_id: str):
pop_prewarm_audio=self.pop_prewarm_audio,
pop_prewarmed_connections=self.pop_prewarmed_connections,
openai_key=self.config.openai_key,
+ xai_key=self.config.xai_key,
# SECURITY (#204): validate the -borne token
# (start.customParameters) against the token minted for this
# call_id BEFORE the provider session opens. Fail-closed by
@@ -2088,6 +2089,7 @@ async def telnyx_stream_handler(websocket: WebSocket, call_id: str):
pop_prewarmed_connections=self.pop_prewarmed_connections,
speech_events=getattr(self, "speech_events", None),
openai_key=self.config.openai_key,
+ xai_key=self.config.xai_key,
on_call_start=_start,
on_call_end=_end,
on_transcript=_transcript,
@@ -2358,6 +2360,7 @@ async def plivo_stream_websocket(websocket: WebSocket, call_id: str):
pop_prewarm_audio=self.pop_prewarm_audio,
pop_prewarmed_connections=self.pop_prewarmed_connections,
openai_key=self.config.openai_key,
+ xai_key=self.config.xai_key,
# SECURITY (#204): validate the X-Patter-Stream-Token
# extra-header against the token minted for this call_id
# BEFORE the provider session opens. Fail-closed by default.
diff --git a/libraries/python/getpatter/services/metrics.py b/libraries/python/getpatter/services/metrics.py
index c9e8a904..5bedafcb 100644
--- a/libraries/python/getpatter/services/metrics.py
+++ b/libraries/python/getpatter/services/metrics.py
@@ -734,6 +734,12 @@ def record_realtime_usage(self, usage: dict, model: str | None = None) -> None:
time (``self.realtime_model``); pass an explicit value to override
per-call (the ``response.done`` payload carries the model used).
"""
+ # Per-minute realtime engines (e.g. xAI Grok Voice Agent) are billed on
+ # session duration in ``_compute_cost``, NOT on token usage — skip the
+ # token accounting so no bogus per-token cost / cached-savings figure
+ # is attributed to them.
+ if self.provider_mode == "xai_realtime":
+ return
resolved_model = model or self.realtime_model or None
self._total_realtime_cost += calculate_realtime_cost(
usage, self._pricing, model=resolved_model
@@ -1111,6 +1117,18 @@ def _compute_cost(self, duration_seconds: float) -> CostBreakdown:
stt_cost = 0.0
tts_cost = 0.0
llm_cost = self._total_realtime_cost
+ elif self.provider_mode == "xai_realtime":
+ # xAI Grok Voice Agent: billed per MINUTE of session audio, not per
+ # token. Bill the call duration through the MINUTE-unit path of
+ # ``calculate_realtime_cost`` (``_total_realtime_cost`` stays 0 here).
+ stt_cost = 0.0
+ tts_cost = 0.0
+ llm_cost = calculate_realtime_cost(
+ {},
+ self._pricing,
+ provider="xai_realtime",
+ duration_seconds=duration_seconds,
+ )
elif self.provider_mode == "elevenlabs_convai":
# ElevenLabs ConvAI: bundled pricing, estimate from duration
stt_cost = 0.0
diff --git a/libraries/python/getpatter/stream_handler.py b/libraries/python/getpatter/stream_handler.py
index 4f10446a..d9f0b959 100644
--- a/libraries/python/getpatter/stream_handler.py
+++ b/libraries/python/getpatter/stream_handler.py
@@ -1427,6 +1427,7 @@ def __init__(
metrics,
*,
openai_key: str,
+ xai_key: str = "",
transfer_fn=None,
hangup_fn=None,
on_transcript=None,
@@ -1455,6 +1456,8 @@ def __init__(
speech_events=speech_events,
)
self._openai_key = openai_key
+ # xAI Grok Voice Agent key (used when ``agent.provider == "xai_realtime"``).
+ self._xai_key = xai_key
self._transfer_fn = transfer_fn
self._hangup_fn = hangup_fn
self._audio_format = audio_format
@@ -1871,7 +1874,17 @@ async def start(self) -> None:
OpenAIRealtime2Adapter,
)
- _adapter_cls = OpenAIRealtime2Adapter
+ # xAI's Grok Voice Agent is OpenAI-Realtime-GA-compatible, so its adapter
+ # subclasses the GA one; select it (and its key) when the engine is xAI.
+ _is_xai = getattr(self.agent, "provider", None) == "xai_realtime"
+ if _is_xai:
+ from getpatter.providers.xai_realtime import ( # type: ignore[import]
+ XaiRealtimeAdapter,
+ )
+
+ _adapter_cls = XaiRealtimeAdapter
+ else:
+ _adapter_cls = OpenAIRealtime2Adapter
# Resolve MCP servers BEFORE the adapter is built so the
# discovered tools are visible in the first ``session.update``.
@@ -1967,6 +1980,14 @@ async def start(self) -> None:
)
if gate_response is not None:
adapter_kwargs["gate_response_on_transcript"] = gate_response
+ if _is_xai:
+ # xAI uses its own key and its own session-tuning knobs (carried on
+ # ``agent.xai_realtime``); forward only the keys that were set so the
+ # adapter's defaults stay authoritative otherwise.
+ adapter_kwargs["api_key"] = self._xai_key or self._openai_key
+ for _k, _v in (getattr(self.agent, "xai_realtime", None) or {}).items():
+ if _v is not None:
+ adapter_kwargs[_k] = _v
self._adapter = _adapter_cls(**adapter_kwargs)
# Try to adopt a Realtime WebSocket parked during the ringing
diff --git a/libraries/python/getpatter/stt/__init__.py b/libraries/python/getpatter/stt/__init__.py
index 3b1db62a..ffdca980 100644
--- a/libraries/python/getpatter/stt/__init__.py
+++ b/libraries/python/getpatter/stt/__init__.py
@@ -22,4 +22,5 @@
"speechmatics",
"assemblyai",
"openai_transcribe",
+ "xai",
]
diff --git a/libraries/python/getpatter/stt/xai.py b/libraries/python/getpatter/stt/xai.py
new file mode 100644
index 00000000..ff9d5630
--- /dev/null
+++ b/libraries/python/getpatter/stt/xai.py
@@ -0,0 +1,59 @@
+"""xAI streaming STT for Patter pipeline mode (Beta)."""
+
+from __future__ import annotations
+
+import os
+from typing import ClassVar, Sequence
+
+from getpatter.providers.xai_stt import XaiSTT as _XaiSTT
+
+__all__ = ["STT"]
+
+
+class STT(_XaiSTT):
+ """xAI real-time streaming STT (Beta).
+
+ Example::
+
+ from getpatter.stt import xai
+
+ stt = xai.STT() # reads XAI_API_KEY
+ stt = xai.STT(api_key="...", language="en", smart_turn=0.7)
+ """
+
+ provider_key: ClassVar[str] = "xai"
+
+ def __init__(
+ self,
+ api_key: str | None = None,
+ language: str | None = None,
+ *,
+ encoding: str = "pcm",
+ sample_rate: int = 16000,
+ interim_results: bool = True,
+ endpointing_ms: int | None = None,
+ smart_turn: float | None = None,
+ smart_turn_timeout_ms: int | None = None,
+ keyterms: Sequence[str] = (),
+ diarize: bool = False,
+ filler_words: bool = False,
+ ) -> None:
+ key = api_key or os.environ.get("XAI_API_KEY")
+ if not key:
+ raise ValueError(
+ "xAI STT requires an api_key. Pass api_key='...' or "
+ "set XAI_API_KEY in the environment."
+ )
+ super().__init__(
+ key,
+ language,
+ encoding=encoding,
+ sample_rate=sample_rate,
+ interim_results=interim_results,
+ endpointing_ms=endpointing_ms,
+ smart_turn=smart_turn,
+ smart_turn_timeout_ms=smart_turn_timeout_ms,
+ keyterms=keyterms,
+ diarize=diarize,
+ filler_words=filler_words,
+ )
diff --git a/libraries/python/getpatter/telemetry/stack.py b/libraries/python/getpatter/telemetry/stack.py
index c5bebdae..3ddd7597 100644
--- a/libraries/python/getpatter/telemetry/stack.py
+++ b/libraries/python/getpatter/telemetry/stack.py
@@ -43,6 +43,7 @@ class constant (minification-safe, the canonical vendor id) and stores its
"rime",
"inworld",
"sarvam",
+ "xai",
"telnyx",
"other",
}
@@ -56,6 +57,8 @@ class constant (minification-safe, the canonical vendor id) and stores its
"openai_transcribe": "openai",
"elevenlabs_ws": "elevenlabs",
"soniox_tts": "soniox",
+ "xai_tts": "xai",
+ "xai_realtime": "xai",
"telnyx_stt": "telnyx",
"telnyx_tts": "telnyx",
}
diff --git a/libraries/python/getpatter/telephony/plivo.py b/libraries/python/getpatter/telephony/plivo.py
index 39b924b0..2c14b825 100644
--- a/libraries/python/getpatter/telephony/plivo.py
+++ b/libraries/python/getpatter/telephony/plivo.py
@@ -350,6 +350,7 @@ async def plivo_stream_bridge(
websocket,
agent,
openai_key: str,
+ xai_key: str = "",
on_call_start=None,
on_call_end=None,
on_transcript=None,
@@ -667,6 +668,7 @@ async def _plivo_send_dtmf(digits: str, delay_ms: int = 0) -> None:
resolved_prompt=resolved_prompt,
metrics=metrics,
openai_key=openai_key,
+ xai_key=xai_key,
transfer_fn=_plivo_transfer,
hangup_fn=_plivo_hangup,
on_transcript=on_transcript,
diff --git a/libraries/python/getpatter/telephony/telnyx.py b/libraries/python/getpatter/telephony/telnyx.py
index 5bf2e6fb..7047e860 100644
--- a/libraries/python/getpatter/telephony/telnyx.py
+++ b/libraries/python/getpatter/telephony/telnyx.py
@@ -275,6 +275,7 @@ async def telnyx_stream_bridge(
websocket,
agent,
openai_key: str,
+ xai_key: str = "",
on_call_start=None,
on_call_end=None,
on_transcript=None,
@@ -672,6 +673,7 @@ async def _telnyx_stop_recording() -> None:
resolved_prompt=resolved_prompt,
metrics=metrics,
openai_key=openai_key,
+ xai_key=xai_key,
transfer_fn=_telnyx_transfer,
hangup_fn=_telnyx_hangup,
on_transcript=on_transcript,
diff --git a/libraries/python/getpatter/telephony/twilio.py b/libraries/python/getpatter/telephony/twilio.py
index 5f416b73..a7961a59 100644
--- a/libraries/python/getpatter/telephony/twilio.py
+++ b/libraries/python/getpatter/telephony/twilio.py
@@ -433,6 +433,7 @@ async def twilio_stream_bridge(
websocket,
agent,
openai_key: str,
+ xai_key: str = "",
on_call_start=None,
on_call_end=None,
on_transcript=None,
@@ -785,6 +786,7 @@ async def _twilio_hangup():
resolved_prompt=resolved_prompt,
metrics=metrics,
openai_key=openai_key,
+ xai_key=xai_key,
transfer_fn=_twilio_transfer,
hangup_fn=_twilio_hangup,
on_transcript=on_transcript,
diff --git a/libraries/python/getpatter/tts/__init__.py b/libraries/python/getpatter/tts/__init__.py
index 425456e0..3bc2d197 100644
--- a/libraries/python/getpatter/tts/__init__.py
+++ b/libraries/python/getpatter/tts/__init__.py
@@ -21,4 +21,5 @@
"inworld",
"soniox",
"sarvam",
+ "xai",
]
diff --git a/libraries/python/getpatter/tts/xai.py b/libraries/python/getpatter/tts/xai.py
new file mode 100644
index 00000000..2b911540
--- /dev/null
+++ b/libraries/python/getpatter/tts/xai.py
@@ -0,0 +1,113 @@
+"""xAI TTS for Patter pipeline mode (Beta)."""
+
+from __future__ import annotations
+
+import os
+from typing import ClassVar, Optional
+
+from getpatter.providers.xai_tts import XaiTTS as _XaiTTS
+
+__all__ = ["TTS"]
+
+
+def _resolve_api_key(api_key: str | None) -> str:
+ key = api_key or os.environ.get("XAI_API_KEY")
+ if not key:
+ raise ValueError(
+ "xAI TTS requires an api_key. Pass api_key='...' or "
+ "set XAI_API_KEY in the environment."
+ )
+ return key
+
+
+class TTS(_XaiTTS):
+ """xAI HTTP TTS (default voice ``eve``, ``language="auto"``) (Beta).
+
+ Example::
+
+ from getpatter.tts import xai
+
+ tts = xai.TTS() # reads XAI_API_KEY
+ tts = xai.TTS(api_key="...", voice="ara", language="en")
+
+ Telephony optimization
+ ----------------------
+ Use :meth:`for_twilio` or :meth:`for_telnyx` on phone calls. ``for_twilio``
+ emits mu-law @ 8 kHz natively (xAI supports G.711 directly), so the pipeline
+ skips resampling and PCM -> mu-law encoding entirely (bit-clean passthrough).
+ """
+
+ provider_key: ClassVar[str] = "xai_tts"
+
+ def __init__(
+ self,
+ api_key: str | None = None,
+ *,
+ voice: str = "eve",
+ language: str = "auto",
+ speed: Optional[float] = None,
+ optimize_streaming_latency: Optional[int] = None,
+ text_normalization: bool = False,
+ codec: Optional[str] = None,
+ sample_rate: Optional[int] = None,
+ bit_rate: Optional[int] = None,
+ ) -> None:
+ super().__init__(
+ api_key=_resolve_api_key(api_key),
+ voice=voice,
+ language=language,
+ speed=speed,
+ optimize_streaming_latency=optimize_streaming_latency,
+ text_normalization=text_normalization,
+ codec=codec,
+ sample_rate=sample_rate,
+ bit_rate=bit_rate,
+ )
+
+ @classmethod
+ def for_twilio(
+ cls,
+ api_key: str | None = None,
+ *,
+ voice: str = "eve",
+ language: str = "auto",
+ speed: Optional[float] = None,
+ ) -> "TTS":
+ """Pipeline TTS pre-configured for Twilio Media Streams.
+
+ Emits mu-law @ 8 kHz natively — Twilio's wire codec — so the pipeline
+ passes the bytes straight through (no resample, no PCM -> mu-law
+ encode). Falls back to ``XAI_API_KEY`` from the env when ``api_key`` is
+ omitted.
+ """
+ return cls(
+ api_key=_resolve_api_key(api_key),
+ voice=voice,
+ language=language,
+ speed=speed,
+ codec="mulaw",
+ sample_rate=8000,
+ )
+
+ @classmethod
+ def for_telnyx(
+ cls,
+ api_key: str | None = None,
+ *,
+ voice: str = "eve",
+ language: str = "auto",
+ speed: Optional[float] = None,
+ ) -> "TTS":
+ """Pipeline TTS pre-configured for Telnyx bidirectional media.
+
+ Emits ``pcm`` @ 16 kHz to match the Telnyx PCM16 pipeline. Falls back to
+ ``XAI_API_KEY`` from the env when ``api_key`` is omitted.
+ """
+ return cls(
+ api_key=_resolve_api_key(api_key),
+ voice=voice,
+ language=language,
+ speed=speed,
+ codec="pcm",
+ sample_rate=16000,
+ )
diff --git a/libraries/python/pyproject.toml b/libraries/python/pyproject.toml
index f6271981..c4339d95 100644
--- a/libraries/python/pyproject.toml
+++ b/libraries/python/pyproject.toml
@@ -54,6 +54,11 @@ local = []
soniox = [
"aiohttp>=3.10",
]
+# xAI voice providers (STT / TTS use pure-aiohttp REST + WS; realtime uses the
+# core ``websockets`` dep).
+xai = [
+ "aiohttp>=3.10",
+]
speechmatics = [
"speechmatics-voice[smart]>=0.2.8",
]
diff --git a/libraries/python/tests/unit/test_xai_realtime.py b/libraries/python/tests/unit/test_xai_realtime.py
new file mode 100644
index 00000000..5a4e7678
--- /dev/null
+++ b/libraries/python/tests/unit/test_xai_realtime.py
@@ -0,0 +1,179 @@
+"""Unit tests for the xAI Grok Voice Agent realtime engine.
+
+These assert the adapter's endpoint / provider key / defaults, the
+``session.update`` body produced by ``_build_ga_session_config`` (no WebSocket is
+opened, so no boundary is mocked), the engine-marker → provider unpack, and the
+three pricing entries. See ``.claude/rules/authentic-tests.md``.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from getpatter.client import Patter
+from getpatter.engines.xai import XaiRealtime
+from getpatter.pricing import (
+ calculate_realtime_cost,
+ calculate_stt_cost,
+ calculate_tts_cost,
+ merge_pricing,
+)
+from getpatter.providers.xai_realtime import XaiRealtimeAdapter
+
+
+def _adapter(**kwargs) -> XaiRealtimeAdapter:
+ return XaiRealtimeAdapter(api_key="xai-test-key", **kwargs)
+
+
+# ---------------------------------------------------------------------------
+# Endpoint / provider key / defaults
+# ---------------------------------------------------------------------------
+
+
+def test_endpoint_is_xai_and_provider_key_matches() -> None:
+ a = _adapter()
+ assert a.OPENAI_REALTIME_URL == "wss://api.x.ai/v1/realtime"
+ assert a.provider_key == "xai_realtime"
+
+
+def test_defaults_are_grok_voice_latest_and_eve() -> None:
+ a = _adapter()
+ assert a.model == "grok-voice-latest"
+ assert a.voice == "eve"
+ # xAI-valid transcription model (NOT OpenAI's whisper-1).
+ assert a.input_audio_transcription_model == "grok-transcribe"
+
+
+def test_explicit_model_and_voice_win_over_defaults() -> None:
+ a = _adapter(model="grok-voice-think-fast-1.0", voice="ara")
+ assert a.model == "grok-voice-think-fast-1.0"
+ assert a.voice == "ara"
+
+
+# ---------------------------------------------------------------------------
+# session.update grafting — keys PRESENT when set
+# ---------------------------------------------------------------------------
+
+
+def test_session_update_grafts_xai_keys_when_set() -> None:
+ a = _adapter(
+ reasoning_effort="none",
+ vad_threshold=0.6,
+ prefix_padding_ms=400,
+ idle_timeout_ms=8000,
+ language_hint="es-MX",
+ keyterms=("Grok", "xAI"),
+ speed=1.1,
+ replace={"Acme Mobile": "Acme Mobull"},
+ resumption=True,
+ server_tools=({"type": "web_search"},),
+ )
+ cfg = a._build_ga_session_config()
+ inp = cfg["audio"]["input"]
+ assert cfg["reasoning"] == {"effort": "none"}
+ assert inp["turn_detection"]["threshold"] == 0.6
+ assert inp["turn_detection"]["prefix_padding_ms"] == 400
+ assert inp["turn_detection"]["idle_timeout_ms"] == 8000
+ assert inp["transcription"]["language_hint"] == "es-MX"
+ assert inp["transcription"]["keyterms"] == ["Grok", "xAI"]
+ assert cfg["audio"]["output"]["speed"] == 1.1
+ assert cfg["replace"] == {"Acme Mobile": "Acme Mobull"}
+ assert cfg["resumption"] == {"enabled": True}
+ assert {"type": "web_search"} in cfg["tools"]
+ # Audio path is inherited from the GA adapter (pcm24 both directions).
+ assert inp["format"] == {"type": "audio/pcm", "rate": 24000}
+
+
+# ---------------------------------------------------------------------------
+# session.update grafting — keys OMITTED when unset
+# ---------------------------------------------------------------------------
+
+
+def test_session_update_omits_xai_keys_when_unset() -> None:
+ cfg = _adapter()._build_ga_session_config()
+ inp = cfg["audio"]["input"]
+ assert "reasoning" not in cfg
+ assert "replace" not in cfg
+ assert "resumption" not in cfg
+ assert "speed" not in cfg["audio"]["output"]
+ assert "idle_timeout_ms" not in inp["turn_detection"]
+ assert "language_hint" not in inp["transcription"]
+ assert "keyterms" not in inp["transcription"]
+ # The OpenAI-only ``language`` key never leaks into the xAI session.
+ assert "language" not in inp["transcription"]
+
+
+def test_server_tools_append_to_agent_function_tools() -> None:
+ # Function tools (from the base builder) + xAI server tools coexist.
+ a = _adapter(
+ tools=[
+ {
+ "name": "lookup",
+ "description": "look up",
+ "parameters": {"type": "object", "properties": {}},
+ }
+ ],
+ server_tools=({"type": "web_search"}, {"type": "x_search"}),
+ )
+ tools = a._build_ga_session_config()["tools"]
+ types = [t.get("type") for t in tools]
+ assert "function" in types
+ assert "web_search" in types
+ assert "x_search" in types
+
+
+# ---------------------------------------------------------------------------
+# Engine-marker unpack in the client
+# ---------------------------------------------------------------------------
+
+
+def test_engine_marker_unpacks_to_xai_realtime() -> None:
+ marker = XaiRealtime(api_key="xai-test-key", reasoning_effort="none", speed=1.2)
+ kind, fields = Patter._unpack_engine(marker)
+ assert kind == "xai_realtime"
+ assert fields["voice"] == "eve"
+ assert fields["model"] == "grok-voice-latest"
+ assert fields["reasoning_effort"] == "none"
+ assert fields["speed"] == 1.2
+
+
+def test_engine_marker_requires_api_key(monkeypatch) -> None:
+ monkeypatch.delenv("XAI_API_KEY", raising=False)
+ with pytest.raises(ValueError, match="api_key"):
+ XaiRealtime()
+
+
+# ---------------------------------------------------------------------------
+# Pricing math
+# ---------------------------------------------------------------------------
+
+
+def test_pricing_entries_for_all_three_providers() -> None:
+ pricing = merge_pricing(None)
+ # STT streaming: $0.20/hr → $0.20/60 per minute.
+ assert calculate_stt_cost("xai", 60.0, pricing) == pytest.approx(0.20 / 60)
+ # TTS: $15/1M chars = $0.015/1k chars.
+ assert calculate_tts_cost("xai_tts", 1000, pricing) == pytest.approx(0.015)
+ # Realtime: $0.05/min via the MINUTE duration path.
+ assert calculate_realtime_cost(
+ {}, pricing, provider="xai_realtime", duration_seconds=120.0
+ ) == pytest.approx(0.10)
+
+
+def test_realtime_minute_path_zero_without_duration() -> None:
+ pricing = merge_pricing(None)
+ assert calculate_realtime_cost({}, pricing, provider="xai_realtime") == 0.0
+ # Documented (not metered) per-message constant is on the entry.
+ assert pricing["xai_realtime"]["text_input_per_message"] == 0.004
+
+
+def test_openai_realtime_token_path_unchanged() -> None:
+ # Backward-compat: the default token path for existing callers is untouched.
+ pricing = merge_pricing(None)
+ usage = {
+ "input_token_details": {"audio_tokens": 1000, "text_tokens": 0},
+ "output_token_details": {"audio_tokens": 500, "text_tokens": 0},
+ }
+ cost = calculate_realtime_cost(usage, pricing, model="gpt-realtime-mini")
+ # 1000 * 1e-5 + 500 * 2e-5 = 0.02
+ assert cost == pytest.approx(0.02)
diff --git a/libraries/python/tests/unit/test_xai_stt.py b/libraries/python/tests/unit/test_xai_stt.py
new file mode 100644
index 00000000..aec8a9fe
--- /dev/null
+++ b/libraries/python/tests/unit/test_xai_stt.py
@@ -0,0 +1,390 @@
+"""Unit tests for the xAI STT provider (streaming adapter + batch helper).
+
+Only the xAI network boundary is mocked; URL/query construction, the
+``transcript.created`` handshake, event mapping, ``finalize()``, and multipart
+field ordering all run against real code. See ``.claude/rules/authentic-tests.md``.
+"""
+
+from __future__ import annotations
+
+import json
+from typing import Any
+from unittest.mock import patch
+from urllib.parse import parse_qsl, urlsplit
+
+import aiohttp
+import pytest
+
+from getpatter.providers.xai_stt import (
+ XAI_STT_REST_URL,
+ XaiSTT,
+ XaiTranscription,
+ _build_transcribe_fields,
+ transcribe,
+)
+
+# ---------------------------------------------------------------------------
+# aiohttp WebSocket fakes — only the network boundary is mocked.
+# ---------------------------------------------------------------------------
+
+
+class _FakeMsg:
+ """Minimal stand-in for an ``aiohttp.WSMessage`` (type + data)."""
+
+ def __init__(self, type_: Any, data: str) -> None:
+ self.type = type_
+ self.data = data
+
+
+def _text(payload: dict) -> _FakeMsg:
+ return _FakeMsg(aiohttp.WSMsgType.TEXT, json.dumps(payload))
+
+
+class _FakeWebSocket:
+ """Fake aiohttp ws: records sends, replays a scripted incoming stream."""
+
+ def __init__(self, incoming: list[_FakeMsg]) -> None:
+ self.closed = False
+ self.sent_text: list[str] = []
+ self.sent_bytes: list[bytes] = []
+ self._incoming = list(incoming)
+
+ async def send_str(self, payload: str) -> None:
+ self.sent_text.append(payload)
+
+ async def send_bytes(self, payload: bytes) -> None:
+ self.sent_bytes.append(payload)
+
+ async def close(self) -> None:
+ self.closed = True
+
+ def exception(self) -> None: # pragma: no cover - only on ERROR frames
+ return None
+
+ def __aiter__(self) -> "_FakeWebSocket":
+ return self
+
+ async def __anext__(self) -> _FakeMsg:
+ if not self._incoming:
+ raise StopAsyncIteration
+ return self._incoming.pop(0)
+
+
+class _FakeSession:
+ """Stand-in for ``aiohttp.ClientSession`` returning a fixed fake ws."""
+
+ def __init__(self, ws: _FakeWebSocket) -> None:
+ self._ws = ws
+ self.closed = False
+ self.last_url: str | None = None
+ self.last_headers: dict[str, str] | None = None
+
+ async def ws_connect(
+ self, url: str, *, headers: dict[str, str] | None = None, **_kw: Any
+ ) -> _FakeWebSocket:
+ self.last_url = url
+ self.last_headers = headers
+ return self._ws
+
+ async def close(self) -> None:
+ self.closed = True
+
+
+async def _connect(stt: XaiSTT, ws: _FakeWebSocket) -> _FakeSession:
+ """Connect ``stt`` against a mocked ClientSession and return the session."""
+ fake_session = _FakeSession(ws)
+ with patch(
+ "getpatter.providers.xai_stt.aiohttp.ClientSession",
+ return_value=fake_session,
+ ):
+ await stt.connect()
+ return fake_session
+
+
+# ---------------------------------------------------------------------------
+# URL / query construction
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.mocked
+class TestQueryConstruction:
+ async def test_default_query_is_pcm_16k_with_interim(self) -> None:
+ ws = _FakeWebSocket([_text({"type": "transcript.created"})])
+ session = await _connect(XaiSTT(api_key="xai-test-key"), ws)
+ assert session.last_url is not None
+ assert session.last_headers == {"Authorization": "Bearer xai-test-key"}
+ params = dict(parse_qsl(urlsplit(session.last_url).query))
+ assert params["encoding"] == "pcm"
+ assert params["sample_rate"] == "16000"
+ assert params["interim_results"] == "true"
+
+ async def test_keyterms_are_repeated_query_params(self) -> None:
+ ws = _FakeWebSocket([_text({"type": "transcript.created"})])
+ stt = XaiSTT(
+ api_key="xai-test-key",
+ language="en",
+ keyterms=("Grok", "xAI"),
+ smart_turn=0.7,
+ smart_turn_timeout_ms=2000,
+ diarize=True,
+ )
+ session = await _connect(stt, ws)
+ pairs = parse_qsl(urlsplit(session.last_url).query)
+ keyterms = [v for k, v in pairs if k == "keyterm"]
+ assert keyterms == ["Grok", "xAI"]
+ flat = dict(pairs)
+ assert flat["language"] == "en"
+ assert flat["smart_turn"] == "0.7"
+ assert flat["smart_turn_timeout"] == "2000"
+ assert flat["diarize"] == "true"
+
+
+# ---------------------------------------------------------------------------
+# Handshake ordering
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.mocked
+class TestHandshake:
+ async def test_connect_waits_for_transcript_created(self) -> None:
+ ws = _FakeWebSocket([_text({"type": "transcript.created"})])
+ stt = XaiSTT(api_key="xai-test-key")
+ await _connect(stt, ws)
+ # No setup message is sent — config is URL-only.
+ assert ws.sent_text == []
+
+ async def test_send_audio_before_connect_raises(self) -> None:
+ stt = XaiSTT(api_key="xai-test-key")
+ with pytest.raises(RuntimeError):
+ await stt.send_audio(b"\x00\x00")
+
+ async def test_close_before_created_raises(self) -> None:
+ ws = _FakeWebSocket([]) # stream ends before the handshake
+ stt = XaiSTT(api_key="xai-test-key")
+ with pytest.raises(RuntimeError):
+ await _connect(stt, ws)
+
+ async def test_error_during_handshake_raises(self) -> None:
+ ws = _FakeWebSocket([_text({"type": "error", "message": "bad key"})])
+ stt = XaiSTT(api_key="xai-test-key")
+ with pytest.raises(RuntimeError, match="bad key"):
+ await _connect(stt, ws)
+
+
+# ---------------------------------------------------------------------------
+# Event mapping (is_final / speech_final) + finalize
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.mocked
+class TestTranscriptMapping:
+ async def test_maps_interim_chunk_final_and_utterance_end(self) -> None:
+ ws = _FakeWebSocket(
+ [
+ _text({"type": "transcript.created"}),
+ _text({"type": "transcript.partial", "text": "hel", "is_final": False}),
+ _text(
+ {
+ "type": "transcript.partial",
+ "text": "hello",
+ "is_final": True,
+ "speech_final": False,
+ }
+ ),
+ _text(
+ {
+ "type": "transcript.partial",
+ "text": "hello world",
+ "is_final": True,
+ "speech_final": True,
+ }
+ ),
+ ]
+ )
+ stt = XaiSTT(api_key="xai-test-key")
+ await _connect(stt, ws)
+ results = [t async for t in stt.receive_transcripts()]
+ assert [(t.text, t.is_final, t.speech_final) for t in results] == [
+ ("hel", False, False),
+ ("hello", True, False),
+ ("hello world", True, True),
+ ]
+
+ async def test_transcript_done_flushes_a_final(self) -> None:
+ ws = _FakeWebSocket(
+ [
+ _text({"type": "transcript.created"}),
+ _text({"type": "transcript.done", "text": "final text"}),
+ ]
+ )
+ stt = XaiSTT(api_key="xai-test-key")
+ await _connect(stt, ws)
+ results = [t async for t in stt.receive_transcripts()]
+ assert len(results) == 1
+ assert results[0].text == "final text"
+ assert results[0].is_final is True
+ assert results[0].speech_final is True
+
+ async def test_error_event_does_not_close_stream(self) -> None:
+ ws = _FakeWebSocket(
+ [
+ _text({"type": "transcript.created"}),
+ _text({"type": "error", "message": "transient"}),
+ _text(
+ {
+ "type": "transcript.partial",
+ "text": "after error",
+ "is_final": True,
+ "speech_final": True,
+ }
+ ),
+ ]
+ )
+ stt = XaiSTT(api_key="xai-test-key")
+ await _connect(stt, ws)
+ results = [t async for t in stt.receive_transcripts()]
+ assert [t.text for t in results] == ["after error"]
+
+ async def test_finalize_sends_finalize_frame(self) -> None:
+ ws = _FakeWebSocket([_text({"type": "transcript.created"})])
+ stt = XaiSTT(api_key="xai-test-key")
+ await _connect(stt, ws)
+ await stt.finalize()
+ assert json.loads(ws.sent_text[-1]) == {"type": "finalize"}
+
+ async def test_close_sends_audio_done(self) -> None:
+ ws = _FakeWebSocket([_text({"type": "transcript.created"})])
+ stt = XaiSTT(api_key="xai-test-key")
+ await _connect(stt, ws)
+ await stt.close()
+ assert json.loads(ws.sent_text[-1]) == {"type": "audio.done"}
+ assert ws.closed is True
+
+
+# ---------------------------------------------------------------------------
+# Batch transcription helper
+# ---------------------------------------------------------------------------
+
+
+class TestBatchFieldOrdering:
+ @pytest.mark.unit
+ def test_file_is_the_last_multipart_field(self) -> None:
+ fields = _build_transcribe_fields(
+ audio=b"RIFFDATA",
+ url=None,
+ language="en",
+ format=True,
+ diarize=True,
+ keyterms=("Grok", "xAI"),
+ filler_words=True,
+ audio_format="pcm",
+ sample_rate=16000,
+ filename="clip.wav",
+ )
+ names = [name for name, _v, _extra in fields]
+ assert names[-1] == "file"
+ assert names.index("file") == len(names) - 1
+ # All scalar fields precede the file; keyterm repeats once per term.
+ assert names[:-1] == [
+ "language",
+ "format",
+ "diarize",
+ "filler_words",
+ "audio_format",
+ "sample_rate",
+ "keyterm",
+ "keyterm",
+ ]
+
+ @pytest.mark.unit
+ def test_url_used_instead_of_file_when_given(self) -> None:
+ fields = _build_transcribe_fields(
+ audio=None,
+ url="https://example.com/audio.wav",
+ language=None,
+ format=False,
+ diarize=False,
+ keyterms=(),
+ filler_words=False,
+ audio_format=None,
+ sample_rate=None,
+ filename="audio.wav",
+ )
+ assert [name for name, _v, _e in fields] == ["url"]
+
+
+# --- REST response parsing (network boundary mocked) ---
+
+
+class _FakeRestResponse:
+ def __init__(self, status: int, payload: dict) -> None:
+ self.status = status
+ self._payload = payload
+
+ async def json(self) -> dict:
+ return self._payload
+
+ async def text(self) -> str: # pragma: no cover - only on non-200
+ return json.dumps(self._payload)
+
+ async def __aenter__(self) -> "_FakeRestResponse":
+ return self
+
+ async def __aexit__(self, *_exc: Any) -> None:
+ return None
+
+
+class _FakeRestSession:
+ def __init__(self, response: _FakeRestResponse) -> None:
+ self.response = response
+ self.last_url: str | None = None
+ self.last_headers: dict[str, str] | None = None
+ self.last_data: Any = None
+
+ def post(
+ self, url: str, *, headers: dict[str, str], data: Any, timeout: Any = None
+ ) -> _FakeRestResponse:
+ self.last_url = url
+ self.last_headers = headers
+ self.last_data = data
+ return self.response
+
+ async def __aenter__(self) -> "_FakeRestSession":
+ return self
+
+ async def __aexit__(self, *_exc: Any) -> None:
+ return None
+
+
+@pytest.mark.mocked
+class TestBatchTranscribe:
+ async def test_parses_response_into_dataclass(self) -> None:
+ payload = {
+ "text": "The balance is $167,983.15.",
+ "language": "English",
+ "duration": 3.45,
+ "words": [
+ {"text": "The", "start": 0.24, "end": 0.48},
+ {"text": "balance", "start": 0.48, "end": 0.9, "speaker": 1},
+ ],
+ }
+ fake = _FakeRestSession(_FakeRestResponse(200, payload))
+ with patch(
+ "getpatter.providers.xai_stt.aiohttp.ClientSession", return_value=fake
+ ):
+ result = await transcribe(b"RIFFDATA", api_key="xai-test-key", diarize=True)
+ assert isinstance(result, XaiTranscription)
+ assert result.text == "The balance is $167,983.15."
+ assert result.language == "English"
+ assert result.duration == 3.45
+ assert len(result.words) == 2
+ assert result.words[0].text == "The"
+ assert result.words[1].speaker == 1
+ assert fake.last_url == XAI_STT_REST_URL
+ assert fake.last_headers == {"Authorization": "Bearer xai-test-key"}
+
+ async def test_requires_exactly_one_of_audio_or_url(self) -> None:
+ with pytest.raises(ValueError):
+ await transcribe(api_key="xai-test-key")
+ with pytest.raises(ValueError):
+ await transcribe(b"x", url="https://x/y.wav", api_key="xai-test-key")
diff --git a/libraries/python/tests/unit/test_xai_tts.py b/libraries/python/tests/unit/test_xai_tts.py
new file mode 100644
index 00000000..7d510fc4
--- /dev/null
+++ b/libraries/python/tests/unit/test_xai_tts.py
@@ -0,0 +1,262 @@
+"""Unit tests for the xAI TTS provider + custom-voice helper.
+
+Mock the aiohttp boundary; payload assembly, Bearer auth, telephony factories,
+source-format declaration, byte streaming, and multipart ordering run against
+real code. See ``.claude/rules/authentic-tests.md``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+from unittest.mock import patch
+
+import pytest
+
+from getpatter.audio.format import AudioFormat
+from getpatter.providers.xai_tts import (
+ XAI_CUSTOM_VOICES_URL,
+ XAI_TTS_REST_URL,
+ XaiTTS,
+ create_custom_voice,
+)
+
+# ---------------------------------------------------------------------------
+# aiohttp fakes — only the network boundary is mocked.
+# ---------------------------------------------------------------------------
+
+
+class _AsyncChunks:
+ def __init__(self, chunks: list[bytes]) -> None:
+ self._chunks = list(chunks)
+
+ def __aiter__(self) -> "_AsyncChunks":
+ return self
+
+ async def __anext__(self) -> bytes:
+ if not self._chunks:
+ raise StopAsyncIteration
+ return self._chunks.pop(0)
+
+
+class _FakeContent:
+ def __init__(self, chunks: list[bytes]) -> None:
+ self._chunks = list(chunks)
+
+ def iter_chunked(self, _size: int) -> _AsyncChunks:
+ return _AsyncChunks(self._chunks)
+
+
+class _FakeResponse:
+ def __init__(self, status: int, chunks: list[bytes], body: str = "") -> None:
+ self.status = status
+ self.content = _FakeContent(chunks)
+ self._body = body
+
+ async def text(self) -> str:
+ return self._body
+
+ async def json(self) -> dict:
+ import json
+
+ return json.loads(self._body)
+
+ async def __aenter__(self) -> "_FakeResponse":
+ return self
+
+ async def __aexit__(self, *_exc: Any) -> None:
+ return None
+
+
+class _FakeSession:
+ def __init__(self, response: _FakeResponse) -> None:
+ self.response = response
+ self.last_url: str | None = None
+ self.last_json: dict[str, Any] | None = None
+ self.last_headers: dict[str, str] | None = None
+ self.last_data: Any = None
+ self.closed = False
+
+ def post(
+ self,
+ url: str,
+ *,
+ headers: dict[str, str],
+ json: dict[str, Any] | None = None, # noqa: A002 - aiohttp signature
+ data: Any = None,
+ timeout: Any = None,
+ ) -> _FakeResponse:
+ self.last_url = url
+ self.last_headers = headers
+ self.last_json = json
+ self.last_data = data
+ return self.response
+
+ async def close(self) -> None:
+ self.closed = True
+
+ async def __aenter__(self) -> "_FakeSession":
+ return self
+
+ async def __aexit__(self, *_exc: Any) -> None:
+ return None
+
+
+# ---------------------------------------------------------------------------
+# Request shape + auth + streaming
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.mocked
+class TestPayloadAndAuth:
+ async def test_posts_to_rest_endpoint_with_bearer_auth(self) -> None:
+ fake = _FakeSession(_FakeResponse(200, [b"hello", b"world"]))
+ tts = XaiTTS(api_key="xai-test-key", session=fake) # type: ignore[arg-type]
+
+ out = b"".join([c async for c in tts.synthesize("ciao")])
+
+ assert out == b"helloworld"
+ assert fake.last_url == XAI_TTS_REST_URL
+ assert fake.last_headers is not None
+ assert fake.last_headers["Authorization"] == "Bearer xai-test-key"
+ assert fake.last_headers["Content-Type"] == "application/json"
+
+ async def test_default_payload_is_eve_auto_pcm_16k(self) -> None:
+ fake = _FakeSession(_FakeResponse(200, []))
+ tts = XaiTTS(api_key="xai-test-key", session=fake) # type: ignore[arg-type]
+ async for _ in tts.synthesize("hi"):
+ pass
+ body = fake.last_json
+ assert body is not None
+ assert body["voice_id"] == "eve"
+ assert body["language"] == "auto"
+ assert body["output_format"] == {"codec": "pcm", "sample_rate": 16000}
+ assert "speed" not in body
+ assert "optimize_streaming_latency" not in body
+ assert "text_normalization" not in body
+
+ async def test_optional_fields_only_added_when_set(self) -> None:
+ fake = _FakeSession(_FakeResponse(200, []))
+ tts = XaiTTS(
+ api_key="xai-test-key",
+ session=fake, # type: ignore[arg-type]
+ voice="ara",
+ language="en",
+ speed=1.2,
+ optimize_streaming_latency=1,
+ text_normalization=True,
+ )
+ async for _ in tts.synthesize("ciao"):
+ pass
+ body = fake.last_json
+ assert body["voice_id"] == "ara"
+ assert body["speed"] == 1.2
+ assert body["optimize_streaming_latency"] == 1
+ assert body["text_normalization"] is True
+
+ async def test_bit_rate_nests_under_output_format_when_set(self) -> None:
+ fake = _FakeSession(_FakeResponse(200, []))
+ tts = XaiTTS(
+ api_key="xai-test-key",
+ session=fake, # type: ignore[arg-type]
+ codec="mp3",
+ sample_rate=44100,
+ bit_rate=192000,
+ )
+ async for _ in tts.synthesize("ciao"):
+ pass
+ body = fake.last_json
+ assert body is not None
+ assert body["output_format"] == {
+ "codec": "mp3",
+ "sample_rate": 44100,
+ "bit_rate": 192000,
+ }
+
+ async def test_non_200_raises(self) -> None:
+ fake = _FakeSession(_FakeResponse(400, [], body="bad request"))
+ tts = XaiTTS(api_key="xai-test-key", session=fake) # type: ignore[arg-type]
+ with pytest.raises(RuntimeError, match="400"):
+ async for _ in tts.synthesize("x"):
+ pass
+
+
+# ---------------------------------------------------------------------------
+# Telephony factories + source format
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.unit
+class TestTelephonyFactories:
+ def test_for_twilio_emits_mulaw_8k_source_format(self) -> None:
+ tts = XaiTTS.for_twilio(api_key="xai-test-key")
+ assert tts.codec == "mulaw"
+ assert tts.sample_rate == 8000
+ assert tts.source_audio_format() == AudioFormat(
+ encoding="mulaw", sample_rate=8000
+ )
+
+ def test_for_telnyx_emits_pcm_16k(self) -> None:
+ tts = XaiTTS.for_telnyx(api_key="xai-test-key")
+ assert tts.codec == "pcm"
+ assert tts.sample_rate == 16000
+ assert tts.source_audio_format() == AudioFormat(
+ encoding="pcm_s16le", sample_rate=16000
+ )
+
+ def test_default_source_format_is_pcm_16k(self) -> None:
+ tts = XaiTTS(api_key="xai-test-key")
+ assert tts.source_audio_format() == AudioFormat(
+ encoding="pcm_s16le", sample_rate=16000
+ )
+
+
+# ---------------------------------------------------------------------------
+# Custom voice creation
+# ---------------------------------------------------------------------------
+
+
+class _FakeCustomVoiceSession:
+ def __init__(self, response: _FakeResponse) -> None:
+ self.response = response
+ self.last_url: str | None = None
+ self.last_headers: dict[str, str] | None = None
+ self.last_data: Any = None
+
+ def post(
+ self, url: str, *, headers: dict[str, str], data: Any, timeout: Any = None
+ ) -> _FakeResponse:
+ self.last_url = url
+ self.last_headers = headers
+ self.last_data = data
+ return self.response
+
+ async def __aenter__(self) -> "_FakeCustomVoiceSession":
+ return self
+
+ async def __aexit__(self, *_exc: Any) -> None:
+ return None
+
+
+@pytest.mark.mocked
+class TestCustomVoice:
+ async def test_create_custom_voice_returns_voice_id(self) -> None:
+ import json
+
+ fake = _FakeCustomVoiceSession(
+ _FakeResponse(200, [], body=json.dumps({"voice_id": "vc_abc123"}))
+ )
+ with patch(
+ "getpatter.providers.xai_tts.aiohttp.ClientSession", return_value=fake
+ ):
+ voice_id = await create_custom_voice(
+ name="Receptionist",
+ language="en",
+ audio=b"RIFFwavdata",
+ api_key="xai-test-key",
+ )
+ assert voice_id == "vc_abc123"
+ assert fake.last_url == XAI_CUSTOM_VOICES_URL
+ assert fake.last_headers == {"Authorization": "Bearer xai-test-key"}
+ # ``file`` must be the LAST multipart field (name, language, then file).
+ names = [f[0]["name"] for f in fake.last_data._fields]
+ assert names == ["name", "language", "file"]
diff --git a/libraries/typescript/src/client.ts b/libraries/typescript/src/client.ts
index bab82879..251ae914 100644
--- a/libraries/typescript/src/client.ts
+++ b/libraries/typescript/src/client.ts
@@ -47,6 +47,7 @@ import { ConvAI as ElevenLabsConvAI } from "./engines/elevenlabs";
import { GeminiLive } from "./engines/gemini";
import { GeminiCascade } from "./engines/gemini-cascade";
import { InworldRealtime } from "./engines/inworld";
+import { XaiRealtime } from "./engines/xai";
import { CloudflareTunnel, Static as StaticTunnel } from "./tunnels";
import { resolveLogRoot } from "./services/call-log";
import { validateAllToolSchemas } from "./tools/schema-validation";
@@ -778,10 +779,23 @@ export class Patter {
working.openaiRealtimeGateResponseOnTranscript ??
engine.gateResponseOnTranscript,
};
+ } else if (engine instanceof XaiRealtime) {
+ working = {
+ ...working,
+ provider: 'xai_realtime',
+ // Explicit agent() kwargs win over the engine marker value.
+ model: working.model ?? engine.model,
+ voice: working.voice ?? engine.voice,
+ realtimeTurnDetection:
+ working.realtimeTurnDetection ?? engine.turnDetection,
+ openaiRealtimeGateResponseOnTranscript:
+ working.openaiRealtimeGateResponseOnTranscript ??
+ engine.gateResponseOnTranscript,
+ };
} else {
this.recordConfigIncomplete('engine_config');
throw new Error(
- "Unknown engine. Expected OpenAIRealtime, OpenAIRealtime2, ElevenLabsConvAI, GeminiLive, GeminiCascade, or InworldRealtime instance.",
+ "Unknown engine. Expected OpenAIRealtime, OpenAIRealtime2, ElevenLabsConvAI, GeminiLive, GeminiCascade, InworldRealtime, or XaiRealtime instance.",
);
}
} else if (
@@ -797,7 +811,7 @@ export class Patter {
// Validate provider
if (working.provider) {
- const valid = ['openai_realtime', 'elevenlabs_convai', 'gemini_live', 'gemini_cascade', 'inworld_realtime', 'pipeline'];
+ const valid = ['openai_realtime', 'elevenlabs_convai', 'gemini_live', 'gemini_cascade', 'inworld_realtime', 'xai_realtime', 'pipeline'];
if (!valid.includes(working.provider)) {
this.recordConfigIncomplete('engine_config');
throw new Error(`provider must be one of: ${valid.join(', ')}. Got: '${working.provider}'`);
@@ -1063,7 +1077,7 @@ export class Patter {
}
// Validate provider
- const validProviders = ['openai_realtime', 'elevenlabs_convai', 'gemini_live', 'gemini_cascade', 'inworld_realtime', 'pipeline'] as const;
+ const validProviders = ['openai_realtime', 'elevenlabs_convai', 'gemini_live', 'gemini_cascade', 'inworld_realtime', 'xai_realtime', 'pipeline'] as const;
if (opts.agent.provider && !validProviders.includes(opts.agent.provider as typeof validProviders[number])) {
throw new Error(`agent.provider must be one of: ${validProviders.join(', ')}`);
}
diff --git a/libraries/typescript/src/engines/xai.ts b/libraries/typescript/src/engines/xai.ts
new file mode 100644
index 00000000..028b12b9
--- /dev/null
+++ b/libraries/typescript/src/engines/xai.ts
@@ -0,0 +1,171 @@
+/**
+ * xAI Grok Voice Agent engine — marker class for Patter client dispatch.
+ *
+ * **Beta: validated against the xAI Voice Agent API spec; not yet exercised
+ * against a live phone call.**
+ *
+ * Selects xAI's Voice Agent (Grok Voice) speech-to-speech API. xAI advertises
+ * an "Migrating from OpenAI Realtime" path — the GA event schema, session
+ * structure, and client/server events are compatible with OpenAI's Realtime GA
+ * API, so the session is driven with the same `session.update` /
+ * `response.create` / streaming-delta wire shape. The runtime lives in
+ * {@link import('../providers/xai-realtime').XaiRealtimeAdapter}, which
+ * subclasses `OpenAIRealtime2Adapter` and swaps the endpoint + grafts the
+ * xAI-specific session keys.
+ *
+ * Like the other engine markers this is a tiny immutable config object: it
+ * carries credentials / voice / model plus the xAI extensions. The session is
+ * constructed server-side by `buildAIAdapter`.
+ *
+ * @example
+ * ```ts
+ * import { Patter, Twilio, XaiRealtime } from "getpatter";
+ *
+ * const phone = new Patter({ carrier: new Twilio(), phoneNumber: "+1..." });
+ * const agent = phone.agent({
+ * engine: new XaiRealtime({ voice: "eve" }), // reads XAI_API_KEY
+ * systemPrompt: "You are a friendly receptionist.",
+ * firstMessage: "Hello! How can I help?",
+ * });
+ * ```
+ */
+
+import type { RealtimeTurnDetection } from '../types';
+import { validateRealtimeTurnDetection } from '../providers/openai-realtime';
+
+/** Constructor options for the `XaiRealtime` engine marker. */
+export interface XaiRealtimeOptions {
+ /**
+ * xAI API key (Bearer). Falls back to the `XAI_API_KEY` env var when omitted.
+ */
+ apiKey?: string;
+ /**
+ * Voice-agent model id. Defaults to `"grok-voice-latest"` (always the newest
+ * model). Also accepts pinned ids such as `"grok-voice-think-fast-1.0"`.
+ */
+ model?: string;
+ /** Voice id (built-in or custom, case-insensitive). Defaults to `"eve"`. */
+ voice?: string;
+ /** Override the WebSocket base URL (no query string). Defaults to `wss://api.x.ai/v1/realtime`. */
+ baseUrl?: string;
+ /**
+ * Reasoning effort. `"high"` (xAI server default) enables reasoning;
+ * `"none"` disables it. Omit to keep the server default.
+ */
+ reasoningEffort?: 'high' | 'none';
+ /**
+ * ISO / BCP-47 language hint for input transcription (e.g. `"it"`, `"es-MX"`).
+ * Biases ASR toward one language instead of auto-detecting. Omit for
+ * auto-detect. Display-only — does not gate the model.
+ */
+ languageHint?: string;
+ /**
+ * Up to 100 key terms (≤50 chars each) biasing transcription toward
+ * domain vocabulary (product names, proper nouns). Omit for none.
+ */
+ keyterms?: readonly string[];
+ /** Assistant playback speed multiplier in `[0.7, 1.5]` (default `1.0`). */
+ speed?: number;
+ /**
+ * Server VAD activation threshold in `[0.1, 0.9]` (xAI default `0.85`).
+ * Higher requires louder audio to trigger a turn.
+ */
+ vadThreshold?: number;
+ /**
+ * Trailing silence (ms) the server VAD waits for before ending the turn,
+ * `[0, 10000]`. Maps to `turn_detection.silence_duration_ms`.
+ */
+ silenceDurationMs?: number;
+ /**
+ * Audio (ms) included before detected speech start, `[0, 10000]` (xAI
+ * default `333`). Maps to `turn_detection.prefix_padding_ms`.
+ */
+ prefixPaddingMs?: number;
+ /**
+ * When set, the server re-engages the user after this many ms of silence
+ * following an assistant turn. Maps to `turn_detection.idle_timeout_ms`.
+ */
+ idleTimeoutMs?: number;
+ /**
+ * Pronunciation map applied to the model's output before TTS, e.g.
+ * `{ "Acme Mobile": "Acme Mobull" }`. Only the spoken audio changes.
+ */
+ replace?: Readonly>;
+ /**
+ * Opt in to session resumption — the server caches conversation turns and
+ * replays them on reconnect. Defaults to `false`.
+ */
+ resumption?: boolean;
+ /**
+ * Raw xAI server-side tool objects appended verbatim to `session.tools`
+ * (e.g. `{ type: 'web_search' }`, `{ type: 'x_search' }`,
+ * `{ type: 'mcp', server_url, server_label }`). Executed server-side at xAI.
+ */
+ serverTools?: ReadonlyArray>;
+ /**
+ * Turn-detection tuning (shared shape with the other realtime engines).
+ * `undefined` (default) keeps the server VAD defaults. Raise the threshold
+ * or switch to `semantic_vad` eagerness `'low'` to stop speakerphone noise
+ * from triggering false barge-ins.
+ */
+ turnDetection?: RealtimeTurnDetection;
+ /**
+ * Gate the model's response on the input transcript (legacy behavior).
+ * `false` (default) — the model responds on `speech_stopped`, independent
+ * of the transcript. `true` — restore transcript-gated responses.
+ */
+ gateResponseOnTranscript?: boolean;
+}
+
+/**
+ * xAI Grok Voice Agent engine marker — selects xAI's OpenAI-GA-compatible
+ * speech-to-speech API.
+ */
+export class XaiRealtime {
+ readonly kind = 'xai_realtime' as const;
+ readonly apiKey: string;
+ readonly model?: string;
+ readonly voice?: string;
+ readonly baseUrl?: string;
+ readonly reasoningEffort?: 'high' | 'none';
+ readonly languageHint?: string;
+ readonly keyterms?: readonly string[];
+ readonly speed?: number;
+ readonly vadThreshold?: number;
+ readonly silenceDurationMs?: number;
+ readonly prefixPaddingMs?: number;
+ readonly idleTimeoutMs?: number;
+ readonly replace?: Readonly>;
+ readonly resumption?: boolean;
+ readonly serverTools?: ReadonlyArray>;
+ readonly turnDetection?: RealtimeTurnDetection;
+ readonly gateResponseOnTranscript?: boolean;
+
+ constructor(opts: XaiRealtimeOptions = {}) {
+ const key = opts.apiKey ?? process.env.XAI_API_KEY;
+ if (!key) {
+ throw new Error(
+ "xAI Realtime requires an apiKey. Pass { apiKey: '...' } or set " +
+ 'XAI_API_KEY in the environment.',
+ );
+ }
+ validateRealtimeTurnDetection(opts.turnDetection);
+ this.apiKey = key;
+ this.model = opts.model;
+ this.voice = opts.voice;
+ this.baseUrl = opts.baseUrl;
+ this.reasoningEffort = opts.reasoningEffort;
+ this.languageHint = opts.languageHint;
+ this.keyterms = opts.keyterms;
+ this.speed = opts.speed;
+ this.vadThreshold = opts.vadThreshold;
+ this.silenceDurationMs = opts.silenceDurationMs;
+ this.prefixPaddingMs = opts.prefixPaddingMs;
+ this.idleTimeoutMs = opts.idleTimeoutMs;
+ this.replace = opts.replace;
+ this.resumption = opts.resumption;
+ this.serverTools = opts.serverTools;
+ this.turnDetection = opts.turnDetection;
+ this.gateResponseOnTranscript = opts.gateResponseOnTranscript;
+ }
+}
diff --git a/libraries/typescript/src/index.ts b/libraries/typescript/src/index.ts
index 4e23c6e2..9477149c 100644
--- a/libraries/typescript/src/index.ts
+++ b/libraries/typescript/src/index.ts
@@ -136,6 +136,23 @@ export {
INWORLD_REALTIME_DEFAULT_VOICE,
} from "./providers/inworld-realtime";
export type { InworldRealtimeAdapterOptions } from "./providers/inworld-realtime";
+export {
+ XaiRealtimeAdapter,
+ XAI_REALTIME_WS_URL,
+ XAI_REALTIME_DEFAULT_MODEL,
+ XAI_REALTIME_DEFAULT_VOICE,
+} from "./providers/xai-realtime";
+export type { XaiRealtimeAdapterOptions } from "./providers/xai-realtime";
+// xAI STT batch helper + streaming encoding enum + result shapes.
+export { xaiTranscribe, XaiSTTEncoding } from "./providers/xai-stt";
+export type {
+ XaiTranscription,
+ XaiWord,
+ XaiTranscribeOptions,
+} from "./providers/xai-stt";
+// xAI TTS custom-voice helper + codec enum.
+export { xaiCreateCustomVoice, XaiTTSCodec } from "./providers/xai-tts";
+export type { XaiCreateCustomVoiceOptions } from "./providers/xai-tts";
export { scheduleCron, scheduleOnce, scheduleInterval } from "./scheduler";
export type { ScheduleHandle, JobCallback } from "./scheduler";
// Provider adapter types (re-exported for advanced users who build custom
@@ -174,6 +191,8 @@ export type { OpenAITranscribeSTTOptions } from "./stt/openai-transcribe";
export { STT as CartesiaSTT } from "./stt/cartesia";
export type { CartesiaSTTOptions } from "./stt/cartesia";
export { STT as SonioxSTT } from "./stt/soniox";
+export { STT as XaiSTT } from "./stt/xai";
+export type { XaiSTTOptions } from "./stt/xai";
export { STT as AssemblyAISTT } from "./stt/assemblyai";
export type { AssemblyAISTTOptions } from "./stt/assemblyai";
export { STT as SpeechmaticsSTT } from "./stt/speechmatics";
@@ -213,6 +232,8 @@ export { TTS as SonioxTTS } from "./tts/soniox";
export type { SonioxTTSOptions, SonioxCarrierOptions } from "./tts/soniox";
export { TTS as SarvamTTS } from "./tts/sarvam";
export type { SarvamTTSOptions, SarvamCarrierOptions } from "./tts/sarvam";
+export { TTS as XaiTTS } from "./tts/xai";
+export type { XaiTTSOptions, XaiCarrierOptions } from "./tts/xai";
// New namespaced LLM classes (Phase 2 of the v0.5.x API refactor).
export { LLM as OpenAILLM } from "./llm/openai";
@@ -329,6 +350,8 @@ export { GeminiCascade } from "./engines/gemini-cascade";
export type { GeminiCascadeOptions } from "./engines/gemini-cascade";
export { InworldRealtime } from "./engines/inworld";
export type { InworldRealtimeOptions } from "./engines/inworld";
+export { XaiRealtime } from "./engines/xai";
+export type { XaiRealtimeOptions } from "./engines/xai";
// Tunnel markers.
export { CloudflareTunnel, Ngrok, Static as StaticTunnel } from "./tunnels";
diff --git a/libraries/typescript/src/metrics.ts b/libraries/typescript/src/metrics.ts
index 113888a2..22fa40b6 100644
--- a/libraries/typescript/src/metrics.ts
+++ b/libraries/typescript/src/metrics.ts
@@ -902,6 +902,13 @@ export class CallMetricsAccumulator {
},
model?: string | null,
): void {
+ // xAI Grok Voice Agent bills per MINUTE (see ``_computeCost``), not per
+ // token. It is OpenAI-Realtime-GA-compatible, so its ``response.done`` MAY
+ // carry an OpenAI-shaped ``usage`` block — but metering it here would both
+ // double-count against the per-minute cost AND surface a bogus
+ // "saved from caching" figure on the dashboard. Ignore token usage for
+ // MINUTE-unit realtime providers; the duration path owns their cost.
+ if (this.providerMode === 'xai_realtime') return;
const resolvedModel = model || this.realtimeModel || null;
this._totalRealtimeCost += calculateRealtimeCost(usage, this._pricing, resolvedModel);
this._totalRealtimeCachedSavings += calculateRealtimeCachedSavings(
@@ -1247,6 +1254,20 @@ export class CallMetricsAccumulator {
stt = 0;
tts = 0;
llm = this._totalRealtimeCost;
+ } else if (this.providerMode === 'xai_realtime') {
+ // xAI Grok Voice Agent bills per MINUTE of session audio (not per token),
+ // so the realtime cost is derived from the call duration at call-end
+ // rather than the per-turn token ``usage`` accumulated in
+ // ``_totalRealtimeCost`` (which stays 0 for xAI).
+ stt = 0;
+ tts = 0;
+ llm = calculateRealtimeCost(
+ {},
+ this._pricing,
+ this.realtimeModel || null,
+ 'xai_realtime',
+ durationSeconds,
+ );
} else if (this.providerMode === 'elevenlabs_convai') {
stt = 0;
tts = 0;
diff --git a/libraries/typescript/src/pricing.ts b/libraries/typescript/src/pricing.ts
index 4fa98009..a4bbbcdc 100644
--- a/libraries/typescript/src/pricing.ts
+++ b/libraries/typescript/src/pricing.ts
@@ -62,6 +62,13 @@ export interface ProviderPricing {
text_output_per_token?: number;
cached_audio_input_per_token?: number;
cached_text_input_per_token?: number;
+ /**
+ * Per-message cost for realtime text-input items (`conversation.item.create`).
+ * Documented constant for MINUTE-unit realtime providers (e.g. xAI bills
+ * $0.004/message) — NOT metered in v1 (no per-message counter in the metrics
+ * accumulator); recorded so the rate is discoverable and can be wired later.
+ */
+ text_input_per_message?: number;
/**
* Per-model rate overrides keyed by model identifier. When the cost-calc
* function receives a ``model`` arg, the matching entry overlays the
@@ -167,6 +174,11 @@ export const DEFAULT_PRICING: Record = {
cartesia_stt: { unit: PricingUnit.MINUTE, price: 0.0025 },
// Soniox real-time STT — $0.12/hr = $0.002/min
soniox: { unit: PricingUnit.MINUTE, price: 0.002 },
+ // xAI streaming STT — $0.20/hr = $0.20/60 per min. (Batch/REST is cheaper at
+ // $0.10/hr ≈ $0.10/60 per min; the pipeline meters the STREAMING rate, so
+ // this is the default. Full-precision expression matches the Python SDK.
+ // Source: docs.x.ai/developers/pricing.)
+ xai: { unit: PricingUnit.MINUTE, price: 0.2 / 60 },
// Speechmatics Pro tier — $0.24/hr = $0.0040/min (new users land here).
// Previous $0.0173 default reflected a legacy Standard tier that was
// retired; users were being over-billed ~4.3x.
@@ -277,6 +289,9 @@ export const DEFAULT_PRICING: Record = {
// chars/hr, $0.70/hr ÷ 54 ≈ $0.013 / 1k chars. Source:
// https://soniox.com/pricing (also https://soniox.com/text-to-speech).
soniox_tts: { unit: PricingUnit.THOUSAND_CHARS, price: 0.013 },
+ // xAI TTS — $15.00 / 1M chars = $0.000015/char = $0.015 / 1k chars.
+ // Source: docs.x.ai/developers/pricing.
+ xai_tts: { unit: PricingUnit.THOUSAND_CHARS, price: 0.015 },
// Sarvam AI Bulbul TTS — per 1,000 characters. INR is authoritative; USD is an
// indicative FX conversion (~Rs 83-84/USD). Bulbul v3 (default): Rs 30 / 10k
// chars ≈ $0.036/1k; Bulbul v2: Rs 15 / 10k ≈ $0.018/1k. Billed per character,
@@ -358,6 +373,18 @@ export const DEFAULT_PRICING: Record = {
},
},
},
+ // xAI Realtime (Grok Voice Agent) — per MINUTE of session audio: $3.00/hr =
+ // $0.05/min. Unlike OpenAI Realtime (token-based), xAI bills per minute, so
+ // ``calculateRealtimeCost`` takes the MINUTE branch when resolved against this
+ // entry (pass ``durationSeconds`` + ``provider: 'xai_realtime'``).
+ // ``text_input_per_message`` ($0.004 per ``conversation.item.create``) is a
+ // documented constant — NOT metered in v1 (no per-message counter).
+ // Source: docs.x.ai/developers/pricing.
+ xai_realtime: {
+ unit: PricingUnit.MINUTE,
+ price: 0.05,
+ text_input_per_message: 0.004,
+ },
// Gemini Cascade — two-leg architecture: Live leg (text in/out per token)
// and TTS leg (text-in / audio-out per token). Rates per 1M tokens (USD).
// Sources: Google AI Studio pricing page (verified 2026-06-18).
@@ -515,14 +542,28 @@ export function calculateTtsCost(
}
/**
- * Calculate OpenAI Realtime cost from token usage.
+ * Calculate realtime session cost.
*
- * OpenAI bills the cached portion of ``input_token_details.audio_tokens`` and
- * ``.text_tokens`` at the reduced cached rate (typically ~3% of full for audio,
- * ~10% of full for text on the mini model). ``cached_tokens_details`` is a
- * nested breakdown of the same ``input_token_details`` totals — the cached
- * counts are already INCLUDED in the top-level totals, so we subtract them
- * out before applying the full rate and add them back at the cached rate.
+ * Two billing shapes are supported, selected by the resolved provider entry's
+ * ``unit``:
+ *
+ * - **TOKEN** (OpenAI Realtime, the default provider): billed from the token
+ * ``usage`` payload. OpenAI bills the cached portion of
+ * ``input_token_details.audio_tokens`` / ``.text_tokens`` at the reduced
+ * cached rate (~3% of full for audio, ~10% for text on the mini model).
+ * ``cached_tokens_details`` is a nested breakdown of the same
+ * ``input_token_details`` totals — already INCLUDED in the top-level totals
+ * — so we subtract them out before applying the full rate and add them back
+ * at the cached rate. ``durationSeconds`` is ignored on this path.
+ * - **MINUTE** (xAI Grok Voice Agent): billed from ``durationSeconds`` at the
+ * entry's per-minute ``price`` (``price * durationSeconds / 60``); ``0`` when
+ * ``durationSeconds`` is omitted. The token ``usage`` is ignored here.
+ *
+ * ``provider`` selects which ``pricing`` entry to resolve. It defaults to
+ * ``"openai_realtime"`` so existing callers are unaffected; pass
+ * ``"xai_realtime"`` (with ``durationSeconds``) for the per-minute path. The
+ * parameter order (``provider`` before ``durationSeconds``) mirrors the Python
+ * ``calculate_realtime_cost`` keyword-only args.
*/
export function calculateRealtimeCost(
usage: {
@@ -535,8 +576,15 @@ export function calculateRealtimeCost(
},
pricing: Record,
model?: string | null,
+ provider: string = 'openai_realtime',
+ durationSeconds?: number,
): number {
- const rates = resolveProviderRates(pricing.openai_realtime, model);
+ const rates = resolveProviderRates(pricing[provider], model);
+ // Per-minute realtime providers (e.g. xAI) bill on session audio duration,
+ // not tokens. 0.0 when the caller has no duration yet.
+ if (rates.unit === 'minute') {
+ return ((rates.price ?? 0) * (durationSeconds ?? 0)) / 60;
+ }
if (rates.unit !== 'token') return 0;
const input = (usage.input_token_details ?? {}) as {
diff --git a/libraries/typescript/src/providers/openai-realtime-2.ts b/libraries/typescript/src/providers/openai-realtime-2.ts
index e6fdf253..23ea8d95 100644
--- a/libraries/typescript/src/providers/openai-realtime-2.ts
+++ b/libraries/typescript/src/providers/openai-realtime-2.ts
@@ -108,8 +108,21 @@ export class OpenAIRealtime2Adapter extends OpenAIRealtimeAdapter {
* threshold. */
private inbound8kCarry: number | null = null;
+ /**
+ * Base Realtime WebSocket URL (no query string). Extracted so
+ * OpenAI-GA-compatible subclasses that speak the SAME wire protocol against
+ * a different host — e.g. `XaiRealtimeAdapter` (`wss://api.x.ai/v1/realtime`)
+ * — can override just the endpoint while inheriting `connect()` /
+ * `openParkedConnection()` / the GA event-translation shim unchanged. The
+ * default returns OpenAI's GA endpoint, so behaviour for this class is
+ * byte-identical to the previous hardcoded literal.
+ */
+ protected realtimeBaseUrl(): string {
+ return 'wss://api.openai.com/v1/realtime';
+ }
+
/** GA-shape `session.update` payload. See module-level docstring. */
- private buildGASessionConfig(): Record {
+ protected buildGASessionConfig(): Record {
const opts = this.options;
// The GA endpoint requires audio/pcm with rate >= 24000 for both
// directions. mulaw / pcma are not honoured by the audio engine
@@ -228,7 +241,7 @@ export class OpenAIRealtime2Adapter extends OpenAIRealtimeAdapter {
* output}` + `output_modalities` + `session.type === "realtime"`.
*/
async connect(): Promise {
- const url = `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(this.model)}`;
+ const url = `${this.realtimeBaseUrl()}?model=${encodeURIComponent(this.model)}`;
this.ws = new WebSocket(url, {
headers: { Authorization: `Bearer ${this.apiKey}` },
});
@@ -399,7 +412,7 @@ export class OpenAIRealtime2Adapter extends OpenAIRealtimeAdapter {
* tokens. An idle parked socket costs $0.
*/
override async openParkedConnection(): Promise {
- const url = `wss://api.openai.com/v1/realtime?model=${encodeURIComponent(this.model)}`;
+ const url = `${this.realtimeBaseUrl()}?model=${encodeURIComponent(this.model)}`;
const ws = new WebSocket(url, {
headers: { Authorization: `Bearer ${this.apiKey}` },
});
diff --git a/libraries/typescript/src/providers/xai-realtime.ts b/libraries/typescript/src/providers/xai-realtime.ts
new file mode 100644
index 00000000..626736ea
--- /dev/null
+++ b/libraries/typescript/src/providers/xai-realtime.ts
@@ -0,0 +1,308 @@
+/**
+ * xAI Grok Voice Agent realtime adapter.
+ *
+ * **Beta: validated against the xAI Voice Agent API spec; not yet exercised
+ * against a live phone call.**
+ *
+ * xAI's Voice Agent API is OpenAI-Realtime-GA-compatible — xAI documents an
+ * "Migrating from OpenAI Realtime" path where only the base URL, the API key,
+ * and the model change; the GA `session.update` / `response.create` / streaming
+ * event flow works as-is. This adapter therefore SUBCLASSES
+ * {@link OpenAIRealtime2Adapter} (the GA adapter) and overrides only:
+ * - {@link realtimeBaseUrl} → `wss://api.x.ai/v1/realtime`, and
+ * - {@link buildGASessionConfig} → grafts the xAI-specific `session.update`
+ * keys (reasoning effort, language hint, keyterms, output speed,
+ * pronunciation `replace`, session `resumption`, extra turn-detection
+ * knobs) on top of the inherited GA body, plus appends any raw
+ * server-side tools.
+ *
+ * Everything else — the `session.created` → `session.update` → `session.updated`
+ * handshake, the GA event-name translation shim, barge-in / truncate, heartbeat,
+ * tool calling, and the inherited PCM-24 ↔ mulaw-8 telephony transcode — is
+ * reused unchanged. Because it `instanceof OpenAIRealtimeAdapter`, every feature
+ * gate in the stream handler fires for xAI with no per-provider branches.
+ *
+ * Audio (v1): inherits the GA `audio/pcm` @ 24 kHz path and the outbound
+ * PCM24→mulaw8 transcode. xAI supports native `audio/pcmu` @ 8 kHz which would
+ * let telephony calls skip the transcode entirely — that is a flagged
+ * follow-up optimisation, NOT implemented here.
+ */
+
+import type WebSocket from 'ws';
+import { getLogger } from '../logger';
+import {
+ OpenAIRealtimeAudioFormat,
+ type OpenAIRealtimeOptions,
+} from './openai-realtime';
+import { OpenAIRealtime2Adapter } from './openai-realtime-2';
+
+/** Default xAI Voice Agent WebSocket base URL (no query string). */
+export const XAI_REALTIME_WS_URL = 'wss://api.x.ai/v1/realtime';
+/** Default model — `grok-voice-latest` always points at the newest voice model. */
+export const XAI_REALTIME_DEFAULT_MODEL = 'grok-voice-latest';
+/** Default voice — xAI's built-in `eve` (case-insensitive). */
+export const XAI_REALTIME_DEFAULT_VOICE = 'eve';
+/**
+ * xAI's input-transcription model. Setting it keeps the session xAI-valid —
+ * the GA base defaults to OpenAI's `whisper-1`, which the xAI endpoint rejects.
+ * An explicit `inputAudioTranscriptionModel` from the caller still wins.
+ */
+export const XAI_REALTIME_TRANSCRIPTION_MODEL = 'grok-transcribe';
+
+/**
+ * Constructor options for {@link XaiRealtimeAdapter}.
+ *
+ * Extends the base {@link OpenAIRealtimeOptions} (VAD tuning, transcription
+ * language, response gating, …) minus `reasoningEffort` — xAI's reasoning
+ * field accepts only `"high"` / `"none"`, a different set from OpenAI's tiers —
+ * and adds the xAI-specific `session.update` knobs. All optional with safe
+ * defaults; unset knobs are omitted from the wire so xAI applies its own
+ * server defaults.
+ */
+export interface XaiRealtimeAdapterOptions
+ extends Omit {
+ /** Voice-agent model id (passed through as `?model=`). */
+ model?: string;
+ /** Voice id (built-in or custom). */
+ voice?: string;
+ /** System prompt / instructions. */
+ instructions?: string;
+ /** Tool/function declarations advertised to the model. */
+ tools?: Array<{ name: string; description: string; parameters: Record; strict?: boolean }>;
+ /** Override the WebSocket base URL (no query string). */
+ baseUrl?: string;
+ /** Wire audio format. Defaults to `g711_ulaw` (carrier-native pass-through). */
+ audioFormat?: OpenAIRealtimeAudioFormat;
+ /**
+ * Reasoning effort. `"high"` (xAI server default) enables reasoning;
+ * `"none"` disables it. Omitted from the wire when unset → server default.
+ * Sent as `session.reasoning.effort`.
+ */
+ reasoningEffort?: 'high' | 'none';
+ /**
+ * Server VAD activation threshold in `[0.1, 0.9]` (xAI default `0.85`).
+ * Higher requires louder audio to trigger. Sent as
+ * `turn_detection.threshold`. Omitted when unset.
+ */
+ vadThreshold?: number;
+ /**
+ * Audio (ms) included before detected speech start, `[0, 10000]` (xAI
+ * default `333`). Sent as `turn_detection.prefix_padding_ms`. Omitted when
+ * unset.
+ */
+ prefixPaddingMs?: number;
+ /**
+ * When set, the server re-engages the user after this many ms of silence
+ * following an assistant turn. Sent as `turn_detection.idle_timeout_ms`.
+ * Omitted when unset.
+ */
+ idleTimeoutMs?: number;
+ /**
+ * BCP-47 code biasing input transcription toward one language. Sent as
+ * `audio.input.transcription.language_hint`. Omitted when unset.
+ */
+ languageHint?: string;
+ /**
+ * Up to 100 key terms (≤50 chars each) biasing transcription toward
+ * domain vocabulary. Sent as `audio.input.transcription.keyterms`. Omitted
+ * when empty.
+ */
+ keyterms?: readonly string[];
+ /**
+ * Assistant playback speed multiplier in `[0.7, 1.5]` (default `1.0`). Sent
+ * as `audio.output.speed`. Omitted when unset.
+ */
+ speed?: number;
+ /**
+ * Pronunciation map applied to the model's output before TTS, e.g.
+ * `{ "Acme Mobile": "Acme Mobull" }`. Only the spoken audio changes; the
+ * transcript keeps the original. Sent as `session.replace`. Omitted when unset.
+ */
+ replace?: Readonly>;
+ /**
+ * Opt in to session resumption — the server caches conversation turns and
+ * replays them on reconnect. Sent as `session.resumption.enabled`. Defaults
+ * to `false` (omitted).
+ */
+ resumption?: boolean;
+ /**
+ * Raw xAI server-side tool objects appended verbatim to `session.tools`
+ * (e.g. `{ type: 'web_search' }`, `{ type: 'x_search' }`,
+ * `{ type: 'mcp', server_url, server_label }`, `{ type: 'file_search', … }`).
+ * These execute server-side at xAI — the client never handles their results.
+ * Defaults to empty.
+ */
+ serverTools?: ReadonlyArray>;
+}
+
+/**
+ * Realtime adapter for xAI's OpenAI-GA-compatible Grok Voice Agent API.
+ *
+ * @example
+ * ```ts
+ * import { Patter, Twilio, XaiRealtime } from "getpatter";
+ *
+ * const phone = new Patter({ carrier: new Twilio(), phoneNumber: "+1..." });
+ * const agent = phone.agent({
+ * engine: new XaiRealtime({ voice: "eve" }), // reads XAI_API_KEY
+ * systemPrompt: "You are a friendly receptionist.",
+ * firstMessage: "Hello! How can I help?",
+ * });
+ * ```
+ */
+export class XaiRealtimeAdapter extends OpenAIRealtime2Adapter {
+ /** Stable pricing/dashboard key. */
+ static readonly providerKey = 'xai_realtime';
+
+ private readonly wsBase: string;
+ private readonly xaiReasoningEffort?: 'high' | 'none';
+ private readonly vadThreshold?: number;
+ private readonly prefixPaddingMs?: number;
+ private readonly idleTimeoutMs?: number;
+ private readonly languageHint?: string;
+ private readonly keyterms: readonly string[];
+ private readonly outputSpeed?: number;
+ private readonly replaceMap?: Readonly>;
+ private readonly resumptionEnabled: boolean;
+ private readonly serverTools: ReadonlyArray>;
+
+ constructor(apiKey: string, opts: XaiRealtimeAdapterOptions = {}) {
+ // Forward only the base OpenAIRealtimeOptions keys to the GA adapter —
+ // `reasoningEffort` is intentionally EXCLUDED (xAI's "high"/"none" values
+ // differ from OpenAI's tiers) and grafted in the session-config override.
+ const baseOptions: OpenAIRealtimeOptions = {
+ temperature: opts.temperature,
+ maxResponseOutputTokens: opts.maxResponseOutputTokens,
+ modalities: opts.modalities,
+ toolChoice: opts.toolChoice,
+ // Default the transcription model to xAI's `grok-transcribe` — the GA
+ // base otherwise emits OpenAI's `whisper-1`, which xAI rejects. Matches
+ // the Python adapter (`XAI_TRANSCRIPTION_MODEL`).
+ inputAudioTranscriptionModel:
+ opts.inputAudioTranscriptionModel ?? XAI_REALTIME_TRANSCRIPTION_MODEL,
+ transcriptionLanguage: opts.transcriptionLanguage,
+ vadType: opts.vadType,
+ silenceDurationMs: opts.silenceDurationMs,
+ noiseReduction: opts.noiseReduction,
+ turnDetection: opts.turnDetection,
+ gateResponseOnTranscript: opts.gateResponseOnTranscript,
+ };
+ super(
+ apiKey,
+ opts.model ?? XAI_REALTIME_DEFAULT_MODEL,
+ opts.voice ?? XAI_REALTIME_DEFAULT_VOICE,
+ opts.instructions ?? '',
+ opts.tools,
+ opts.audioFormat ?? OpenAIRealtimeAudioFormat.G711_ULAW,
+ baseOptions,
+ );
+ this.wsBase = (opts.baseUrl ?? XAI_REALTIME_WS_URL).replace(/\/+$/, '');
+ this.xaiReasoningEffort = opts.reasoningEffort;
+ this.vadThreshold = opts.vadThreshold;
+ this.prefixPaddingMs = opts.prefixPaddingMs;
+ this.idleTimeoutMs = opts.idleTimeoutMs;
+ this.languageHint = opts.languageHint;
+ this.keyterms = opts.keyterms ?? [];
+ this.outputSpeed = opts.speed;
+ this.replaceMap = opts.replace;
+ this.resumptionEnabled = opts.resumption ?? false;
+ this.serverTools = opts.serverTools ?? [];
+ }
+
+ /** xAI Voice Agent endpoint — overrides the inherited OpenAI GA URL. */
+ protected override realtimeBaseUrl(): string {
+ return this.wsBase;
+ }
+
+ /**
+ * GA-shape `session.update` body with the xAI-specific keys grafted on.
+ *
+ * Calls the inherited GA builder (which already emits the nested
+ * `audio.{input,output}.format`, `turn_detection`, `voice`, `instructions`,
+ * and function `tools`), then adds the xAI extensions ONLY when the caller
+ * configured them so xAI's own server defaults apply otherwise.
+ */
+ protected override buildGASessionConfig(): Record {
+ const config = super.buildGASessionConfig();
+
+ // Reasoning: xAI accepts "high" | "none". Grafted here (not inherited)
+ // because the base option type carries OpenAI's tiers, not xAI's values.
+ if (this.xaiReasoningEffort !== undefined) {
+ config.reasoning = { effort: this.xaiReasoningEffort };
+ }
+
+ const audio = config.audio as
+ | { input?: Record; output?: Record }
+ | undefined;
+ const audioInput = audio?.input;
+
+ // Transcription biasing (language hint + keyterms) nests under
+ // audio.input.transcription, which the base builder always emits.
+ if (audioInput) {
+ const transcription = (audioInput.transcription ??= {}) as Record;
+ // xAI biases ASR via `language_hint` (BCP-47), NOT OpenAI's `language`.
+ // Drop the OpenAI-only key if the base builder emitted it (matches the
+ // Python adapter, which pops `transcription["language"]`).
+ delete transcription.language;
+ if (this.languageHint !== undefined) {
+ transcription.language_hint = this.languageHint;
+ }
+ if (this.keyterms.length > 0) {
+ transcription.keyterms = [...this.keyterms];
+ }
+ // Extra turn-detection knobs overlay the base turn_detection dict.
+ const turnDetection = audioInput.turn_detection as Record | undefined;
+ if (turnDetection) {
+ if (this.vadThreshold !== undefined) turnDetection.threshold = this.vadThreshold;
+ if (this.prefixPaddingMs !== undefined) {
+ turnDetection.prefix_padding_ms = this.prefixPaddingMs;
+ }
+ if (this.idleTimeoutMs !== undefined) {
+ turnDetection.idle_timeout_ms = this.idleTimeoutMs;
+ }
+ }
+ }
+
+ // Output playback speed nests under audio.output.
+ if (this.outputSpeed !== undefined && audio?.output) {
+ audio.output.speed = this.outputSpeed;
+ }
+
+ // Pronunciation replacements + session resumption live at the session root.
+ if (this.replaceMap !== undefined) {
+ config.replace = { ...this.replaceMap };
+ }
+ if (this.resumptionEnabled) {
+ config.resumption = { enabled: true };
+ }
+
+ // Server-side tools (web_search / x_search / mcp / file_search) append to
+ // whatever function tools the base already advertised.
+ if (this.serverTools.length > 0) {
+ const existing = Array.isArray(config.tools)
+ ? (config.tools as unknown[])
+ : [];
+ config.tools = [...existing, ...this.serverTools.map((t) => ({ ...t }))];
+ }
+
+ return config;
+ }
+
+ /**
+ * No-op warmup. The base {@link OpenAIRealtimeAdapter.warmup} opens a socket
+ * to OpenAI's endpoint (wrong host for an xAI key). xAI is not wired into the
+ * prewarm pipeline, so warmup is skipped.
+ */
+ override async warmup(): Promise {
+ getLogger().debug('xAI Realtime: warmup is a no-op (not parked)');
+ }
+
+ /**
+ * Parking is not supported for xAI — the base park path would target the
+ * OpenAI edge keepalive. Reject so any caller treats it as a cache miss and
+ * falls through to the cold {@link connect} path.
+ */
+ override async openParkedConnection(): Promise {
+ throw new Error('xAI Realtime: openParkedConnection is not supported');
+ }
+}
diff --git a/libraries/typescript/src/providers/xai-stt.ts b/libraries/typescript/src/providers/xai-stt.ts
new file mode 100644
index 00000000..5c6deaab
--- /dev/null
+++ b/libraries/typescript/src/providers/xai-stt.ts
@@ -0,0 +1,517 @@
+/**
+ * xAI Speech-to-Text adapter for Patter (TypeScript).
+ *
+ * **Beta: validated against the xAI STT API spec; not yet exercised against a
+ * live phone call.**
+ *
+ * Pure WebSocket client for xAI's real-time STT endpoint
+ * (`wss://api.x.ai/v1/stt`). Config is carried entirely in the URL query string
+ * (there is NO setup message); audio is sent as raw binary frames. The adapter
+ * WAITS for the server's `transcript.created` event before declaring the
+ * connection ready, then maps `transcript.partial` (`is_final` / `speech_final`)
+ * onto Patter's interim / chunk-final / utterance-final transcript semantics —
+ * the same two-flag scheme Deepgram uses.
+ *
+ * Also exports {@link xaiTranscribe}, a one-shot REST helper for the batch
+ * transcription endpoint, plus the {@link XaiTranscription} / {@link XaiWord}
+ * result shapes.
+ *
+ * Credential: `XAI_API_KEY`. Auth is an `Authorization: Bearer ` header
+ * on both the streaming and batch endpoints.
+ */
+
+import WebSocket from 'ws';
+import { getLogger } from '../logger';
+
+/** xAI real-time STT WebSocket endpoint. */
+export const XAI_STT_WS_URL = 'wss://api.x.ai/v1/stt';
+/** xAI batch (one-shot) STT REST endpoint. */
+export const XAI_STT_REST_URL = 'https://api.x.ai/v1/stt';
+
+/** Raw audio encodings accepted by xAI's STT endpoints. */
+export const XaiSTTEncoding = {
+ PCM: 'pcm',
+ MULAW: 'mulaw',
+ ALAW: 'alaw',
+} as const;
+export type XaiSTTEncoding = (typeof XaiSTTEncoding)[keyof typeof XaiSTTEncoding];
+
+/** Per-word timing / metadata for a Patter transcript. */
+export interface XaiSTTWord {
+ readonly word?: string;
+ readonly start?: number;
+ readonly end?: number;
+ readonly speaker?: number;
+}
+
+/** Patter-normalised transcript event emitted by {@link XaiSTT}. */
+export interface Transcript {
+ readonly text: string;
+ readonly isFinal: boolean;
+ /** Overall / end-of-turn confidence in [0, 1]. */
+ readonly confidence?: number;
+ /** xAI utterance-end hint (`speech_final`) — faster than `isFinal`. */
+ readonly speechFinal?: boolean;
+ /** True when this final was produced in response to a `finalize` command. */
+ readonly fromFinalize?: boolean;
+ /** Per-word timings/metadata when xAI emits them. */
+ readonly words?: ReadonlyArray;
+ /** Which provider event this transcript represents (always `Results`). */
+ readonly eventType?: string;
+}
+
+type TranscriptCallback = (transcript: Transcript) => void | Promise;
+
+/** Raw `transcript.partial` / `transcript.done` word entry from xAI. */
+interface XaiWireWord {
+ text?: string;
+ start?: number;
+ end?: number;
+ speaker?: number;
+}
+
+/** Raw xAI streaming server event (JSON text frame). */
+interface XaiSTTMessage {
+ type?: string;
+ text?: string;
+ words?: XaiWireWord[];
+ is_final?: boolean;
+ speech_final?: boolean;
+ start?: number;
+ duration?: number;
+ end_of_turn_confidence?: number;
+ message?: string;
+}
+
+function mapWords(words: XaiWireWord[] | undefined): ReadonlyArray | undefined {
+ if (!words || words.length === 0) return undefined;
+ return words.map((w) => ({
+ word: w.text,
+ start: w.start,
+ end: w.end,
+ speaker: w.speaker,
+ }));
+}
+
+/** Constructor options for {@link XaiSTT}. */
+export interface XaiSTTOptions {
+ /**
+ * Audio encoding of the frames fed to {@link XaiSTT.sendAudio}. Default
+ * `pcm` — the Patter pipeline decodes carrier μ-law to linear PCM16 @ 16 kHz
+ * before the STT stage, so `pcm` @ 16000 is the correct wire match. Set
+ * `mulaw` / `alaw` @ 8000 only if you feed raw carrier audio directly.
+ */
+ readonly encoding?: XaiSTTEncoding | string;
+ /** Input sample rate (Hz). Default `16000` (matches the pipeline's PCM16 feed). */
+ readonly sampleRate?: number;
+ /**
+ * Emit interim (`is_final=false`) partials every ~500 ms. Default `true`
+ * (mirrors Deepgram/Soniox — the pipeline consumes partials for barge-in).
+ */
+ readonly interimResults?: boolean;
+ /**
+ * BCP-47 language code enabling server-side text formatting (ITN). Omitted
+ * when unset (auto).
+ */
+ readonly language?: string;
+ /**
+ * Silence (ms) before an utterance is marked final, `[0, 5000]` (xAI default
+ * `10`). Omitted when unset.
+ */
+ readonly endpointingMs?: number;
+ /**
+ * Smart Turn end-of-turn confidence threshold in `[0.0, 1.0]`. Enables xAI's
+ * ML end-of-turn model. Omitted when unset.
+ */
+ readonly smartTurn?: number;
+ /**
+ * Max silence (ms) before forcing `speech_final` when Smart Turn is on,
+ * `[1, 5000]`. Omitted when unset.
+ */
+ readonly smartTurnTimeoutMs?: number;
+ /** Up to 100 bias terms (≤50 chars each), sent as repeated `keyterm=` params. */
+ readonly keyterms?: readonly string[];
+ /** Speaker diarization — words gain a `speaker` index. Default `false`. */
+ readonly diarize?: boolean;
+ /** Include filler words ("uh"/"um") in the transcript. Default `false`. */
+ readonly fillerWords?: boolean;
+ /** Override the WebSocket base URL (e.g. for tests). */
+ readonly baseUrl?: string;
+}
+
+/** Streaming STT adapter for xAI's real-time WebSocket API. */
+export class XaiSTT {
+ /** Stable pricing/dashboard key — read by stream-handler/metrics. */
+ static readonly providerKey = 'xai';
+
+ private ws: WebSocket | null = null;
+ private readonly callbacks = new Set();
+ private finalizePending = false;
+
+ private readonly apiKey: string;
+ private readonly encoding: string;
+ private readonly sampleRate: number;
+ private readonly interimResults: boolean;
+ private readonly language?: string;
+ private readonly endpointingMs?: number;
+ private readonly smartTurn?: number;
+ private readonly smartTurnTimeoutMs?: number;
+ private readonly keyterms: readonly string[];
+ private readonly diarize: boolean;
+ private readonly fillerWords: boolean;
+ private readonly baseUrl: string;
+
+ /** Construction args replayed by clone(). */
+ private readonly patterCtorArgs: unknown[];
+
+ constructor(apiKey: string, options: XaiSTTOptions = {}) {
+ this.patterCtorArgs = [apiKey, options];
+ if (!apiKey) {
+ throw new Error('xAI STT: apiKey is required');
+ }
+ this.apiKey = apiKey;
+ this.encoding = options.encoding ?? XaiSTTEncoding.PCM;
+ this.sampleRate = options.sampleRate ?? 16000;
+ this.interimResults = options.interimResults ?? true;
+ this.language = options.language;
+ this.endpointingMs = options.endpointingMs;
+ this.smartTurn = options.smartTurn;
+ this.smartTurnTimeoutMs = options.smartTurnTimeoutMs;
+ this.keyterms = options.keyterms ?? [];
+ this.diarize = options.diarize ?? false;
+ this.fillerWords = options.fillerWords ?? false;
+ this.baseUrl = options.baseUrl ?? XAI_STT_WS_URL;
+ }
+
+ /**
+ * Fresh adapter built with this instance's construction arguments — called
+ * per call by the stream handler so concurrent calls never share connection
+ * state (sockets/queues; cross-call transcript bleed).
+ */
+ clone(): this {
+ const ctor = this.constructor as new (...args: unknown[]) => this;
+ return new ctor(...this.patterCtorArgs);
+ }
+
+ /** Build the query-parameterised WebSocket URL (config is URL-only). */
+ private buildUrl(): string {
+ const params = new URLSearchParams({
+ sample_rate: String(this.sampleRate),
+ encoding: this.encoding,
+ interim_results: this.interimResults ? 'true' : 'false',
+ });
+ if (this.language !== undefined) params.set('language', this.language);
+ if (this.endpointingMs !== undefined) params.set('endpointing', String(this.endpointingMs));
+ if (this.smartTurn !== undefined) params.set('smart_turn', String(this.smartTurn));
+ if (this.smartTurnTimeoutMs !== undefined) {
+ params.set('smart_turn_timeout', String(this.smartTurnTimeoutMs));
+ }
+ if (this.diarize) params.set('diarize', 'true');
+ if (this.fillerWords) params.set('filler_words', 'true');
+ // `keyterm` is REPEATED (one param per term), not comma-joined.
+ for (const term of this.keyterms) params.append('keyterm', term);
+ return `${this.baseUrl}?${params.toString()}`;
+ }
+
+ /**
+ * Open the streaming WebSocket and wait for the server's `transcript.created`
+ * ready signal before resolving. Audio sent before `transcript.created` can
+ * be dropped, so callers must await this promise.
+ */
+ async connect(): Promise {
+ this.finalizePending = false;
+ this.ws = new WebSocket(this.buildUrl(), {
+ headers: { Authorization: `Bearer ${this.apiKey}` },
+ });
+
+ await new Promise((resolve, reject) => {
+ let settled = false;
+ const ws = this.ws!;
+
+ const onSetupMessage = (raw: Buffer | string): void => {
+ if (settled) return;
+ let msg: XaiSTTMessage;
+ try {
+ msg = JSON.parse(raw.toString()) as XaiSTTMessage;
+ } catch (e) {
+ getLogger().warn(`xAI STT: failed to parse setup message: ${String(e)}`);
+ return;
+ }
+ if (msg.type === 'transcript.created') {
+ cleanup();
+ resolve();
+ } else if (msg.type === 'error') {
+ cleanup();
+ try { ws.close(); } catch { /* ignore */ }
+ reject(new Error(`xAI STT setup error: ${msg.message ?? JSON.stringify(msg)}`));
+ }
+ };
+
+ const onSetupError = (err: Error): void => {
+ cleanup();
+ try { ws.close(); } catch { /* ignore */ }
+ reject(err);
+ };
+
+ const cleanup = (): void => {
+ if (settled) return;
+ settled = true;
+ clearTimeout(timer);
+ ws.off('message', onSetupMessage);
+ ws.off('error', onSetupError);
+ };
+
+ const timer = setTimeout(() => {
+ cleanup();
+ try { ws.close(); } catch { /* ignore */ }
+ reject(new Error('xAI STT connect timeout (no transcript.created within 10s)'));
+ }, 10000);
+
+ ws.on('message', onSetupMessage);
+ ws.on('error', onSetupError);
+ });
+
+ // Persistent listeners for the live stream.
+ this.ws.on('message', (raw) => this.handleMessage(raw.toString()));
+ this.ws.on('error', (err) => {
+ getLogger().error(`XaiSTT WebSocket error: ${String(err)}`);
+ });
+ }
+
+ private handleMessage(raw: string): void {
+ let msg: XaiSTTMessage;
+ try {
+ msg = JSON.parse(raw) as XaiSTTMessage;
+ } catch {
+ return;
+ }
+
+ switch (msg.type) {
+ case 'transcript.partial': {
+ const text = (msg.text ?? '').trim();
+ if (!text) return;
+ const speechFinal = Boolean(msg.speech_final);
+ // Mirror Deepgram: is_final OR speech_final marks a stable utterance;
+ // speech_final is the faster utterance-end hint.
+ const isFinal = Boolean(msg.is_final) || speechFinal;
+ const fromFinalize = isFinal && this.finalizePending;
+ if (fromFinalize) this.finalizePending = false;
+ this.emit({
+ text,
+ isFinal,
+ confidence: msg.end_of_turn_confidence ?? 1,
+ speechFinal,
+ fromFinalize,
+ words: mapWords(msg.words),
+ eventType: 'Results',
+ });
+ break;
+ }
+ case 'transcript.done': {
+ // Final flush after audio.done — surface any trailing text as a final.
+ const text = (msg.text ?? '').trim();
+ if (text) {
+ this.emit({
+ text,
+ isFinal: true,
+ confidence: msg.end_of_turn_confidence ?? 1,
+ speechFinal: true,
+ fromFinalize: this.finalizePending,
+ words: mapWords(msg.words),
+ eventType: 'Results',
+ });
+ }
+ this.finalizePending = false;
+ break;
+ }
+ case 'error':
+ // xAI keeps the connection open on error frames — log, do not close.
+ getLogger().error(`XaiSTT error: ${msg.message ?? JSON.stringify(msg)}`);
+ break;
+ default:
+ // transcript.created (post-setup) and unknown events — ignore.
+ break;
+ }
+ }
+
+ private emit(transcript: Transcript): void {
+ for (const cb of this.callbacks) {
+ // The registered callback is async (stream-handler's handler) — contain
+ // both sync throws and async rejections so an unhandled rejection can't
+ // kill the process.
+ try {
+ Promise.resolve(cb(transcript)).catch((err) =>
+ getLogger().error(`STT transcript callback failed: ${String(err)}`),
+ );
+ } catch (err) {
+ getLogger().error(`STT transcript callback threw: ${String(err)}`);
+ }
+ }
+ }
+
+ /** Send a raw binary audio chunk (PCM16 / μ-law / A-law per `encoding`). */
+ sendAudio(audio: Buffer): void {
+ if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
+ if (audio.length === 0) return;
+ this.ws.send(audio);
+ }
+
+ /**
+ * Force the current utterance to `speech_final` immediately (push-to-talk /
+ * VAD speech-end fast-path). The pipeline duck-types `stt.finalize` — without
+ * it every xAI turn waits out the full endpointing delay.
+ */
+ finalize(): void {
+ if (!this.ws || this.ws.readyState !== WebSocket.OPEN) return;
+ this.finalizePending = true;
+ try {
+ this.ws.send(JSON.stringify({ type: 'finalize' }));
+ } catch (err) {
+ getLogger().debug(`xAI STT finalize failed: ${String(err)}`);
+ }
+ }
+
+ /** Register a transcript listener. */
+ onTranscript(callback: TranscriptCallback): void {
+ this.callbacks.add(callback);
+ }
+
+ /** Unregister a previously registered transcript listener. */
+ offTranscript(callback: TranscriptCallback): void {
+ this.callbacks.delete(callback);
+ }
+
+ /** Signal end-of-audio (`audio.done`) best-effort, then close the socket. */
+ close(): void {
+ if (this.ws) {
+ try {
+ if (this.ws.readyState === WebSocket.OPEN) {
+ this.ws.send(JSON.stringify({ type: 'audio.done' }));
+ }
+ } catch {
+ // ignore
+ }
+ try {
+ this.ws.close();
+ } catch {
+ // ignore
+ }
+ this.ws = null;
+ }
+ }
+}
+
+/** A single word in a batch {@link XaiTranscription}. */
+export interface XaiWord {
+ readonly text: string;
+ readonly start: number;
+ readonly end: number;
+ /** Speaker index — present only when `diarize` was requested. */
+ readonly speaker?: number;
+}
+
+/** Parsed result of a batch {@link xaiTranscribe} call. */
+export interface XaiTranscription {
+ readonly text: string;
+ readonly language: string;
+ readonly duration: number;
+ readonly words: ReadonlyArray;
+}
+
+/** Options for the one-shot {@link xaiTranscribe} batch REST helper. */
+export interface XaiTranscribeOptions {
+ /** Raw audio bytes to transcribe. Provide this OR {@link url}. */
+ readonly audio?: Buffer | Uint8Array;
+ /** URL of an audio file for xAI to download and transcribe server-side. */
+ readonly url?: string;
+ /** API key. Falls back to `XAI_API_KEY` when omitted. */
+ readonly apiKey?: string;
+ /** BCP-47 language code (used with `format` for text formatting). */
+ readonly language?: string;
+ /** Inverse Text Normalization ("one hundred dollars" → "$100"). Requires `language`. */
+ readonly format?: boolean;
+ /** Speaker diarization — each word gains a `speaker` index. */
+ readonly diarize?: boolean;
+ /** Up to 100 bias terms (≤50 chars each), sent as repeated `keyterm` fields. */
+ readonly keyterms?: readonly string[];
+ /** Include filler words ("uh"/"um"). */
+ readonly fillerWords?: boolean;
+ /** Raw-audio encoding (`pcm` / `mulaw` / `alaw`). Only for headerless audio. */
+ readonly audioFormat?: XaiSTTEncoding | string;
+ /** Sample rate (Hz) — required for raw audio. */
+ readonly sampleRate?: number;
+ /** Filename for the multipart `file` part. Defaults to `audio.wav`. */
+ readonly filename?: string;
+ /** Override the REST endpoint (e.g. for tests). */
+ readonly baseUrl?: string;
+}
+
+interface XaiTranscriptionResponse {
+ text?: string;
+ language?: string;
+ duration?: number;
+ words?: Array<{ text?: string; start?: number; end?: number; speaker?: number }>;
+}
+
+/**
+ * One-shot batch transcription via xAI's REST endpoint (`POST /v1/stt`).
+ *
+ * **Beta.** Multipart order matters: all scalar fields are appended FIRST and
+ * the `file` part LAST (xAI requires the audio to be the final form field). Pass
+ * either `audio` bytes or a `url`.
+ */
+export async function xaiTranscribe(options: XaiTranscribeOptions): Promise {
+ const apiKey = options.apiKey ?? process.env.XAI_API_KEY;
+ if (!apiKey) {
+ throw new Error(
+ "xAI transcribe requires an apiKey. Pass { apiKey: '...' } or set XAI_API_KEY.",
+ );
+ }
+ if (!options.audio && !options.url) {
+ throw new Error('xAI transcribe requires either `audio` bytes or a `url`.');
+ }
+
+ const form = new FormData();
+ // Scalar fields FIRST.
+ if (options.url !== undefined) form.append('url', options.url);
+ if (options.language !== undefined) form.append('language', options.language);
+ if (options.format !== undefined) form.append('format', String(options.format));
+ if (options.diarize !== undefined) form.append('diarize', String(options.diarize));
+ if (options.fillerWords !== undefined) form.append('filler_words', String(options.fillerWords));
+ if (options.audioFormat !== undefined) form.append('audio_format', options.audioFormat);
+ if (options.sampleRate !== undefined) form.append('sample_rate', String(options.sampleRate));
+ for (const term of options.keyterms ?? []) form.append('keyterm', term);
+ // File LAST.
+ if (options.audio !== undefined) {
+ const bytes = options.audio instanceof Buffer ? options.audio : Buffer.from(options.audio);
+ const arrayBuffer = bytes.buffer.slice(
+ bytes.byteOffset,
+ bytes.byteOffset + bytes.byteLength,
+ ) as ArrayBuffer;
+ form.append('file', new Blob([arrayBuffer]), options.filename ?? 'audio.wav');
+ }
+
+ const response = await fetch(options.baseUrl ?? XAI_STT_REST_URL, {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${apiKey}` },
+ body: form,
+ signal: AbortSignal.timeout(120_000),
+ });
+ if (!response.ok) {
+ const body = await response.text();
+ throw new Error(`xAI transcribe error ${response.status}: ${body}`);
+ }
+
+ const data = (await response.json()) as XaiTranscriptionResponse;
+ return {
+ text: data.text ?? '',
+ language: data.language ?? '',
+ duration: data.duration ?? 0,
+ words: (data.words ?? []).map((w) => ({
+ text: w.text ?? '',
+ start: w.start ?? 0,
+ end: w.end ?? 0,
+ ...(w.speaker !== undefined ? { speaker: w.speaker } : {}),
+ })),
+ };
+}
diff --git a/libraries/typescript/src/providers/xai-tts.ts b/libraries/typescript/src/providers/xai-tts.ts
new file mode 100644
index 00000000..9968b8d4
--- /dev/null
+++ b/libraries/typescript/src/providers/xai-tts.ts
@@ -0,0 +1,340 @@
+/**
+ * xAI Text-to-Speech provider — HTTP one-shot `/tts` bytes endpoint.
+ *
+ * **Beta: validated against the xAI TTS API spec; not yet exercised against a
+ * live phone call.**
+ *
+ * Calls `POST https://api.x.ai/v1/tts`. The response body is raw audio bytes
+ * whose codec matches the requested `output_format.codec` (e.g. `pcm`,
+ * `mulaw`, `mp3`). This maps cleanly onto Patter's `synthesizeStream(text)`
+ * contract and keeps the provider dependency-free (just `fetch`).
+ *
+ * xAI also exposes a WebSocket streaming variant (`wss://api.x.ai/v1/tts`) for
+ * lower TTFB; this provider uses the REST bytes path which already satisfies
+ * the pipeline's streaming contract — the same trade-off `SonioxTTS` makes. A
+ * WebSocket variant would be a separate `xai_ws_tts` provider (precedent:
+ * `elevenlabs_ws`).
+ *
+ * Credential: `XAI_API_KEY`. REST auth is an `Authorization: Bearer ` header.
+ *
+ * **Telephony optimization** — the constructor default `codec="pcm"` @
+ * `sampleRate=16000` is correct for web playback and 16 kHz pipelines. For real
+ * phone calls use the carrier-specific factories:
+ *
+ * - {@link XaiTTS.forTwilio} emits `mulaw` @ 8 kHz — Twilio's exact wire codec —
+ * so the pipeline skips resampling AND PCM → μ-law encoding (bit-clean
+ * passthrough). xAI supports G.711 μ-law @ 8 kHz natively.
+ * - {@link XaiTTS.forTelnyx} emits `pcm` @ 16 kHz (the SDK's Telnyx pipeline rate).
+ */
+
+import { getLogger } from '../logger';
+
+/** xAI one-shot TTS REST endpoint. */
+export const XAI_TTS_REST_URL = 'https://api.x.ai/v1/tts';
+/** xAI custom-voice creation REST endpoint. */
+export const XAI_CUSTOM_VOICES_URL = 'https://api.x.ai/v1/custom-voices';
+// WebSocket streaming endpoint (documented for reference; not used by this
+// HTTP-bytes adapter).
+export const XAI_TTS_WS_URL = 'wss://api.x.ai/v1/tts';
+
+/** xAI default voice — the built-in `eve` (case-insensitive). */
+const XAI_DEFAULT_VOICE = 'eve';
+
+/**
+ * Output codecs accepted by the xAI `output_format.codec` field. `MULAW` /
+ * `ALAW` @ 8 kHz are the carrier-native G.711 codecs (telephony passthrough);
+ * `PCM` targets 16 kHz pipelines; `MP3` / `WAV` target web/dashboard use.
+ */
+export const XaiTTSCodec = {
+ PCM: 'pcm',
+ MULAW: 'mulaw',
+ ALAW: 'alaw',
+ WAV: 'wav',
+ MP3: 'mp3',
+} as const;
+export type XaiTTSCodec = (typeof XaiTTSCodec)[keyof typeof XaiTTSCodec];
+
+/** Sample rates (Hz) accepted by the xAI `output_format.sample_rate` field. */
+export const XaiTTSSampleRate = {
+ HZ_8000: 8000,
+ HZ_16000: 16000,
+ HZ_22050: 22050,
+ HZ_24000: 24000,
+ HZ_44100: 44100,
+ HZ_48000: 48000,
+} as const;
+export type XaiTTSSampleRate = (typeof XaiTTSSampleRate)[keyof typeof XaiTTSSampleRate];
+
+/** Max characters accepted per one-shot TTS request. */
+const XAI_TTS_MAX_CHARS = 15000;
+
+/** Constructor options for {@link XaiTTS}. */
+export interface XaiTTSOptions {
+ /** Voice id (built-in or custom, case-insensitive). Defaults to `"eve"`. */
+ voice?: string;
+ /**
+ * BCP-47 language tag, e.g. `"en"`, `"it"`, `"pt-BR"`, or `"auto"` for
+ * autodetection. Required by xAI; defaults to `"auto"`.
+ */
+ language?: string;
+ /**
+ * Output codec. Default `pcm` (raw linear PCM16). Set `mulaw` with
+ * `sampleRate=8000` to emit Twilio/Telnyx wire-native μ-law directly — the
+ * pipeline then skips resampling and PCM → μ-law encoding (see
+ * {@link XaiTTS.forTwilio}).
+ */
+ codec?: XaiTTSCodec | string;
+ /** Output sample rate in Hz. Defaults to 16000. */
+ sampleRate?: XaiTTSSampleRate | number;
+ /** Bit rate (bits/sec) — MP3 only. */
+ bitRate?: number;
+ /** Speaking-rate multiplier in `[0.7, 1.5]` (default `1.0`). Omitted when unset. */
+ speed?: number;
+ /** Streaming latency optimization level `0 | 1 | 2` (xAI default `0`). */
+ optimizeStreamingLatency?: number;
+ /** Normalise numbers/abbreviations/symbols to spoken form before synthesis. Default `false`. */
+ textNormalization?: boolean;
+ /** Override the REST endpoint (e.g. for tests). */
+ baseUrl?: string;
+}
+
+/** xAI TTS adapter backed by the HTTP one-shot `/tts` bytes endpoint. */
+export class XaiTTS {
+ /** Stable pricing/dashboard key — read by stream-handler/metrics. */
+ static readonly providerKey = 'xai_tts';
+
+ private readonly apiKey: string;
+ private readonly voice: string;
+ private readonly language: string;
+ private readonly codec: string;
+ private readonly sampleRate: number;
+ private readonly bitRate?: number;
+ private readonly speed?: number;
+ private readonly optimizeStreamingLatency?: number;
+ private readonly textNormalization: boolean;
+ private readonly baseUrl: string;
+
+ constructor(apiKey: string, opts: XaiTTSOptions = {}) {
+ if (!apiKey) {
+ throw new Error('xAI TTS: apiKey is required');
+ }
+ this.apiKey = apiKey;
+ this.voice = opts.voice ?? XAI_DEFAULT_VOICE;
+ this.language = opts.language ?? 'auto';
+ this.codec = opts.codec ?? XaiTTSCodec.PCM;
+ this.sampleRate = opts.sampleRate ?? XaiTTSSampleRate.HZ_16000;
+ this.bitRate = opts.bitRate;
+ this.speed = opts.speed;
+ this.optimizeStreamingLatency = opts.optimizeStreamingLatency;
+ this.textNormalization = opts.textNormalization ?? false;
+ this.baseUrl = opts.baseUrl ?? XAI_TTS_REST_URL;
+ }
+
+ /**
+ * Declare the audio format this adapter emits, so the pipeline sender can
+ * derive the correct resample ratio (or skip it for μ-law passthrough)
+ * instead of assuming a fixed 16 kHz source.
+ *
+ * - `mulaw` → `{ encoding: 'mulaw', sampleRate }` (carrier-native).
+ * - `alaw` → `{ encoding: 'alaw', sampleRate }`.
+ * - everything else → `{ encoding: 'pcm_s16le', sampleRate }`.
+ */
+ sourceAudioFormat(): {
+ encoding: 'pcm_s16le' | 'mulaw' | 'alaw';
+ sampleRate: number;
+ } {
+ if (this.codec === XaiTTSCodec.MULAW) {
+ return { encoding: 'mulaw', sampleRate: this.sampleRate };
+ }
+ if (this.codec === XaiTTSCodec.ALAW) {
+ return { encoding: 'alaw', sampleRate: this.sampleRate };
+ }
+ return { encoding: 'pcm_s16le', sampleRate: this.sampleRate };
+ }
+
+ /**
+ * Construct an instance pre-configured for Twilio Media Streams.
+ *
+ * Emits `mulaw` @ 8 kHz — exactly Twilio's wire codec — so the pipeline takes
+ * the passthrough path: zero resampling, zero PCM → μ-law encoding, bit-clean
+ * audio. xAI supports μ-law @ 8 kHz natively (true carrier-native synthesis).
+ */
+ static forTwilio(
+ apiKey: string,
+ options: Omit = {},
+ ): XaiTTS {
+ return new XaiTTS(apiKey, {
+ ...options,
+ codec: XaiTTSCodec.MULAW,
+ sampleRate: XaiTTSSampleRate.HZ_8000,
+ });
+ }
+
+ /**
+ * Construct an instance pre-configured for Telnyx bidirectional media.
+ *
+ * Emits `pcm` @ 16 kHz — the SDK's Telnyx pipeline rate — so the audio flows
+ * through the standard PCM16 path with no codec transcode.
+ */
+ static forTelnyx(
+ apiKey: string,
+ options: Omit = {},
+ ): XaiTTS {
+ return new XaiTTS(apiKey, {
+ ...options,
+ codec: XaiTTSCodec.PCM,
+ sampleRate: XaiTTSSampleRate.HZ_16000,
+ });
+ }
+
+ /** Build the JSON payload for the xAI `/tts` endpoint. */
+ private buildPayload(text: string): Record {
+ const outputFormat: Record = {
+ codec: this.codec,
+ sample_rate: this.sampleRate,
+ };
+ if (this.bitRate !== undefined) outputFormat.bit_rate = this.bitRate;
+ const payload: Record = {
+ text,
+ voice_id: this.voice,
+ language: this.language,
+ output_format: outputFormat,
+ };
+ if (this.speed !== undefined) payload.speed = this.speed;
+ if (this.optimizeStreamingLatency !== undefined) {
+ payload.optimize_streaming_latency = this.optimizeStreamingLatency;
+ }
+ if (this.textNormalization) payload.text_normalization = true;
+ return payload;
+ }
+
+ /** Synthesize text and return the concatenated audio buffer. */
+ async synthesize(text: string): Promise {
+ const chunks: Buffer[] = [];
+ for await (const chunk of this.synthesizeStream(text)) {
+ chunks.push(chunk);
+ }
+ return Buffer.concat(chunks);
+ }
+
+ /**
+ * Synthesize text and yield raw audio chunks as they arrive from xAI. With
+ * the default `codec="pcm"` these are PCM16 bytes at the configured
+ * `sampleRate`; with `mulaw` (see {@link XaiTTS.forTwilio}) they are
+ * carrier-native G.711 μ-law bytes.
+ */
+ async *synthesizeStream(text: string): AsyncGenerator {
+ if (text.length > XAI_TTS_MAX_CHARS) {
+ // xAI rejects >15,000 chars with HTTP 400. We do NOT silently truncate —
+ // send as-is and let the API error surface, but warn so the cause is clear.
+ getLogger().warn(
+ `xAI TTS: text length ${text.length} exceeds the ${XAI_TTS_MAX_CHARS}-char ` +
+ 'per-request limit; the API will reject it. Split long text upstream.',
+ );
+ }
+ const response = await fetch(this.baseUrl, {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${this.apiKey}`,
+ 'Content-Type': 'application/json',
+ },
+ body: JSON.stringify(this.buildPayload(text)),
+ signal: AbortSignal.timeout(30_000),
+ });
+
+ if (!response.ok) {
+ const body = await response.text();
+ throw new Error(`xAI TTS error ${response.status}: ${body}`);
+ }
+ if (!response.body) {
+ throw new Error('xAI TTS: no response body');
+ }
+
+ const reader = response.body.getReader();
+ try {
+ while (true) {
+ const { done, value } = await reader.read();
+ if (done) break;
+ if (value && value.length > 0) {
+ yield Buffer.from(value);
+ }
+ }
+ } finally {
+ if (typeof reader.cancel === 'function') await reader.cancel().catch(() => {});
+ reader.releaseLock();
+ }
+ }
+}
+
+/** Options for {@link xaiCreateCustomVoice}. */
+export interface XaiCreateCustomVoiceOptions {
+ /** Display name for the cloned voice. */
+ readonly name: string;
+ /** BCP-47 language of the reference clip (e.g. `"en"`). */
+ readonly language: string;
+ /** Reference audio clip bytes (WAV, ≤120 s). */
+ readonly audio: Buffer | Uint8Array;
+ /** API key. Falls back to `XAI_API_KEY` when omitted. */
+ readonly apiKey?: string;
+ /** Filename for the multipart `file` part. Defaults to `reference.wav`. */
+ readonly filename?: string;
+ /** Override the REST endpoint (e.g. for tests). */
+ readonly baseUrl?: string;
+}
+
+interface XaiCustomVoiceResponse {
+ voice_id?: string;
+}
+
+/**
+ * Clone a custom voice from a short reference clip via xAI's Custom Voices API
+ * (`POST /v1/custom-voices`). Returns the resulting `voice_id`, usable as the
+ * `voice` on {@link XaiTTS} and the xAI Voice Agent engine.
+ *
+ * **Beta.** Multipart order matters: the scalar `name` / `language` fields are
+ * appended FIRST and the `file` part (audio/wav, ≤120 s clip) LAST.
+ */
+export async function xaiCreateCustomVoice(
+ options: XaiCreateCustomVoiceOptions,
+): Promise {
+ const apiKey = options.apiKey ?? process.env.XAI_API_KEY;
+ if (!apiKey) {
+ throw new Error(
+ "xAI custom voice requires an apiKey. Pass { apiKey: '...' } or set XAI_API_KEY.",
+ );
+ }
+
+ const bytes =
+ options.audio instanceof Buffer ? options.audio : Buffer.from(options.audio);
+ const arrayBuffer = bytes.buffer.slice(
+ bytes.byteOffset,
+ bytes.byteOffset + bytes.byteLength,
+ ) as ArrayBuffer;
+
+ const form = new FormData();
+ // Scalar fields FIRST, file LAST.
+ form.append('name', options.name);
+ form.append('language', options.language);
+ form.append(
+ 'file',
+ new Blob([arrayBuffer], { type: 'audio/wav' }),
+ options.filename ?? 'reference.wav',
+ );
+
+ const response = await fetch(options.baseUrl ?? XAI_CUSTOM_VOICES_URL, {
+ method: 'POST',
+ headers: { Authorization: `Bearer ${apiKey}` },
+ body: form,
+ signal: AbortSignal.timeout(120_000),
+ });
+ if (!response.ok) {
+ const body = await response.text();
+ throw new Error(`xAI custom voice error ${response.status}: ${body}`);
+ }
+
+ const data = (await response.json()) as XaiCustomVoiceResponse;
+ if (!data.voice_id) {
+ throw new Error('xAI custom voice: response did not include a voice_id');
+ }
+ return data.voice_id;
+}
diff --git a/libraries/typescript/src/server.ts b/libraries/typescript/src/server.ts
index 41f82dfa..9c4b4eda 100644
--- a/libraries/typescript/src/server.ts
+++ b/libraries/typescript/src/server.ts
@@ -16,6 +16,7 @@ import { ElevenLabsConvAIAdapter } from './providers/elevenlabs-convai';
import { GeminiLiveAdapter } from './providers/gemini-live';
import { GeminiCascadeAdapter } from './providers/gemini-cascade';
import { InworldRealtimeAdapter } from './providers/inworld-realtime';
+import { XaiRealtimeAdapter } from './providers/xai-realtime';
import { PlivoAdapter, dropPlivoVoicemail, plivoInboundCustomParams, parsePlivoExtraHeaders } from './providers/plivo-adapter';
import { TwilioAdapter } from './providers/twilio-adapter';
import { PlivoBridge, classifyPlivoAmd, validatePlivoSignature } from './telephony/plivo';
@@ -739,6 +740,48 @@ export function buildAIAdapter(config: LocalConfig, agent: AgentOptions, resolve
tools,
});
}
+ // xAI Realtime (Grok Voice Agent) mode: construct XaiRealtimeAdapter from the
+ // engine marker. xAI's Voice Agent API is OpenAI-Realtime-GA-compatible, so
+ // the adapter subclasses OpenAIRealtime2Adapter and advertises the SAME tool
+ // list (agent tools + transfer_call/end_call/handoff/use_skill) built above.
+ if (agent.provider === 'xai_realtime') {
+ if (!engine || engine.kind !== 'xai_realtime') {
+ throw new Error(
+ "xAI Realtime mode requires `agent.engine = new XaiRealtime({...})`.",
+ );
+ }
+ const agentOpts = agent as {
+ realtimeTurnDetection?: import('./types').RealtimeTurnDetection;
+ openaiRealtimeGateResponseOnTranscript?: boolean;
+ };
+ return new XaiRealtimeAdapter(engine.apiKey, {
+ ...(agent.model ?? engine.model ? { model: agent.model ?? engine.model } : {}),
+ ...(agent.voice ?? engine.voice ? { voice: agent.voice ?? engine.voice } : {}),
+ ...(engine.baseUrl ? { baseUrl: engine.baseUrl } : {}),
+ ...(engine.reasoningEffort !== undefined ? { reasoningEffort: engine.reasoningEffort } : {}),
+ ...(engine.languageHint !== undefined ? { languageHint: engine.languageHint } : {}),
+ ...(engine.keyterms !== undefined ? { keyterms: engine.keyterms } : {}),
+ ...(engine.speed !== undefined ? { speed: engine.speed } : {}),
+ ...(engine.vadThreshold !== undefined ? { vadThreshold: engine.vadThreshold } : {}),
+ ...(engine.silenceDurationMs !== undefined ? { silenceDurationMs: engine.silenceDurationMs } : {}),
+ ...(engine.prefixPaddingMs !== undefined ? { prefixPaddingMs: engine.prefixPaddingMs } : {}),
+ ...(engine.idleTimeoutMs !== undefined ? { idleTimeoutMs: engine.idleTimeoutMs } : {}),
+ ...(engine.replace !== undefined ? { replace: engine.replace } : {}),
+ ...(engine.resumption !== undefined ? { resumption: engine.resumption } : {}),
+ ...(engine.serverTools !== undefined ? { serverTools: engine.serverTools } : {}),
+ ...((agentOpts.realtimeTurnDetection ?? engine.turnDetection) !== undefined
+ ? { turnDetection: agentOpts.realtimeTurnDetection ?? engine.turnDetection }
+ : {}),
+ ...((agentOpts.openaiRealtimeGateResponseOnTranscript ?? engine.gateResponseOnTranscript) !== undefined
+ ? {
+ gateResponseOnTranscript:
+ agentOpts.openaiRealtimeGateResponseOnTranscript ?? engine.gateResponseOnTranscript,
+ }
+ : {}),
+ instructions: resolvedPrompt ?? agent.systemPrompt,
+ tools,
+ });
+ }
const isOpenAIEngine = engine && (engine.kind === 'openai_realtime' || engine.kind === 'openai_realtime_2');
const openaiKey = isOpenAIEngine ? engine.apiKey : (config.openaiKey ?? '');
// Forward optional engine-level Realtime knobs so the high-level
diff --git a/libraries/typescript/src/stt/xai.ts b/libraries/typescript/src/stt/xai.ts
new file mode 100644
index 00000000..dfd6779a
--- /dev/null
+++ b/libraries/typescript/src/stt/xai.ts
@@ -0,0 +1,45 @@
+/** xAI streaming STT for Patter pipeline mode. */
+import { XaiSTT as _XaiSTT, type XaiSTTEncoding } from "../providers/xai-stt";
+
+/** Constructor options for the xAI `STT` adapter. */
+export interface XaiSTTOptions {
+ /** API key. Falls back to XAI_API_KEY env var when omitted. */
+ apiKey?: string;
+ encoding?: XaiSTTEncoding | string;
+ sampleRate?: number;
+ interimResults?: boolean;
+ language?: string;
+ endpointingMs?: number;
+ smartTurn?: number;
+ smartTurnTimeoutMs?: number;
+ keyterms?: readonly string[];
+ diarize?: boolean;
+ fillerWords?: boolean;
+ baseUrl?: string;
+}
+
+/**
+ * xAI streaming STT (Beta).
+ *
+ * @example
+ * ```ts
+ * import * as xai from "getpatter/stt/xai";
+ * const stt = new xai.STT(); // reads XAI_API_KEY
+ * const stt = new xai.STT({ apiKey: "..." });
+ * ```
+ */
+export class STT extends _XaiSTT {
+ static readonly providerKey = "xai";
+ constructor(opts: XaiSTTOptions = {}) {
+ const key = opts.apiKey ?? process.env.XAI_API_KEY;
+ if (!key) {
+ throw new Error(
+ "xAI STT requires an apiKey. Pass { apiKey: '...' } or " +
+ "set XAI_API_KEY in the environment.",
+ );
+ }
+ const { apiKey: _ignored, ...rest } = opts;
+ void _ignored;
+ super(key, rest);
+ }
+}
diff --git a/libraries/typescript/src/telemetry/stack.ts b/libraries/typescript/src/telemetry/stack.ts
index 15b3289a..b48e042d 100644
--- a/libraries/typescript/src/telemetry/stack.ts
+++ b/libraries/typescript/src/telemetry/stack.ts
@@ -35,6 +35,7 @@ export const STACK_VENDORS: ReadonlySet = new Set([
'rime',
'inworld',
'sarvam',
+ 'xai',
'telnyx',
'other',
]);
@@ -47,6 +48,8 @@ const VENDOR_ALIASES: Record = {
openai_transcribe: 'openai',
elevenlabs_ws: 'elevenlabs',
soniox_tts: 'soniox',
+ xai_tts: 'xai',
+ xai_realtime: 'xai',
telnyx_stt: 'telnyx',
telnyx_tts: 'telnyx',
};
diff --git a/libraries/typescript/src/tts/xai.ts b/libraries/typescript/src/tts/xai.ts
new file mode 100644
index 00000000..aae2293b
--- /dev/null
+++ b/libraries/typescript/src/tts/xai.ts
@@ -0,0 +1,93 @@
+/** xAI TTS for Patter pipeline mode. */
+import {
+ XaiTTS as _XaiTTS,
+ XaiTTSCodec,
+ type XaiTTSSampleRate,
+} from "../providers/xai-tts";
+
+/** Constructor options for the xAI `TTS` adapter. */
+export interface XaiTTSOptions {
+ /** API key. Falls back to XAI_API_KEY env var when omitted. */
+ apiKey?: string;
+ voice?: string;
+ /** BCP-47 language tag, or `"auto"` (default). */
+ language?: string;
+ /** Output codec. Default `pcm`; `mulaw` @ 8 kHz = carrier-native. */
+ codec?: XaiTTSCodec | string;
+ sampleRate?: XaiTTSSampleRate | number;
+ bitRate?: number;
+ speed?: number;
+ optimizeStreamingLatency?: number;
+ textNormalization?: boolean;
+ baseUrl?: string;
+}
+
+/** Options for the carrier-specific factories — same as the constructor minus `sampleRate`/`codec`. */
+export type XaiCarrierOptions = Omit;
+
+function resolveApiKey(apiKey: string | undefined): string {
+ const key = apiKey ?? process.env.XAI_API_KEY;
+ if (!key) {
+ throw new Error(
+ "xAI TTS requires an apiKey. Pass { apiKey: '...' } or " +
+ "set XAI_API_KEY in the environment.",
+ );
+ }
+ return key;
+}
+
+/**
+ * xAI TTS (default voice `eve`) — Beta.
+ *
+ * @example
+ * ```ts
+ * import * as xai from "getpatter/tts/xai";
+ * const tts = new xai.TTS(); // reads XAI_API_KEY
+ * const tts = new xai.TTS({ apiKey: "...", voice: "leo", language: "en" });
+ * ```
+ *
+ * **Telephony** — use {@link TTS.forTwilio} (μ-law @ 8 kHz native) or
+ * {@link TTS.forTelnyx} (pcm @ 16 kHz) on phone calls.
+ */
+export class TTS extends _XaiTTS {
+ static readonly providerKey = "xai_tts";
+ constructor(opts: XaiTTSOptions = {}) {
+ const key = resolveApiKey(opts.apiKey);
+ const { apiKey: _ignored, ...rest } = opts;
+ void _ignored;
+ super(key, rest);
+ }
+
+ /** Pipeline TTS pre-configured for Twilio Media Streams (μ-law @ 8 kHz native passthrough). */
+ static override forTwilio(opts?: XaiCarrierOptions): TTS;
+ // Parent-compatible overload — accepts the legacy positional form too.
+ static override forTwilio(apiKey: string, options?: XaiCarrierOptions): TTS;
+ static override forTwilio(
+ arg1?: string | XaiCarrierOptions,
+ arg2?: XaiCarrierOptions,
+ ): TTS {
+ const opts: XaiCarrierOptions =
+ typeof arg1 === "string" ? { apiKey: arg1, ...(arg2 ?? {}) } : (arg1 ?? {});
+ return new TTS({
+ ...opts,
+ codec: XaiTTSCodec.MULAW,
+ sampleRate: 8000,
+ });
+ }
+
+ /** Pipeline TTS pre-configured for Telnyx (pcm @ 16 kHz). */
+ static override forTelnyx(opts?: XaiCarrierOptions): TTS;
+ static override forTelnyx(apiKey: string, options?: XaiCarrierOptions): TTS;
+ static override forTelnyx(
+ arg1?: string | XaiCarrierOptions,
+ arg2?: XaiCarrierOptions,
+ ): TTS {
+ const opts: XaiCarrierOptions =
+ typeof arg1 === "string" ? { apiKey: arg1, ...(arg2 ?? {}) } : (arg1 ?? {});
+ return new TTS({
+ ...opts,
+ codec: XaiTTSCodec.PCM,
+ sampleRate: 16000,
+ });
+ }
+}
diff --git a/libraries/typescript/src/types.ts b/libraries/typescript/src/types.ts
index d4772607..b8b23207 100644
--- a/libraries/typescript/src/types.ts
+++ b/libraries/typescript/src/types.ts
@@ -18,6 +18,7 @@ import type { ConvAI } from "./engines/elevenlabs";
import type { GeminiLive } from "./engines/gemini";
import type { GeminiCascade } from "./engines/gemini-cascade";
import type { InworldRealtime } from "./engines/inworld";
+import type { XaiRealtime } from "./engines/xai";
import type { CloudflareTunnel, Static as StaticTunnel } from "./tunnels";
import type { Tool as ToolInstance } from "./public-api";
import type { STTAdapter, TTSAdapter } from "./provider-factory";
@@ -880,13 +881,13 @@ export interface AgentOptions {
* matching mode (``openai_realtime`` or ``elevenlabs_convai``). When absent,
* pipeline mode is selected if ``stt`` and ``tts`` are provided.
*/
- readonly engine?: Realtime | Realtime2 | ConvAI | GeminiLive | GeminiCascade | InworldRealtime;
+ readonly engine?: Realtime | Realtime2 | ConvAI | GeminiLive | GeminiCascade | InworldRealtime | XaiRealtime;
/**
* Provider mode. Normally derived from ``engine`` / ``stt`` + ``tts``. Pass
* ``'pipeline'`` explicitly when building a pipeline-mode agent without
* an engine instance.
*/
- readonly provider?: 'openai_realtime' | 'elevenlabs_convai' | 'gemini_live' | 'gemini_cascade' | 'inworld_realtime' | 'pipeline';
+ readonly provider?: 'openai_realtime' | 'elevenlabs_convai' | 'gemini_live' | 'gemini_cascade' | 'inworld_realtime' | 'xai_realtime' | 'pipeline';
/** Pre-instantiated STT adapter (e.g. ``new DeepgramSTT({ apiKey })``). */
readonly stt?: STTAdapter;
/** Pre-instantiated TTS adapter (e.g. ``new ElevenLabsTTS({ apiKey })``). */
diff --git a/libraries/typescript/tests/unit/xai-stt.test.ts b/libraries/typescript/tests/unit/xai-stt.test.ts
new file mode 100644
index 00000000..1e08b3a2
--- /dev/null
+++ b/libraries/typescript/tests/unit/xai-stt.test.ts
@@ -0,0 +1,302 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+
+/**
+ * xAI STT — streaming WebSocket adapter + batch REST helper.
+ *
+ * The `ws` module is mocked so `connect()` opens no real socket; the batch
+ * helper's `fetch` boundary is stubbed. Everything inward (URL/query assembly,
+ * transcript.created gating, is_final/speech_final mapping, finalize, multipart
+ * ordering, response parsing) runs for real.
+ */
+
+vi.mock('ws', () => {
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
+ const EventEmitter = require('events');
+ class MockWebSocket extends EventEmitter {
+ static OPEN = 1;
+ static CLOSED = 3;
+ static instances: MockWebSocket[] = [];
+ readyState = MockWebSocket.OPEN;
+ sent: Array = [];
+ url: string;
+ options: { headers?: Record };
+ constructor(url: string, options?: { headers?: Record }) {
+ super();
+ this.url = url;
+ this.options = options ?? {};
+ MockWebSocket.instances.push(this);
+ }
+ send(data: string | Buffer): void {
+ this.sent.push(data);
+ }
+ close(): void {
+ this.readyState = MockWebSocket.CLOSED;
+ }
+ }
+ return { default: MockWebSocket };
+});
+
+import { XaiSTT, xaiTranscribe, XAI_STT_WS_URL } from '../../src/providers/xai-stt';
+
+const CLOSED = 3;
+
+type MockWS = {
+ readyState: number;
+ sent: Array;
+ url: string;
+ options: { headers?: Record };
+ emit: (event: string, ...args: unknown[]) => void;
+};
+
+async function instances(): Promise {
+ const mod = (await import('ws')).default as unknown as { instances: MockWS[] };
+ return mod.instances;
+}
+
+async function latest(): Promise {
+ const all = await instances();
+ return all[all.length - 1];
+}
+
+/** Connect an adapter and resolve once the ready signal is emitted. */
+async function connect(stt: XaiSTT): Promise {
+ const p = stt.connect();
+ const ws = await latest();
+ ws.emit('message', JSON.stringify({ type: 'transcript.created' }));
+ await p;
+ return ws;
+}
+
+describe('[unit] XaiSTT streaming', () => {
+ beforeEach(async () => {
+ (await instances()).length = 0;
+ });
+
+ it('builds the query-parameterised URL with a Bearer header and repeated keyterm', async () => {
+ const stt = new XaiSTT('xai-test-key', {
+ encoding: 'pcm',
+ sampleRate: 16000,
+ language: 'it',
+ endpointingMs: 20,
+ smartTurn: 0.5,
+ smartTurnTimeoutMs: 2000,
+ keyterms: ['Grok', 'xAI'],
+ diarize: true,
+ fillerWords: true,
+ });
+ const ws = await connect(stt);
+
+ expect(ws.url.startsWith(`${XAI_STT_WS_URL}?`)).toBe(true);
+ const url = new URL(ws.url);
+ expect(url.searchParams.get('encoding')).toBe('pcm');
+ expect(url.searchParams.get('sample_rate')).toBe('16000');
+ expect(url.searchParams.get('interim_results')).toBe('true');
+ expect(url.searchParams.get('language')).toBe('it');
+ expect(url.searchParams.get('endpointing')).toBe('20');
+ expect(url.searchParams.get('smart_turn')).toBe('0.5');
+ expect(url.searchParams.get('smart_turn_timeout')).toBe('2000');
+ expect(url.searchParams.get('diarize')).toBe('true');
+ expect(url.searchParams.get('filler_words')).toBe('true');
+ // `keyterm` is REPEATED, one param per term.
+ expect(url.searchParams.getAll('keyterm')).toEqual(['Grok', 'xAI']);
+ expect(ws.options.headers?.Authorization).toBe('Bearer xai-test-key');
+ stt.close();
+ });
+
+ it('defaults to pcm @ 16000 with interim_results on (pipeline PCM16 feed)', async () => {
+ const stt = new XaiSTT('xai-test-key');
+ const ws = await connect(stt);
+ const url = new URL(ws.url);
+ expect(url.searchParams.get('encoding')).toBe('pcm');
+ expect(url.searchParams.get('sample_rate')).toBe('16000');
+ expect(url.searchParams.get('interim_results')).toBe('true');
+ // No keyterms / diarize / language by default.
+ expect(url.searchParams.getAll('keyterm')).toEqual([]);
+ expect(url.searchParams.get('diarize')).toBeNull();
+ expect(url.searchParams.get('language')).toBeNull();
+ stt.close();
+ });
+
+ it('does not resolve connect() until transcript.created arrives', async () => {
+ const stt = new XaiSTT('xai-test-key');
+ let resolved = false;
+ const p = stt.connect().then(() => {
+ resolved = true;
+ });
+ const ws = await latest();
+ // Flush microtasks — still pending because no transcript.created yet.
+ await Promise.resolve();
+ expect(resolved).toBe(false);
+ ws.emit('message', JSON.stringify({ type: 'transcript.created' }));
+ await p;
+ expect(resolved).toBe(true);
+ stt.close();
+ });
+
+ it('maps interim / chunk-final / utterance-final transcripts like Deepgram', async () => {
+ const stt = new XaiSTT('xai-test-key');
+ const ws = await connect(stt);
+ const received: Array> = [];
+ stt.onTranscript((t) => {
+ received.push(t as unknown as Record);
+ });
+
+ // Interim: is_final=false → not final.
+ ws.emit('message', JSON.stringify({ type: 'transcript.partial', text: 'hel', is_final: false, speech_final: false }));
+ // Chunk final: is_final=true, speech_final=false.
+ ws.emit('message', JSON.stringify({ type: 'transcript.partial', text: 'hello there', is_final: true, speech_final: false }));
+ // Utterance final: both true.
+ ws.emit('message', JSON.stringify({ type: 'transcript.partial', text: 'hello there world', is_final: true, speech_final: true }));
+
+ expect(received).toHaveLength(3);
+ expect(received[0]).toMatchObject({ text: 'hel', isFinal: false, speechFinal: false, eventType: 'Results' });
+ expect(received[1]).toMatchObject({ text: 'hello there', isFinal: true, speechFinal: false });
+ expect(received[2]).toMatchObject({ text: 'hello there world', isFinal: true, speechFinal: true });
+ stt.close();
+ });
+
+ it('maps xAI word entries to Patter word shape (text→word, speaker preserved)', async () => {
+ const stt = new XaiSTT('xai-test-key', { diarize: true });
+ const ws = await connect(stt);
+ const received: Array<{ words?: ReadonlyArray<{ word?: string; speaker?: number }> }> = [];
+ stt.onTranscript((t) => received.push(t as never));
+
+ ws.emit('message', JSON.stringify({
+ type: 'transcript.partial',
+ text: 'hi bob',
+ is_final: true,
+ speech_final: true,
+ words: [
+ { text: 'hi', start: 0.1, end: 0.2, speaker: 0 },
+ { text: 'bob', start: 0.3, end: 0.5, speaker: 1 },
+ ],
+ }));
+
+ expect(received[0].words).toEqual([
+ { word: 'hi', start: 0.1, end: 0.2, speaker: 0 },
+ { word: 'bob', start: 0.3, end: 0.5, speaker: 1 },
+ ]);
+ stt.close();
+ });
+
+ it('finalize() sends {type:"finalize"} and stamps fromFinalize on the next final', async () => {
+ const stt = new XaiSTT('xai-test-key');
+ const ws = await connect(stt);
+ const received: Array<{ fromFinalize?: boolean }> = [];
+ stt.onTranscript((t) => received.push(t as never));
+
+ stt.finalize();
+ const finalizeFrame = ws.sent.find(
+ (s) => typeof s === 'string' && s.includes('finalize'),
+ ) as string;
+ expect(JSON.parse(finalizeFrame)).toEqual({ type: 'finalize' });
+
+ ws.emit('message', JSON.stringify({ type: 'transcript.partial', text: 'done', is_final: true, speech_final: true }));
+ expect(received[0].fromFinalize).toBe(true);
+ stt.close();
+ });
+
+ it('close() sends {type:"audio.done"} best-effort before closing', async () => {
+ const stt = new XaiSTT('xai-test-key');
+ const ws = await connect(stt);
+ stt.close();
+ const doneFrame = ws.sent.find(
+ (s) => typeof s === 'string' && s.includes('audio.done'),
+ ) as string;
+ expect(JSON.parse(doneFrame)).toEqual({ type: 'audio.done' });
+ expect(ws.readyState).toBe(CLOSED);
+ });
+
+ it('sends raw binary audio frames (no base64) and drops empty frames', async () => {
+ const stt = new XaiSTT('xai-test-key');
+ const ws = await connect(stt);
+ ws.sent.length = 0;
+ stt.sendAudio(Buffer.from([1, 2, 3, 4]));
+ stt.sendAudio(Buffer.alloc(0)); // dropped
+ expect(ws.sent).toHaveLength(1);
+ expect(Buffer.isBuffer(ws.sent[0])).toBe(true);
+ expect(ws.sent[0]).toEqual(Buffer.from([1, 2, 3, 4]));
+ stt.close();
+ });
+
+ it('clone() produces a fresh adapter with the same construction args', () => {
+ const stt = new XaiSTT('xai-test-key', { language: 'en' });
+ const twin = stt.clone();
+ expect(twin).not.toBe(stt);
+ expect(twin).toBeInstanceOf(XaiSTT);
+ });
+});
+
+describe('[mocked] xaiTranscribe batch REST helper', () => {
+ const ORIGINAL_FETCH = global.fetch;
+ let lastUrl: string | null = null;
+ let lastHeaders: Record | null = null;
+ let lastBody: FormData | null = null;
+
+ beforeEach(() => {
+ lastUrl = null;
+ lastHeaders = null;
+ lastBody = null;
+ global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+ lastUrl = typeof input === 'string' ? input : input.toString();
+ lastHeaders = init?.headers as Record;
+ lastBody = init?.body as FormData;
+ return new Response(
+ JSON.stringify({
+ text: 'The balance is $167.',
+ language: 'English',
+ duration: 3.45,
+ words: [{ text: 'The', start: 0.24, end: 0.48 }],
+ }),
+ { status: 200, headers: { 'Content-Type': 'application/json' } },
+ );
+ }) as typeof global.fetch;
+ });
+
+ afterEach(() => {
+ global.fetch = ORIGINAL_FETCH;
+ });
+
+ it('posts multipart with scalar fields first and the file LAST', async () => {
+ await xaiTranscribe({
+ audio: Buffer.from([0, 1, 2, 3]),
+ apiKey: 'xai-test-key',
+ language: 'en',
+ format: true,
+ diarize: true,
+ keyterms: ['Grok'],
+ filename: 'clip.wav',
+ });
+
+ expect(lastUrl).toBe('https://api.x.ai/v1/stt');
+ expect(lastHeaders?.Authorization).toBe('Bearer xai-test-key');
+
+ const keys = [...(lastBody as FormData).keys()];
+ // Scalars appear before the file; the file is the final field.
+ expect(keys[keys.length - 1]).toBe('file');
+ expect(keys.slice(0, -1)).toContain('language');
+ expect(keys.slice(0, -1)).toContain('format');
+ expect(keys.slice(0, -1)).toContain('keyterm');
+ const file = (lastBody as FormData).get('file');
+ expect(file).toBeInstanceOf(Blob);
+ });
+
+ it('parses the response into text/language/duration/words', async () => {
+ const result = await xaiTranscribe({ audio: Buffer.from([0]), apiKey: 'xai-test-key' });
+ expect(result.text).toBe('The balance is $167.');
+ expect(result.language).toBe('English');
+ expect(result.duration).toBe(3.45);
+ expect(result.words).toEqual([{ text: 'The', start: 0.24, end: 0.48 }]);
+ });
+
+ it('sends `url` instead of a file when audio bytes are absent', async () => {
+ await xaiTranscribe({ url: 'https://example.com/a.wav', apiKey: 'xai-test-key' });
+ const keys = [...(lastBody as FormData).keys()];
+ expect(keys).toContain('url');
+ expect(keys).not.toContain('file');
+ });
+
+ it('throws when neither audio nor url is provided', async () => {
+ await expect(xaiTranscribe({ apiKey: 'xai-test-key' })).rejects.toThrow(/audio.*or a `url`/);
+ });
+});
diff --git a/libraries/typescript/tests/unit/xai-tts.mocked.test.ts b/libraries/typescript/tests/unit/xai-tts.mocked.test.ts
new file mode 100644
index 00000000..ccdb8241
--- /dev/null
+++ b/libraries/typescript/tests/unit/xai-tts.mocked.test.ts
@@ -0,0 +1,203 @@
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { XaiTTS, xaiCreateCustomVoice, XaiTTSCodec } from '../../src/providers/xai-tts';
+import { TTS as XaiPipelineTTS } from '../../src/tts/xai';
+
+/**
+ * [mocked] xAI TTS — request shape, Bearer auth, telephony factories, custom
+ * voices. The xAI `/tts` and `/custom-voices` endpoints are mocked at the
+ * global `fetch` boundary; every other code path (payload assembly, byte
+ * streaming, source-format declaration, multipart order, env-var fallback) runs
+ * for real.
+ */
+describe('[mocked] xAI TTS — request + streaming', () => {
+ const ORIGINAL_FETCH = global.fetch;
+ let lastBody: Record | null = null;
+ let lastHeaders: Record | null = null;
+ let lastUrl: string | null = null;
+
+ function mockBytesResponse(chunks: string[], status = 200): Response {
+ const stream = new ReadableStream({
+ start(controller) {
+ const enc = new TextEncoder();
+ for (const c of chunks) controller.enqueue(enc.encode(c));
+ controller.close();
+ },
+ });
+ return new Response(stream, { status });
+ }
+
+ beforeEach(() => {
+ lastBody = null;
+ lastHeaders = null;
+ lastUrl = null;
+ global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+ lastUrl = typeof input === 'string' ? input : input.toString();
+ lastBody = JSON.parse(init?.body as string);
+ lastHeaders = init?.headers as Record;
+ return mockBytesResponse(['hello', 'world']);
+ }) as typeof global.fetch;
+ });
+
+ afterEach(() => {
+ global.fetch = ORIGINAL_FETCH;
+ });
+
+ it('posts to the REST endpoint with a Bearer auth header and JSON content-type', async () => {
+ const tts = new XaiTTS('xai-test-key');
+ await tts.synthesize('ciao');
+ expect(lastUrl).toBe('https://api.x.ai/v1/tts');
+ expect(lastHeaders?.Authorization).toBe('Bearer xai-test-key');
+ expect(lastHeaders?.['Content-Type']).toBe('application/json');
+ });
+
+ it('defaults to voice=eve, language=auto, pcm @ 16000 (no mp3 default)', async () => {
+ const tts = new XaiTTS('xai-test-key');
+ await tts.synthesize('hi');
+ expect(lastBody).toMatchObject({
+ text: 'hi',
+ voice_id: 'eve',
+ language: 'auto',
+ output_format: { codec: 'pcm', sample_rate: 16000 },
+ });
+ // Optional knobs omitted when unset.
+ expect(lastBody).not.toHaveProperty('speed');
+ expect(lastBody).not.toHaveProperty('optimize_streaming_latency');
+ expect(lastBody).not.toHaveProperty('text_normalization');
+ });
+
+ it('threads voice / language / speed / latency / normalization into the payload', async () => {
+ const tts = new XaiTTS('xai-test-key', {
+ voice: 'leo',
+ language: 'it',
+ speed: 1.2,
+ optimizeStreamingLatency: 2,
+ textNormalization: true,
+ });
+ await tts.synthesize('ciao');
+ expect(lastBody).toMatchObject({
+ voice_id: 'leo',
+ language: 'it',
+ speed: 1.2,
+ optimize_streaming_latency: 2,
+ text_normalization: true,
+ });
+ });
+
+ it('nests an MP3 bit_rate under output_format when set (omitted otherwise)', async () => {
+ const tts = new XaiTTS('xai-test-key', {
+ codec: 'mp3',
+ sampleRate: 44100,
+ bitRate: 192000,
+ });
+ await tts.synthesize('ciao');
+ expect(lastBody).toMatchObject({
+ output_format: { codec: 'mp3', sample_rate: 44100, bit_rate: 192000 },
+ });
+ });
+
+ it('streams the response bytes through synthesizeStream', async () => {
+ const tts = new XaiTTS('xai-test-key');
+ const chunks: Buffer[] = [];
+ for await (const c of tts.synthesizeStream('ciao')) chunks.push(c);
+ expect(Buffer.concat(chunks).toString()).toBe('helloworld');
+ });
+
+ it('forTwilio emits μ-law @ 8 kHz and declares carrier-native source format', async () => {
+ const tts = XaiTTS.forTwilio('xai-test-key');
+ expect(tts.sourceAudioFormat()).toEqual({ encoding: 'mulaw', sampleRate: 8000 });
+ await tts.synthesize('hi');
+ expect(lastBody).toMatchObject({ output_format: { codec: 'mulaw', sample_rate: 8000 } });
+ });
+
+ it('forTelnyx emits pcm @ 16 kHz', async () => {
+ const tts = XaiTTS.forTelnyx('xai-test-key');
+ expect(tts.sourceAudioFormat()).toEqual({ encoding: 'pcm_s16le', sampleRate: 16000 });
+ await tts.synthesize('hi');
+ expect(lastBody).toMatchObject({ output_format: { codec: 'pcm', sample_rate: 16000 } });
+ });
+
+ it('surfaces a non-2xx status as an error', async () => {
+ global.fetch = (async () => new Response('nope', { status: 400 })) as typeof global.fetch;
+ const tts = new XaiTTS('xai-test-key');
+ await expect(async () => {
+ for await (const _ of tts.synthesizeStream('hi')) void _;
+ }).rejects.toThrow(/xAI TTS error 400/);
+ });
+
+ describe('pipeline TTS facade (getpatter/tts/xai)', () => {
+ afterEach(() => {
+ delete process.env.XAI_API_KEY;
+ });
+
+ it('reads XAI_API_KEY from the environment', async () => {
+ process.env.XAI_API_KEY = 'xai-env-key';
+ const tts = new XaiPipelineTTS();
+ await tts.synthesize('hi');
+ expect(lastHeaders?.Authorization).toBe('Bearer xai-env-key');
+ });
+
+ it('forTwilio on the facade emits μ-law @ 8 kHz', () => {
+ const tts = XaiPipelineTTS.forTwilio({ apiKey: 'xai-test-key' });
+ expect(tts.sourceAudioFormat()).toEqual({ encoding: 'mulaw', sampleRate: 8000 });
+ });
+ });
+});
+
+describe('[mocked] xaiCreateCustomVoice', () => {
+ const ORIGINAL_FETCH = global.fetch;
+ let lastUrl: string | null = null;
+ let lastHeaders: Record | null = null;
+ let lastBody: FormData | null = null;
+
+ beforeEach(() => {
+ lastUrl = null;
+ lastHeaders = null;
+ lastBody = null;
+ global.fetch = (async (input: RequestInfo | URL, init?: RequestInit) => {
+ lastUrl = typeof input === 'string' ? input : input.toString();
+ lastHeaders = init?.headers as Record;
+ lastBody = init?.body as FormData;
+ return new Response(JSON.stringify({ voice_id: 'custom-abc123' }), {
+ status: 200,
+ headers: { 'Content-Type': 'application/json' },
+ });
+ }) as typeof global.fetch;
+ });
+
+ afterEach(() => {
+ global.fetch = ORIGINAL_FETCH;
+ });
+
+ it('posts multipart (name, language, then file LAST) and returns the voice_id', async () => {
+ const voiceId = await xaiCreateCustomVoice({
+ name: 'Narrator',
+ language: 'en',
+ audio: Buffer.from([1, 2, 3]),
+ apiKey: 'xai-test-key',
+ });
+ expect(voiceId).toBe('custom-abc123');
+ expect(lastUrl).toBe('https://api.x.ai/v1/custom-voices');
+ expect(lastHeaders?.Authorization).toBe('Bearer xai-test-key');
+ const keys = [...(lastBody as FormData).keys()];
+ expect(keys).toEqual(['name', 'language', 'file']);
+ const file = (lastBody as FormData).get('file') as File;
+ expect(file).toBeInstanceOf(Blob);
+ expect(file.type).toBe('audio/wav');
+ });
+
+ it('throws when the response omits a voice_id', async () => {
+ global.fetch = (async () =>
+ new Response(JSON.stringify({}), {
+ status: 200,
+ headers: { 'Content-Type': 'application/json' },
+ })) as typeof global.fetch;
+ await expect(
+ xaiCreateCustomVoice({ name: 'x', language: 'en', audio: Buffer.from([1]), apiKey: 'k' }),
+ ).rejects.toThrow(/did not include a voice_id/);
+ });
+
+ it('exposes the codec enum for callers', () => {
+ expect(XaiTTSCodec.MULAW).toBe('mulaw');
+ expect(XaiTTSCodec.PCM).toBe('pcm');
+ });
+});
diff --git a/libraries/typescript/tests/xai-realtime.mocked.test.ts b/libraries/typescript/tests/xai-realtime.mocked.test.ts
new file mode 100644
index 00000000..3af253ae
--- /dev/null
+++ b/libraries/typescript/tests/xai-realtime.mocked.test.ts
@@ -0,0 +1,374 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+// Mock the 'ws' module so connect() makes no real network connection. The mock
+// records constructor args (url + headers) and lets tests drive server frames.
+vi.mock('ws', () => {
+ // eslint-disable-next-line @typescript-eslint/no-require-imports
+ const EventEmitter = require('events');
+ class MockWebSocket extends EventEmitter {
+ static CONNECTING = 0;
+ static OPEN = 1;
+ static CLOSING = 2;
+ static CLOSED = 3;
+ static instances: MockWebSocket[] = [];
+ readyState = MockWebSocket.OPEN;
+ sent: string[] = [];
+ url: string;
+ options: unknown;
+ constructor(url: string, options?: unknown) {
+ super();
+ this.url = url;
+ this.options = options;
+ MockWebSocket.instances.push(this);
+ }
+ send(data: string): void {
+ this.sent.push(data);
+ }
+ ping(): void {
+ /* heartbeat no-op */
+ }
+ close(): void {
+ this.readyState = MockWebSocket.CLOSED;
+ }
+ }
+ return { default: MockWebSocket };
+});
+
+import {
+ XaiRealtimeAdapter,
+ XAI_REALTIME_WS_URL,
+ XAI_REALTIME_DEFAULT_MODEL,
+ XAI_REALTIME_DEFAULT_VOICE,
+} from '../src/providers/xai-realtime';
+import { OpenAIRealtime2Adapter } from '../src/providers/openai-realtime-2';
+import { OpenAIRealtimeAdapter } from '../src/providers/openai-realtime';
+import { XaiRealtime } from '../src/engines/xai';
+import { buildAIAdapter, type LocalConfig } from '../src/server';
+import type { AgentOptions } from '../src/types';
+import { Patter } from '../src/client';
+import { Twilio } from '../src/index';
+import {
+ DEFAULT_PRICING,
+ calculateRealtimeCost,
+ calculateSttCost,
+ calculateTtsCost,
+} from '../src/pricing';
+import { CallMetricsAccumulator, type CostBreakdown } from '../src/metrics';
+
+type MockWS = {
+ readonly sent: string[];
+ readonly url: string;
+ readonly options: { headers?: Record };
+ emit: (event: string, ...args: unknown[]) => void;
+};
+
+async function getWsCtor(): Promise<{ instances: MockWS[] }> {
+ const mod = (await import('ws')).default as unknown as { instances: MockWS[] };
+ return mod;
+}
+
+const CONFIG: LocalConfig = {
+ phoneNumber: '+15555550100',
+ webhookUrl: 'https://example.com/voice',
+};
+
+/** Read the session body from the first `session.update` sent after `session.created`. */
+function sessionUpdateBody(ws: MockWS): Record {
+ const parsed = JSON.parse(ws.sent[0]) as { type: string; session: Record };
+ expect(parsed.type).toBe('session.update');
+ return parsed.session;
+}
+
+describe('[unit] XaiRealtime engine marker', () => {
+ it('throws without an apiKey and no env', () => {
+ const prev = process.env.XAI_API_KEY;
+ delete process.env.XAI_API_KEY;
+ try {
+ expect(() => new XaiRealtime()).toThrow(/xAI Realtime requires an apiKey/);
+ } finally {
+ if (prev !== undefined) process.env.XAI_API_KEY = prev;
+ }
+ });
+
+ it('reads XAI_API_KEY from the environment and carries the xai_realtime kind', () => {
+ const prev = process.env.XAI_API_KEY;
+ process.env.XAI_API_KEY = 'xai-env-key';
+ try {
+ const engine = new XaiRealtime();
+ expect(engine.apiKey).toBe('xai-env-key');
+ expect(engine.kind).toBe('xai_realtime');
+ } finally {
+ if (prev === undefined) delete process.env.XAI_API_KEY;
+ else process.env.XAI_API_KEY = prev;
+ }
+ });
+
+ it('captures the xAI-specific knobs on the marker', () => {
+ const engine = new XaiRealtime({
+ apiKey: 'xai-test-key',
+ reasoningEffort: 'none',
+ languageHint: 'it',
+ keyterms: ['Grok'],
+ serverTools: [{ type: 'web_search' }],
+ });
+ expect(engine.reasoningEffort).toBe('none');
+ expect(engine.languageHint).toBe('it');
+ expect(engine.keyterms).toEqual(['Grok']);
+ expect(engine.serverTools).toEqual([{ type: 'web_search' }]);
+ });
+
+ it('validates turnDetection eagerly (rejects bad values)', () => {
+ expect(
+ () =>
+ new XaiRealtime({
+ apiKey: 'xai-test-key',
+ // @ts-expect-error intentionally invalid for the runtime guard
+ turnDetection: { type: 'nope' },
+ }),
+ ).toThrow(/server_vad|semantic_vad/);
+ });
+});
+
+describe('[unit] XaiRealtime dispatch', () => {
+ it('Patter.agent({ engine: new XaiRealtime() }) selects provider xai_realtime', () => {
+ const phone = new Patter({
+ carrier: new Twilio({ accountSid: 'AC_test', authToken: 'tok_test' }),
+ phoneNumber: '+15550000000',
+ webhookUrl: 'abc.ngrok.io',
+ });
+ const agent = phone.agent({
+ engine: new XaiRealtime({ apiKey: 'xai-test-key', voice: 'leo' }),
+ systemPrompt: 'You are helpful.',
+ });
+ expect(agent.provider).toBe('xai_realtime');
+ expect(agent.voice).toBe('leo');
+ });
+
+ it('buildAIAdapter routes xai_realtime to XaiRealtimeAdapter (GA + base subclass)', () => {
+ const agent: AgentOptions = {
+ systemPrompt: 'You are helpful.',
+ engine: new XaiRealtime({ apiKey: 'xai-test-key' }),
+ provider: 'xai_realtime',
+ };
+ const adapter = buildAIAdapter(CONFIG, agent);
+ expect(adapter).toBeInstanceOf(XaiRealtimeAdapter);
+ // Subclass of the GA adapter (→ inherits the PCM24↔mulaw8 transcode path)
+ // AND the base adapter (→ every `instanceof OpenAIRealtimeAdapter` feature
+ // gate in the stream handler fires for xAI with no per-provider branches).
+ expect(adapter).toBeInstanceOf(OpenAIRealtime2Adapter);
+ expect(adapter).toBeInstanceOf(OpenAIRealtimeAdapter);
+ });
+
+ it('throws a clear error when provider is xai_realtime but the engine is missing', () => {
+ const agent = { systemPrompt: 'hi', provider: 'xai_realtime' } as unknown as AgentOptions;
+ expect(() => buildAIAdapter(CONFIG, agent)).toThrow(/xAI Realtime mode requires/);
+ });
+});
+
+describe('[mocked] XaiRealtimeAdapter connect', () => {
+ beforeEach(async () => {
+ const ws = await getWsCtor();
+ ws.instances.length = 0;
+ });
+
+ it('connects to the xAI endpoint with a Bearer key and grafts the xAI session keys', async () => {
+ const adapter = new XaiRealtimeAdapter('xai-test-key', {
+ reasoningEffort: 'none',
+ languageHint: 'it',
+ keyterms: ['Grok', 'xAI'],
+ speed: 1.2,
+ vadThreshold: 0.6,
+ prefixPaddingMs: 250,
+ idleTimeoutMs: 5000,
+ replace: { 'Acme Mobile': 'Acme Mobull' },
+ resumption: true,
+ serverTools: [{ type: 'web_search' }],
+ });
+ const p = adapter.connect();
+ const ws = (await getWsCtor()).instances[0];
+
+ // Endpoint + auth + default model.
+ expect(ws.url).toBe(`${XAI_REALTIME_WS_URL}?model=grok-voice-latest`);
+ expect(ws.options.headers?.Authorization).toBe('Bearer xai-test-key');
+
+ // Handshake: server says session.created -> adapter sends the grafted update.
+ ws.emit('message', JSON.stringify({ type: 'session.created' }));
+ const s = sessionUpdateBody(ws);
+ const audio = s.audio as {
+ input: { transcription: Record; turn_detection: Record };
+ output: Record;
+ };
+ expect(s.reasoning).toEqual({ effort: 'none' });
+ // xAI-valid transcription model (NOT OpenAI's whisper-1), and the
+ // OpenAI-only `language` key never leaks into the xAI session.
+ expect(audio.input.transcription.model).toBe('grok-transcribe');
+ expect(audio.input.transcription.language).toBeUndefined();
+ expect(audio.input.transcription.language_hint).toBe('it');
+ expect(audio.input.transcription.keyterms).toEqual(['Grok', 'xAI']);
+ expect(audio.output.speed).toBe(1.2);
+ expect(audio.output.voice).toBe('eve');
+ expect(audio.input.turn_detection.threshold).toBe(0.6);
+ expect(audio.input.turn_detection.prefix_padding_ms).toBe(250);
+ expect(audio.input.turn_detection.idle_timeout_ms).toBe(5000);
+ expect(s.replace).toEqual({ 'Acme Mobile': 'Acme Mobull' });
+ expect(s.resumption).toEqual({ enabled: true });
+ expect(s.tools).toContainEqual({ type: 'web_search' });
+
+ ws.emit('message', JSON.stringify({ type: 'session.updated' }));
+ await p;
+ adapter.close();
+ });
+
+ it('omits every xAI-specific key when unset and uses the eve/grok defaults', async () => {
+ const adapter = new XaiRealtimeAdapter('xai-test-key');
+ const p = adapter.connect();
+ const ws = (await getWsCtor()).instances[0];
+
+ expect(ws.url).toBe(
+ `${XAI_REALTIME_WS_URL}?model=${encodeURIComponent(XAI_REALTIME_DEFAULT_MODEL)}`,
+ );
+
+ ws.emit('message', JSON.stringify({ type: 'session.created' }));
+ const s = sessionUpdateBody(ws);
+ const audio = s.audio as {
+ input: { transcription: Record; turn_detection: Record };
+ output: Record;
+ };
+ expect(s.reasoning).toBeUndefined();
+ // Transcription model still defaults to xAI's grok-transcribe, and no
+ // OpenAI-only `language` key leaks even with no config.
+ expect(audio.input.transcription.model).toBe('grok-transcribe');
+ expect(audio.input.transcription.language).toBeUndefined();
+ expect(audio.input.transcription.language_hint).toBeUndefined();
+ expect(audio.input.transcription.keyterms).toBeUndefined();
+ expect(audio.output.speed).toBeUndefined();
+ expect(audio.output.voice).toBe(XAI_REALTIME_DEFAULT_VOICE);
+ expect(audio.input.turn_detection.idle_timeout_ms).toBeUndefined();
+ // Inherited GA default threshold (0.5) when vadThreshold is unset.
+ expect(audio.input.turn_detection.threshold).toBe(0.5);
+ expect(s.replace).toBeUndefined();
+ expect(s.resumption).toBeUndefined();
+ expect(s.tools).toBeUndefined();
+
+ ws.emit('message', JSON.stringify({ type: 'session.updated' }));
+ await p;
+ adapter.close();
+ });
+
+ it('honors a baseUrl override (trailing slash trimmed)', async () => {
+ const adapter = new XaiRealtimeAdapter('xai-test-key', { baseUrl: 'wss://example.test/rt/', model: 'grok-voice-fast-1.0' });
+ const p = adapter.connect();
+ const ws = (await getWsCtor()).instances[0];
+ expect(ws.url).toBe('wss://example.test/rt?model=grok-voice-fast-1.0');
+ ws.emit('message', JSON.stringify({ type: 'session.created' }));
+ ws.emit('message', JSON.stringify({ type: 'session.updated' }));
+ await p;
+ adapter.close();
+ });
+
+ it('surfaces a setup error frame as a clear rejection', async () => {
+ const adapter = new XaiRealtimeAdapter('xai-test-key', { model: 'bad-model' });
+ const p = adapter.connect();
+ const ws = (await getWsCtor()).instances[0];
+ ws.emit('message', JSON.stringify({ type: 'session.created' }));
+ ws.emit('message', JSON.stringify({ type: 'error', error: { message: 'model not found' } }));
+ await expect(p).rejects.toThrow(/model not found/);
+ });
+
+ it('warmup is a no-op and openParkedConnection is unsupported (no OpenAI endpoint hit)', async () => {
+ const adapter = new XaiRealtimeAdapter('xai-test-key');
+ await expect(adapter.warmup()).resolves.toBeUndefined();
+ await expect(adapter.openParkedConnection()).rejects.toThrow(/not supported/);
+ const ws = await getWsCtor();
+ expect(ws.instances.length).toBe(0);
+ });
+});
+
+describe('[mocked] openai-realtime-2 realtimeBaseUrl() refactor regression', () => {
+ beforeEach(async () => {
+ const ws = await getWsCtor();
+ ws.instances.length = 0;
+ });
+
+ it('OpenAIRealtime2Adapter still connects to the OpenAI GA endpoint (byte-identical URL)', async () => {
+ const adapter = new OpenAIRealtime2Adapter('sk-test', 'gpt-realtime-2');
+ const p = adapter.connect();
+ const ws = (await getWsCtor()).instances[0];
+ expect(ws.url).toBe('wss://api.openai.com/v1/realtime?model=gpt-realtime-2');
+ ws.emit('message', JSON.stringify({ type: 'session.created' }));
+ ws.emit('message', JSON.stringify({ type: 'session.updated' }));
+ await p;
+ adapter.close();
+ });
+});
+
+describe('[unit] xAI pricing', () => {
+ it('bills xai_realtime per MINUTE of session duration (0 when duration unknown)', () => {
+ // Arg order (…, provider, durationSeconds) mirrors the Python keyword args.
+ expect(calculateRealtimeCost({}, DEFAULT_PRICING, 'grok-voice-latest', 'xai_realtime', 60)).toBeCloseTo(0.05, 6);
+ expect(calculateRealtimeCost({}, DEFAULT_PRICING, 'grok-voice-latest', 'xai_realtime', 30)).toBeCloseTo(0.025, 6);
+ expect(calculateRealtimeCost({}, DEFAULT_PRICING, 'grok-voice-latest', 'xai_realtime', undefined)).toBe(0);
+ });
+
+ it('leaves the OpenAI token path unchanged (durationSeconds ignored)', () => {
+ const usage = { output_token_details: { audio_tokens: 1000 } };
+ expect(calculateRealtimeCost(usage, DEFAULT_PRICING, 'gpt-realtime-2')).toBeCloseTo(0.064, 6);
+ // Passing a duration must NOT change the token-based result.
+ expect(calculateRealtimeCost(usage, DEFAULT_PRICING, 'gpt-realtime-2', 'openai_realtime', 9999)).toBeCloseTo(0.064, 6);
+ });
+
+ it('prices xAI streaming STT ($0.20/hr) and TTS ($15/1M chars)', () => {
+ expect(calculateSttCost('xai', 60, DEFAULT_PRICING)).toBeCloseTo(0.003333, 6);
+ expect(calculateTtsCost('xai_tts', 1000, DEFAULT_PRICING)).toBeCloseTo(0.015, 6);
+ });
+
+ it('exposes the documented (unmetered) per-message realtime constant', () => {
+ expect(DEFAULT_PRICING.xai_realtime.text_input_per_message).toBe(0.004);
+ });
+});
+
+describe('[unit] xAI realtime metrics wiring', () => {
+ // Reach the private cost builder directly with a controlled duration — a real
+ // call derives duration from wall-clock, which can't be pinned to 2 min in a
+ // unit test. Mirrors the house pattern (openai-realtime-2-session).
+ function computeCost(acc: CallMetricsAccumulator, durationSeconds: number): CostBreakdown {
+ return (acc as unknown as { _computeCost(d: number): CostBreakdown })._computeCost(
+ durationSeconds,
+ );
+ }
+
+ it('bills the xai_realtime session per minute via the metrics accumulator', () => {
+ // 2 min × $0.05/min = $0.10. This exercises the metrics call-site arg order
+ // (…, provider, durationSeconds): a future swap makes pricing[]
+ // undefined → $0, which this assertion catches loudly.
+ const acc = new CallMetricsAccumulator({
+ callId: 'c-xai',
+ providerMode: 'xai_realtime',
+ telephonyProvider: 'twilio',
+ realtimeModel: 'grok-voice-latest',
+ });
+ expect(computeCost(acc, 120).llm).toBeCloseTo(0.1, 6);
+ });
+
+ it('ignores a stray OpenAI-shaped usage block for an xai_realtime call', () => {
+ const acc = new CallMetricsAccumulator({
+ callId: 'c-xai-2',
+ providerMode: 'xai_realtime',
+ telephonyProvider: 'twilio',
+ realtimeModel: 'grok-voice-latest',
+ });
+ // xAI is GA-compatible and could emit a token usage block on response.done;
+ // it must NOT inflate the per-minute cost nor surface a bogus cached-savings.
+ acc.recordRealtimeUsage({
+ input_token_details: {
+ audio_tokens: 5000,
+ text_tokens: 1000,
+ cached_tokens_details: { audio_tokens: 4000, text_tokens: 500 },
+ },
+ output_token_details: { audio_tokens: 8000, text_tokens: 200 },
+ });
+ const cost = computeCost(acc, 120);
+ expect(cost.llm).toBeCloseTo(0.1, 6); // still just the per-minute charge
+ expect(cost.llm_cached_savings).toBe(0);
+ });
+});