Skip to content

feat(voice_agent): add Azure TTS and STT provider support #247

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vizsatiz merged 2 commits into from
Mar 20, 2026
Merged

feat(voice_agent): add Azure TTS and STT provider support #247

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3a551ec
feat(voice_agent): add Azure TTS and STT provider support
rootflo-hardik Mar 20, 2026
d830db2
resolved review comments
rootflo-hardik Mar 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 81 additions & 2 deletions wavefront/client/src/config/voice-providers.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ export interface VoiceProvidersConfig {
*/
export const VOICE_PROVIDERS_CONFIG: VoiceProvidersConfig = {
tts: {
providers: ['elevenlabs', 'deepgram', 'cartesia', 'sarvam'] as const,
providers: ['elevenlabs', 'deepgram', 'cartesia', 'sarvam', 'azure'] as const,
configs: {
elevenlabs: {
name: 'ElevenLabs',
Expand Down Expand Up @@ -217,10 +217,61 @@ export const VOICE_PROVIDERS_CONFIG: VoiceProvidersConfig = {
},
},
},
azure: {
name: 'Azure',
badge: {
bg: 'bg-sky-100',
text: 'text-sky-700',
},
parameters: {
style: {
type: 'string' as const,
default: '',
description: 'Speaking style (e.g. cheerful, sad, angry)',
placeholder: 'cheerful',
},
rate: {
type: 'string' as const,
default: '',
description: 'Speech rate (e.g. +10%, fast, slow)',
placeholder: '+0%',
},
pitch: {
type: 'string' as const,
default: '',
description: 'Pitch adjustment (e.g. +0Hz, high, low)',
placeholder: '+0Hz',
},
role: {
type: 'string' as const,
default: '',
description: 'Voice role for expression (e.g. YoungAdultFemale)',
placeholder: 'YoungAdultFemale',
},
style_degree: {
type: 'string' as const,
default: '',
description: 'Intensity of speaking style (0.01 to 2.0)',
placeholder: '1.0',
},
volume: {
type: 'string' as const,
default: '',
description: 'Volume level (e.g. +20%, loud, x-soft)',
placeholder: '+0%',
},
sample_rate: {
type: 'number' as const,
default: undefined,
description: 'Audio sample rate in Hz',
placeholder: '16000',
},
},
},
},
},
stt: {
providers: ['deepgram', 'sarvam', 'elevenlabs'] as const,
providers: ['deepgram', 'sarvam', 'elevenlabs', 'azure'] as const,
configs: {
deepgram: {
name: 'Deepgram',
Expand Down Expand Up @@ -352,6 +403,34 @@ export const VOICE_PROVIDERS_CONFIG: VoiceProvidersConfig = {
},
},
},
azure: {
name: 'Azure',
badge: {
bg: 'bg-sky-100',
text: 'text-sky-700',
},
parameters: {
endpoint_id: {
type: 'string' as const,
default: '',
description: 'Custom model endpoint ID (optional)',
placeholder: '',
},
sample_rate: {
type: 'number' as const,
default: undefined,
description: 'Audio sample rate in Hz',
placeholder: '8000',
},
ttfs_p99_latency: {
type: 'number' as const,
default: undefined,
description: 'P99 latency threshold in seconds for first speech detection',
placeholder: '1.5',
step: 0.1,
},
},
},
},
},
};
Expand Down
41 changes: 35 additions & 6 deletions wavefront/client/src/pages/apps/[appId]/voice-agents/stt-configs/CreateSttConfigDialog.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,18 @@ import React, { useEffect, useState } from 'react';
import { useForm } from 'react-hook-form';
import { z } from 'zod';

const createSttConfigSchema = z.object({
display_name: z.string().min(1, 'Display name is required').max(100, 'Display name must be 100 characters or less'),
description: z.string().max(500, 'Description must be 500 characters or less').optional(),
provider: z.enum(['deepgram', 'sarvam', 'elevenlabs'] as [string, ...string[]]),
api_key: z.string().min(1, 'API key is required'),
});
const createSttConfigSchema = z
.object({
display_name: z.string().min(1, 'Display name is required').max(100, 'Display name must be 100 characters or less'),
description: z.string().max(500, 'Description must be 500 characters or less').optional(),
provider: z.enum(['deepgram', 'sarvam', 'elevenlabs', 'azure'] as [string, ...string[]]),
api_key: z.string().min(1, 'API key is required'),
region: z.string().optional(),
})
.refine((data) => data.provider !== 'azure' || (data.region && data.region.trim().length > 0), {
message: 'Region is required for Azure',
path: ['region'],
});

type CreateSttConfigInput = z.infer<typeof createSttConfigSchema>;

Expand All @@ -56,9 +62,12 @@ const CreateSttConfigDialog: React.FC<CreateSttConfigDialogProps> = ({ isOpen, o
description: '',
provider: 'deepgram',
api_key: '',
region: '',
},
});

const selectedProvider = form.watch('provider');

// Reset form when dialog closes
useEffect(() => {
if (!isOpen) {
Expand All @@ -67,6 +76,7 @@ const CreateSttConfigDialog: React.FC<CreateSttConfigDialogProps> = ({ isOpen, o
description: '',
provider: 'deepgram',
api_key: '',
region: '',
});
}
}, [isOpen, form]);
Expand All @@ -79,6 +89,7 @@ const CreateSttConfigDialog: React.FC<CreateSttConfigDialogProps> = ({ isOpen, o
description: data.description?.trim() || null,
provider: data.provider as SttProvider,
api_key: data.api_key.trim(),
region: data.region?.trim() || null,
});
notifySuccess('STT configuration created successfully');
onSuccess?.();
Expand Down Expand Up @@ -186,6 +197,24 @@ const CreateSttConfigDialog: React.FC<CreateSttConfigDialogProps> = ({ isOpen, o
)}
/>

{selectedProvider === 'azure' && (
<FormField
control={form.control}
name="region"
render={({ field }) => (
<FormItem>
<FormLabel>
Region <span className="text-red-500">*</span>
</FormLabel>
<FormControl>
<Input placeholder="e.g., eastus, westus2" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
)}

<Alert variant="info">
<AlertDescription>
<strong>Security Note:</strong> API keys are stored securely and never returned in API responses.
Expand Down
44 changes: 38 additions & 6 deletions wavefront/client/src/pages/apps/[appId]/voice-agents/stt-configs/EditSttConfigDialog.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,12 +30,18 @@ import React, { useEffect, useState } from 'react';
import { useForm } from 'react-hook-form';
import { z } from 'zod';

const updateSttConfigSchema = z.object({
display_name: z.string().min(1, 'Display name is required').max(100, 'Display name must be 100 characters or less'),
description: z.string().max(500, 'Description must be 500 characters or less').optional(),
provider: z.enum(['deepgram', 'sarvam', 'elevenlabs'] as [string, ...string[]]),
api_key: z.string().optional(),
});
const updateSttConfigSchema = z
.object({
display_name: z.string().min(1, 'Display name is required').max(100, 'Display name must be 100 characters or less'),
description: z.string().max(500, 'Description must be 500 characters or less').optional(),
provider: z.enum(['deepgram', 'sarvam', 'elevenlabs', 'azure'] as [string, ...string[]]),
api_key: z.string().optional(),
region: z.string().optional(),
})
.refine((data) => data.provider !== 'azure' || (data.region && data.region.trim().length > 0), {
message: 'Region is required for Azure',
path: ['region'],
});

type UpdateSttConfigInput = z.infer<typeof updateSttConfigSchema>;

Expand All @@ -57,9 +63,12 @@ const EditSttConfigDialog: React.FC<EditSttConfigDialogProps> = ({ isOpen, onOpe
description: config.description || '',
provider: config.provider,
api_key: '',
region: config.region || '',
},
});

const selectedProvider = form.watch('provider');

// Reset form when dialog opens or config changes
useEffect(() => {
if (isOpen && config) {
Expand All @@ -68,6 +77,7 @@ const EditSttConfigDialog: React.FC<EditSttConfigDialogProps> = ({ isOpen, onOpe
description: config.description || '',
provider: config.provider,
api_key: '',
region: config.region || '',
});
}
}, [isOpen, config, form]);
Expand All @@ -84,6 +94,10 @@ const EditSttConfigDialog: React.FC<EditSttConfigDialogProps> = ({ isOpen, onOpe
updateData.api_key = data.api_key.trim();
}

if (data.region !== undefined) {
updateData.region = data.region?.trim() || null;
}

await floConsoleService.sttConfigService.updateSttConfig(config.id, updateData);
notifySuccess('STT configuration updated successfully');
onSuccess?.();
Expand Down Expand Up @@ -196,6 +210,24 @@ const EditSttConfigDialog: React.FC<EditSttConfigDialogProps> = ({ isOpen, onOpe
)}
/>

{selectedProvider === 'azure' && (
<FormField
control={form.control}
name="region"
render={({ field }) => (
<FormItem>
<FormLabel>
Region <span className="text-red-500">*</span>
</FormLabel>
<FormControl>
<Input placeholder="e.g., eastus, westus2" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
)}

<DialogFooter>
<Button type="button" variant="outline" onClick={() => onOpenChange(false)} disabled={loading}>
Cancel
Expand Down
43 changes: 36 additions & 7 deletions wavefront/client/src/pages/apps/[appId]/voice-agents/tts-configs/CreateTtsConfigDialog.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,12 +29,18 @@ import React, { useEffect, useState } from 'react';
import { useForm } from 'react-hook-form';
import { z } from 'zod';

const createTtsConfigSchema = z.object({
display_name: z.string().min(1, 'Display name is required').max(100, 'Display name must be 100 characters or less'),
description: z.string().max(500, 'Description must be 500 characters or less').optional(),
provider: z.enum(['elevenlabs', 'deepgram', 'cartesia', 'sarvam'] as [string, ...string[]]),
api_key: z.string().min(1, 'API key is required'),
});
const createTtsConfigSchema = z
.object({
display_name: z.string().min(1, 'Display name is required').max(100, 'Display name must be 100 characters or less'),
description: z.string().max(500, 'Description must be 500 characters or less').optional(),
provider: z.enum(['elevenlabs', 'deepgram', 'cartesia', 'sarvam', 'azure'] as [string, ...string[]]),
api_key: z.string().min(1, 'API key is required'),
region: z.string().optional(),
})
.refine((data) => data.provider !== 'azure' || (data.region && data.region.trim().length > 0), {
message: 'Region is required for Azure',
path: ['region'],
});

type CreateTtsConfigInput = z.infer<typeof createTtsConfigSchema>;

Expand All @@ -55,9 +61,12 @@ const CreateTtsConfigDialog: React.FC<CreateTtsConfigDialogProps> = ({ isOpen, o
description: '',
provider: 'elevenlabs',
api_key: '',
region: '',
},
});

const selectedProvider = form.watch('provider');

// Reset form when dialog closes
useEffect(() => {
if (!isOpen) {
Expand All @@ -66,6 +75,7 @@ const CreateTtsConfigDialog: React.FC<CreateTtsConfigDialogProps> = ({ isOpen, o
description: '',
provider: 'elevenlabs',
api_key: '',
region: '',
});
}
}, [isOpen, form]);
Expand All @@ -76,8 +86,9 @@ const CreateTtsConfigDialog: React.FC<CreateTtsConfigDialogProps> = ({ isOpen, o
await floConsoleService.ttsConfigService.createTtsConfig({
display_name: data.display_name.trim(),
description: data.description?.trim() || null,
provider: data.provider as 'elevenlabs' | 'deepgram' | 'cartesia',
provider: data.provider as 'elevenlabs' | 'deepgram' | 'cartesia' | 'sarvam' | 'azure',
api_key: data.api_key.trim(),
region: data.region?.trim() || null,
});
notifySuccess('TTS configuration created successfully');
onSuccess?.();
Expand Down Expand Up @@ -185,6 +196,24 @@ const CreateTtsConfigDialog: React.FC<CreateTtsConfigDialogProps> = ({ isOpen, o
)}
/>

{selectedProvider === 'azure' && (
<FormField
control={form.control}
name="region"
render={({ field }) => (
<FormItem>
<FormLabel>
Region <span className="text-red-500">*</span>
</FormLabel>
<FormControl>
<Input placeholder="e.g., eastus, westus2" {...field} />
</FormControl>
<FormMessage />
</FormItem>
)}
/>
)}

<Alert variant="info">
<AlertDescription>
<strong>Security Note:</strong> API keys are stored securely and never returned in API responses.
Expand Down
Loading
Loading