From 1f1dd97d7a4a5d5f5d877589becfa03eaac3d3dc Mon Sep 17 00:00:00 2001 From: seymourtang Date: Fri, 31 Jul 2026 14:15:28 +0800 Subject: [PATCH] feat(agentkit): add support for mllm & tts --- docs/concepts/vendors.md | 23 ++++-- docs/guides/mllm-flow.md | 6 +- docs/guides/regional-routing.md | 8 +- docs/index.md | 6 +- docs/reference/vendors.md | 66 ++++++++++++++-- src/agora_agent/agentkit/__init__.py | 6 ++ src/agora_agent/agentkit/regional_agent.py | 14 +++- src/agora_agent/agentkit/vendors/__init__.py | 7 +- src/agora_agent/agentkit/vendors/catalog.py | 15 ++++ src/agora_agent/agentkit/vendors/cn.py | 64 ++++++++++++++- src/agora_agent/agentkit/vendors/mllm.py | 55 +++++++++++++ .../agentkit/vendors/namespaces.py | 19 +++++ src/agora_agent/agentkit/vendors/region.py | 12 +++ src/agora_agent/agentkit/vendors/stt.py | 11 ++- src/agora_agent/agentkit/vendors/tts.py | 25 ++++++ src/agora_agent/cn.py | 5 +- tests/custom/test_request_body.py | 78 +++++++++++++++++++ 17 files changed, 391 insertions(+), 29 deletions(-) diff --git a/docs/concepts/vendors.md b/docs/concepts/vendors.md index 0d5113b..28d2d0e 100644 --- a/docs/concepts/vendors.md +++ b/docs/concepts/vendors.md @@ -70,6 +70,7 @@ Used with `agent.with_tts()`. Each TTS vendor produces audio at a specific sampl | `DeepgramTTS` | Deepgram | `api_key`, `model` | Configurable | | `GradiumTTS` | Gradium | `api_key` | Configurable | | `MistralTTS` | Mistral | `api_key` | — | +| `TypecastTTS` | Typecast | `api_key`, `voice_id`, `model` | — | | `SarvamTTS` | Sarvam | `api_key` | — | | `XaiTTS` | xAI | `api_key`, `language` | Configurable | @@ -109,6 +110,8 @@ Used with `agent.with_stt()`. Use `turn_detection.language` for Agora interaction language; it defaults to `en-US`. STT vendor `language` options are serialized under `asr.params` using each provider's own format. If `with_stt()` is omitted, AgentKit defaults to `AresSTT` for global clients and `FengmingSTT` for `Area.CN` clients. Ares does not take a provider language option; AgentKit uses `turn_detection.language` for REST `asr.language`. +`AresSTT` and `FengmingSTT` accept an optional `keywords: List[str]` hotword list. + | Class | Provider | Required Parameters | |---|---|---| | `SpeechmaticsSTT` | Speechmatics | `api_key`, `language` | @@ -146,12 +149,14 @@ stt = DeepgramSTT(api_key='your-deepgram-key', language='en-US', model='nova-2') Used with `agent.with_mllm()` for the [MLLM flow](../guides/mllm-flow.md). These handle audio input and output end-to-end. -| Class | Provider | Required Parameters | -|---|---|---| -| `OpenAIRealtime` | OpenAI Realtime | `api_key`; optional `turn_detection` | -| `GeminiLive` | Google Gemini Live API | `api_key`, `model`; optional `turn_detection` | -| `VertexAI` | Vertex AI (Gemini Live) | `model`, `project_id`, `location`, `adc_credentials_string`; optional `turn_detection` | -| `XaiGrok` | xAI Grok (`mllm.vendor`: `xai`) | `api_key`; optional `voice`, `language`, `sample_rate`, `turn_detection` | +| Class | Provider | Area | Required Parameters | +|---|---|---|---| +| `OpenAIRealtime` | OpenAI Realtime | Global | `api_key`; optional `turn_detection` | +| `AzureOpenAIRealtime` | Azure OpenAI Realtime | Global | `api_key`, `url`, `turn_detection`; optional `max_history` | +| `GeminiLive` | Google Gemini Live API | Global | `api_key`, `model`; optional `turn_detection` | +| `VertexAI` | Vertex AI (Gemini Live) | Global | `model`, `project_id`, `location`, `adc_credentials_string`; optional `turn_detection` | +| `XaiGrok` | xAI Grok (`mllm.vendor`: `xai`) | Global | `api_key`; optional `voice`, `language`, `sample_rate`, `turn_detection` | +| `QwenOmni` | Alibaba Cloud Qwen Omni Realtime | CN | `api_key`, `turn_detection`; optional `url` | ```python @@ -160,6 +165,12 @@ from agora_agent import OpenAIRealtime mllm = OpenAIRealtime(api_key='your-openai-key', model='gpt-4o-realtime-preview') ``` +Import the CN-only Qwen helper from `agora_agent.cn`: + +```python +from agora_agent.cn import QwenOmni +``` + ## Avatar Vendors Used with `agent.with_avatar()` in the cascading ASR + LLM + TTS pipeline. Some avatars require specific TTS sample rates — see [Avatar Integration](../guides/avatars.md). diff --git a/docs/guides/mllm-flow.md b/docs/guides/mllm-flow.md index 75f4a9c..ae1d036 100644 --- a/docs/guides/mllm-flow.md +++ b/docs/guides/mllm-flow.md @@ -1,7 +1,7 @@ --- sidebar_position: 2 title: MLLM Flow (Multimodal) -description: Use OpenAI Realtime, Gemini Live, Vertex AI, or xAI for end-to-end audio processing. +description: Use supported global or CN realtime multimodal models for end-to-end audio processing. --- # MLLM Flow (Multimodal) @@ -11,9 +11,11 @@ The MLLM (Multimodal LLM) flow uses a single model to handle both audio input an MLLM vendors supported by AgentKit: - **OpenAI Realtime** — `gpt-4o-realtime-preview` and related models +- **Azure OpenAI Realtime** — global Azure-hosted realtime deployments - **Gemini Live** — direct Google AI API access for audio-native Gemini models - **Vertex AI** — Gemini Live through Google Cloud Vertex AI - **xAI Grok** — xAI Realtime API +- **Qwen Omni** — Alibaba Cloud Qwen Omni Realtime for `Area.CN` ## Enable MLLM Mode @@ -167,7 +169,7 @@ mllm = OpenAIRealtime( |---|---|---| | Latency | Lower — single model, no pipeline | Higher — three models in sequence | | Voice control | Model-dependent | Full vendor choice for TTS | -| Vendor flexibility | Limited to supported MLLM providers (OpenAI Realtime, Gemini Live, Vertex AI, xAI Grok) | Mix and match LLM, TTS, and STT vendors | +| Vendor flexibility | Limited to supported global and CN MLLM providers | Mix and match LLM, TTS, and STT vendors | | Audio understanding | Model hears tone, pacing, emotion | STT produces text only | ## Next Steps diff --git a/docs/guides/regional-routing.md b/docs/guides/regional-routing.md index 23bea9b..a6051bd 100644 --- a/docs/guides/regional-routing.md +++ b/docs/guides/regional-routing.md @@ -38,10 +38,10 @@ client = Agora( Bind `client` into `Agent(client=client, ...)` and construct vendors directly with SDK classes. The bound client selects `CNAgent` or `GlobalAgent` for IDE hints based on `area`, but does not restrict which vendor classes you can configure. If you omit `with_stt()`, AgentKit uses `FengmingSTT` by default for `Area.CN` clients and `AresSTT` for global clients. -| Client area | STT classes | LLM classes | TTS classes | Avatar classes | -|---|---|---|---|---| -| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | -| `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` | +| Client area | STT classes | LLM classes | MLLM classes | TTS classes | Avatar classes | +|---|---|---|---|---|---| +| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | +| `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `QwenOmni` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` | Global client example: diff --git a/docs/index.md b/docs/index.md index d94d3df..11f3196 100644 --- a/docs/index.md +++ b/docs/index.md @@ -17,12 +17,12 @@ The Agora Conversational AI Python SDK lets you build voice-powered AI agents on **Cascading flow** uses ASR -> LLM -> TTS and supports the broadest set of vendor combinations. -**MLLM flow** uses a multimodal model such as OpenAI Realtime, Gemini Live, Vertex AI, or xAI Grok for end-to-end audio. +**MLLM flow** uses a multimodal model such as OpenAI Realtime, Azure OpenAI Realtime, Gemini Live, Vertex AI, xAI Grok, or Qwen Omni for end-to-end audio. ## Start here - Start with [Quick Start](./getting-started/quick-start.md). It shows the baseline app-credentials setup and starts a cascading ASR -> LLM -> TTS agent. -- Use [MLLM Flow](./guides/mllm-flow.md) when your agent uses one realtime multimodal model, such as OpenAI Realtime, Gemini Live, Vertex AI, or xAI Grok. +- Use [MLLM Flow](./guides/mllm-flow.md) when your agent uses one supported realtime multimodal model. - Use [Cascading Flow](./guides/cascading-flow.md) for more examples of the default ASR -> LLM -> TTS flow, including provider-specific configuration. ## How the SDK is organized @@ -46,7 +46,7 @@ The Agora Conversational AI Python SDK lets you build voice-powered AI agents on | [AgentSession](./concepts/session.md) | Manage the agent lifecycle | | [Vendors](./concepts/vendors.md) | Browse all LLM, TTS, STT, MLLM, and Avatar providers | | [Cascading Flow](./guides/cascading-flow.md) | Build an ASR -> LLM -> TTS pipeline | -| [MLLM Flow](./guides/mllm-flow.md) | Use OpenAI Realtime, Gemini Live, Vertex AI, or xAI Grok for end-to-end audio | +| [MLLM Flow](./guides/mllm-flow.md) | Use a global or CN realtime multimodal model for end-to-end audio | | [Avatars](./guides/avatars.md) | Add a digital avatar with LiveAvatar, Akool, Anam, Generic Avatar, SenseTime, or Spatius | | [Regional Routing](./guides/regional-routing.md) | Route requests to the nearest region | | [Error Handling](./guides/error-handling.md) | Handle API errors with ApiError | diff --git a/docs/reference/vendors.md b/docs/reference/vendors.md index 8741453..9ab2250 100644 --- a/docs/reference/vendors.md +++ b/docs/reference/vendors.md @@ -10,17 +10,18 @@ All vendor classes are available from `agora_agent`: ```python -from agora_agent import OpenAI, ElevenLabsTTS, DeepgramTTS, DeepgramSTT, OpenAIRealtime, XaiGrok, GenericAvatar +from agora_agent import AzureOpenAIRealtime, DeepgramSTT, OpenAI, TypecastTTS +from agora_agent.cn import QwenOmni ``` ## Recommended vendors by area Construct vendors directly from `agora_agent`, then bind a client with `Agent(client=client, ...)`. The bound client selects `CNAgent` or `GlobalAgent` for IDE hints based on `area`, but does not restrict which vendor classes you can configure. -| Area | STT classes | LLM classes | TTS classes | Avatar classes | -|---|---|---|---|---| -| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | -| `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` | +| Area | STT classes | LLM classes | MLLM classes | TTS classes | Avatar classes | +|---|---|---|---|---|---| +| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | +| `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `QwenOmni` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` | Global example: @@ -411,6 +412,16 @@ AgentKit serializes `credential_mode` at the top level of the Rime TTS configura | `additional_params` | `Dict[str, Any]` | No | `None` | Additional Mistral TTS parameters, flattened into `params` | | `skip_patterns` | `List[int]` | No | `None` | Skip patterns | +### `TypecastTTS` + +| Parameter | Type | Required | Default | Description | +|---|---|---|---|---| +| `api_key` | `str` | Yes | — | Typecast API key | +| `voice_id` | `str` | Yes | — | Typecast voice identifier | +| `model` | `str` | Yes | — | Typecast TTS model name, for example `ssfm-v30` | +| `additional_params` | `Dict[str, Any]` | No | `None` | Additional Typecast parameters, flattened into `params` | +| `skip_patterns` | `List[int]` | No | `None` | Skip patterns | + ### `MurfTTS` | Parameter | Type | Required | Default | Description | @@ -550,6 +561,7 @@ For `nova-2` and `nova-3`, omit `api_key` to use Agora-managed credentials. For | Parameter | Type | Required | Default | Description | |---|---|---|---|---| +| `keywords` | `List[str]` | No | `None` | Hotwords that improve ASR accuracy | | `additional_params` | `Dict[str, Any]` | No | `None` | Additional parameters | ### `SarvamSTT` @@ -700,7 +712,9 @@ All CN TTS vendor classes support `skip_patterns` and `additional_params`. #### `FengmingSTT` -No constructor parameters. Use `FengmingSTT()`. +| Parameter | Type | Required | Default | Description | +|---|---|---|---|---| +| `keywords` | `List[str]` | No | `None` | Hotwords that improve ASR accuracy | #### `XfyunSTT` @@ -789,6 +803,46 @@ No constructor parameters. Use `FengmingSTT()`. | `params` | `Dict[str, Any]` | No | `None` | Additional parameters | | `turn_detection` | `MllmTurnDetectionConfig` | No | `None` | MLLM turn detection configuration; overrides top-level `turn_detection` | +### `AzureOpenAIRealtime` + +Global Azure OpenAI Realtime vendor (`mllm.vendor`: `"azure"`). + +| Parameter | Type | Required | Default | Description | +|---|---|---|---|---| +| `api_key` | `str` | Yes | — | Azure OpenAI API key | +| `url` | `str` | Yes | — | Azure OpenAI Realtime WebSocket URL | +| `model` | `str` | No | `None` | Model or deployment name | +| `voice` | `str` | No | `None` | Voice identifier | +| `instructions` | `str` | No | `None` | System instructions | +| `input_audio_transcription` | `Dict[str, Any]` | No | `None` | Audio transcription settings | +| `max_history` | `int` | No | `None` | Number of conversation history messages to cache; Azure-only MLLM field | +| `greeting_message` | `str` | No | `None` | Greeting message | +| `failure_message` | `str` | No | `None` | Message played when the model call fails | +| `output_modalities` | `List[str]` | No | `None` | Output modalities | +| `messages` | `List[Dict]` | No | `None` | Conversation messages | +| `params` | `Dict[str, Any]` | No | `None` | Additional Azure OpenAI parameters | +| `turn_detection` | `MllmTurnDetectionConfig` | Yes | — | Required MLLM turn detection configuration; overrides top-level `turn_detection` | + +### `QwenOmni` + +CN Alibaba Cloud Qwen Omni Realtime vendor (`mllm.vendor`: `"qwen_omni"`). Import it with `from agora_agent.cn import QwenOmni`. + +| Parameter | Type | Required | Default | Description | +|---|---|---|---|---| +| `api_key` | `str` | Yes | — | Alibaba Cloud Model Studio API key | +| `url` | `str` | No | `None` | Qwen Omni Realtime WebSocket URL | +| `model` | `str` | No | `None` | Qwen Omni Realtime model name | +| `voice` | `str` | No | `None` | Voice identifier | +| `instructions` | `str` | No | `None` | System instructions | +| `input_audio_transcription` | `Dict[str, Any]` | No | `None` | Audio transcription settings | +| `greeting_message` | `str` | No | `None` | Greeting message | +| `failure_message` | `str` | No | `None` | Message played when the model call fails | +| `input_modalities` | `List[str]` | No | `None` | Input modalities | +| `output_modalities` | `List[str]` | No | `None` | Output modalities | +| `messages` | `List[Dict]` | No | `None` | Conversation messages | +| `params` | `Dict[str, Any]` | No | `None` | Additional Qwen Omni parameters | +| `turn_detection` | `MllmTurnDetectionConfig` | Yes | — | Required MLLM turn detection configuration; overrides top-level `turn_detection` | + ### `GeminiLive` | Parameter | Type | Required | Default | Description | diff --git a/src/agora_agent/agentkit/__init__.py b/src/agora_agent/agentkit/__init__.py index 7ed3577..7afe863 100644 --- a/src/agora_agent/agentkit/__init__.py +++ b/src/agora_agent/agentkit/__init__.py @@ -156,6 +156,7 @@ AresSTT, AssemblyAISTT, AzureOpenAI, + AzureOpenAIRealtime, BaseAvatar, BaseLLM, BaseMLLM, @@ -195,9 +196,11 @@ OpenAITTS, GenericTTS, RimeTTS, + QwenOmni, SampleRate, SarvamSTT, SarvamTTS, + TypecastTTS, SpeechmaticsSTT, XaiSTT, XaiTTS, @@ -403,6 +406,7 @@ "MiniMaxTTS", "MiniMaxCNTTS", "MistralTTS", + "TypecastTTS", "TencentTTS", "BytedanceTTS", "CosyVoiceTTS", @@ -428,7 +432,9 @@ "XfyunDialectSTT", "XfyunSTT", "OpenAIRealtime", + "AzureOpenAIRealtime", "GeminiLive", + "QwenOmni", "VertexAI", "XaiGrok", "HeyGenAvatar", diff --git a/src/agora_agent/agentkit/regional_agent.py b/src/agora_agent/agentkit/regional_agent.py index d71a7b6..0a91dd2 100644 --- a/src/agora_agent/agentkit/regional_agent.py +++ b/src/agora_agent/agentkit/regional_agent.py @@ -2,7 +2,7 @@ import typing from .agent import Agent -from .vendors.base import BaseAvatar, BaseLLM, BaseSTT, BaseTTS +from .vendors.base import BaseAvatar, BaseLLM, BaseMLLM, BaseSTT, BaseTTS from .vendors.cn import ( AliyunLLM, BytedanceDuplexTTS, @@ -14,6 +14,7 @@ MiniMaxTTS as MiniMaxCNTTS, MicrosoftSTT as MicrosoftCNSTT, MicrosoftTTS as MicrosoftCNTTS, + QwenOmni, SenseTimeAvatar, SpatiusAvatar, StepFunTTS, @@ -47,6 +48,7 @@ OpenAI, VertexAILLM, ) +from .vendors.mllm import AzureOpenAIRealtime, GeminiLive, OpenAIRealtime, VertexAI, XaiGrok from .vendors.tts import ( AmazonTTS, CartesiaTTS, @@ -64,6 +66,7 @@ GenericTTS, RimeTTS, SarvamTTS, + TypecastTTS, XaiTTS, ) from .vendors.avatar import AkoolAvatar, AnamAvatar, GenericAvatar, HeyGenAvatar, LiveAvatarAvatar @@ -71,6 +74,7 @@ CNSTT = typing.Union[TencentSTT, FengmingSTT, MicrosoftCNSTT, XfyunSTT, XfyunBigModelSTT, XfyunDialectSTT] CNTTS = typing.Union[MiniMaxCNTTS, TencentTTS, BytedanceTTS, MicrosoftCNTTS, CosyVoiceTTS, BytedanceDuplexTTS, StepFunTTS, GenericTTS] CNLLM = typing.Union[AliyunLLM, BytedanceLLM, DeepSeekLLM, TencentLLM] +CNMLLM = QwenOmni CNAvatar = typing.Union[SenseTimeAvatar, SpatiusAvatar] GlobalSTT = typing.Union[ @@ -103,6 +107,7 @@ DeepgramTTS, GradiumTTS, MistralTTS, + TypecastTTS, ] GlobalLLM = typing.Union[ OpenAI, @@ -115,6 +120,7 @@ Dify, CustomLLM, ] +GlobalMLLM = typing.Union[OpenAIRealtime, AzureOpenAIRealtime, GeminiLive, VertexAI, XaiGrok] GlobalAvatar = typing.Union[AkoolAvatar, LiveAvatarAvatar, AnamAvatar, GenericAvatar, HeyGenAvatar] @@ -128,6 +134,9 @@ def with_llm(self, vendor: BaseLLM) -> "CNAgent": def with_tts(self, vendor: BaseTTS) -> "CNAgent": return typing.cast("CNAgent", super().with_tts(vendor)) + def with_mllm(self, vendor: BaseMLLM) -> "CNAgent": + return typing.cast("CNAgent", super().with_mllm(vendor)) + def with_avatar(self, vendor: BaseAvatar) -> "CNAgent": return typing.cast("CNAgent", super().with_avatar(vendor)) @@ -142,6 +151,9 @@ def with_llm(self, vendor: BaseLLM) -> "GlobalAgent": def with_tts(self, vendor: BaseTTS) -> "GlobalAgent": return typing.cast("GlobalAgent", super().with_tts(vendor)) + def with_mllm(self, vendor: BaseMLLM) -> "GlobalAgent": + return typing.cast("GlobalAgent", super().with_mllm(vendor)) + def with_avatar(self, vendor: BaseAvatar) -> "GlobalAgent": return typing.cast("GlobalAgent", super().with_avatar(vendor)) diff --git a/src/agora_agent/agentkit/vendors/__init__.py b/src/agora_agent/agentkit/vendors/__init__.py index 9bd07fa..c69669d 100644 --- a/src/agora_agent/agentkit/vendors/__init__.py +++ b/src/agora_agent/agentkit/vendors/__init__.py @@ -13,7 +13,7 @@ ) from .avatar import AkoolAvatar, AnamAvatar, GenericAvatar, HeyGenAvatar, LiveAvatarAvatar from .llm import AmazonBedrock, Anthropic, AzureOpenAI, CustomLLM, Dify, Gemini, Groq, OpenAI, VertexAILLM -from .mllm import GeminiLive, OpenAIRealtime, VertexAI, XaiGrok +from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIRealtime, VertexAI, XaiGrok from .cn import ( AliyunLLM, BytedanceDuplexTTS, @@ -25,6 +25,7 @@ MicrosoftSTT as MicrosoftCNSTT, MicrosoftTTS as MicrosoftCNTTS, MiniMaxTTS as MiniMaxCNTTS, + QwenOmni, SenseTimeAvatar, SpatiusAvatar, StepFunTTS, @@ -64,6 +65,7 @@ GenericTTS, RimeTTS, SarvamTTS, + TypecastTTS, XaiTTS, ) @@ -102,6 +104,7 @@ "MiniMaxTTS", "MiniMaxCNTTS", "MistralTTS", + "TypecastTTS", "MurfTTS", "SarvamTTS", "GenericTTS", @@ -133,7 +136,9 @@ "DeepSeekLLM", "TencentLLM", "OpenAIRealtime", + "AzureOpenAIRealtime", "GeminiLive", + "QwenOmni", "VertexAI", "XaiGrok", "HeyGenAvatar", diff --git a/src/agora_agent/agentkit/vendors/catalog.py b/src/agora_agent/agentkit/vendors/catalog.py index f44b94e..25c99df 100644 --- a/src/agora_agent/agentkit/vendors/catalog.py +++ b/src/agora_agent/agentkit/vendors/catalog.py @@ -16,6 +16,7 @@ OpenAI, VertexAILLM, ) +from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIRealtime, VertexAI, XaiGrok from .stt import ( AmazonSTT, AresSTT, @@ -45,6 +46,7 @@ GenericTTS, RimeTTS, SarvamTTS, + TypecastTTS, XaiTTS, ) @@ -55,11 +57,13 @@ def __init__( *, asr: typing.Mapping[str, typing.Any], llm: typing.Mapping[str, typing.Any], + mllm: typing.Mapping[str, typing.Any], tts: typing.Mapping[str, typing.Any], avatar: typing.Mapping[str, typing.Any], ) -> None: self.asr = dict(asr) self.llm = dict(llm) + self.mllm = dict(mllm) self.tts = dict(tts) self.avatar = dict(avatar) @@ -88,6 +92,13 @@ def __init__( "dify": Dify, "custom": CustomLLM, }, + mllm={ + "openai": OpenAIRealtime, + "azure": AzureOpenAIRealtime, + "gemini": GeminiLive, + "vertexai": VertexAI, + "xai": XaiGrok, + }, tts={ "microsoft": MicrosoftTTS, "elevenlabs": ElevenLabsTTS, @@ -106,6 +117,7 @@ def __init__( "deepgram": DeepgramTTS, "gradium": GradiumTTS, "mistral": MistralTTS, + "typecast": TypecastTTS, }, avatar={ "akool": AkoolAvatar, @@ -131,6 +143,9 @@ def __init__( "deepseek": cn_vendors.DeepSeekLLM, "tencent": cn_vendors.TencentLLM, }, + mllm={ + "qwen_omni": cn_vendors.QwenOmni, + }, tts={ "minimax": cn_vendors.MiniMaxTTS, "tencent": cn_vendors.TencentTTS, diff --git a/src/agora_agent/agentkit/vendors/cn.py b/src/agora_agent/agentkit/vendors/cn.py index ee888c0..fbefa30 100644 --- a/src/agora_agent/agentkit/vendors/cn.py +++ b/src/agora_agent/agentkit/vendors/cn.py @@ -4,8 +4,9 @@ from pydantic import ConfigDict, Field, model_validator +from ...types.mllm_turn_detection import MllmTurnDetection from .avatar import BaseAvatar -from .base import BaseLLM +from .base import BaseLLM, BaseMLLM from .llm import ( _OPENAI_MANAGED_MODELS, LlmGreetingConfigs, @@ -43,8 +44,13 @@ def to_config(self) -> Dict[str, Any]: class FengmingSTT(_BaseSTTCompat): model_config = ConfigDict(extra="forbid") + keywords: Optional[List[str]] = Field(default=None, description="Hotwords that improve ASR accuracy") + def to_config(self) -> Dict[str, Any]: - return {"vendor": "fengming"} + config: Dict[str, Any] = {"vendor": "fengming"} + if self.keywords is not None: + config["params"] = {"keywords": self.keywords} + return config class XfyunSTT(_BaseSTTCompat): @@ -788,6 +794,60 @@ def to_config(self) -> Dict[str, Any]: return config +class QwenOmni(BaseMLLM): + """Alibaba Cloud Qwen Omni Realtime MLLM vendor (`mllm.vendor`: ``qwen_omni``).""" + + model_config = ConfigDict(extra="forbid") + + api_key: str = Field(..., description="Alibaba Cloud Model Studio API key") + url: Optional[str] = Field(default=None, description="Qwen Omni Realtime WebSocket URL") + model: Optional[str] = Field(default=None, description="Qwen Omni Realtime model name") + voice: Optional[str] = Field(default=None, description="Voice identifier") + instructions: Optional[str] = Field(default=None, description="System instructions") + input_audio_transcription: Optional[Dict[str, Any]] = Field( + default=None, description="Audio transcription settings" + ) + greeting_message: Optional[str] = Field(default=None, description="Agent greeting message") + input_modalities: Optional[List[str]] = Field(default=None, description="Input modalities") + output_modalities: Optional[List[str]] = Field(default=None, description="Output modalities") + messages: Optional[List[Dict[str, Any]]] = Field(default=None, description="Conversation messages") + params: Optional[Dict[str, Any]] = Field(default=None, description="Additional Qwen Omni parameters") + turn_detection: MllmTurnDetection = Field(..., description="MLLM turn detection configuration") + failure_message: Optional[str] = Field(default=None, description="Message played on failure") + + def to_config(self) -> Dict[str, Any]: + inner_params: Dict[str, Any] = dict(self.params or {}) + if self.model is not None: + inner_params["model"] = self.model + if self.voice is not None: + inner_params["voice"] = self.voice + if self.instructions is not None: + inner_params["instructions"] = self.instructions + if self.input_audio_transcription is not None: + inner_params["input_audio_transcription"] = self.input_audio_transcription + + config: Dict[str, Any] = { + "vendor": "qwen_omni", + "api_key": self.api_key, + } + if self.url is not None: + config["url"] = self.url + if inner_params: + config["params"] = inner_params + if self.greeting_message is not None: + config["greeting_message"] = self.greeting_message + if self.input_modalities is not None: + config["input_modalities"] = self.input_modalities + if self.output_modalities is not None: + config["output_modalities"] = self.output_modalities + if self.messages is not None: + config["messages"] = self.messages + if self.failure_message is not None: + config["failure_message"] = self.failure_message + config["turn_detection"] = self.turn_detection + return config + + class SenseTimeAvatar(BaseAvatar): model_config = ConfigDict(extra="forbid", populate_by_name=True) diff --git a/src/agora_agent/agentkit/vendors/mllm.py b/src/agora_agent/agentkit/vendors/mllm.py index 4bc74cd..01563c4 100644 --- a/src/agora_agent/agentkit/vendors/mllm.py +++ b/src/agora_agent/agentkit/vendors/mllm.py @@ -70,6 +70,61 @@ def to_config(self) -> Dict[str, Any]: return config +class AzureOpenAIRealtime(BaseMLLM): + """Azure OpenAI Realtime MLLM vendor (`mllm.vendor`: ``azure``).""" + + model_config = ConfigDict(extra="forbid") + + api_key: str = Field(..., description="Azure OpenAI API key") + url: str = Field(..., description="Azure OpenAI Realtime WebSocket URL") + model: Optional[str] = Field(default=None, description="Azure OpenAI Realtime model or deployment name") + voice: Optional[str] = Field(default=None, description="Voice identifier") + instructions: Optional[str] = Field(default=None, description="System instructions") + input_audio_transcription: Optional[Dict[str, Any]] = Field( + default=None, description="Audio transcription settings" + ) + max_history: Optional[int] = Field( + default=None, gt=0, description="Number of conversation history messages to cache" + ) + greeting_message: Optional[str] = Field(default=None, description="Agent greeting message") + output_modalities: Optional[List[str]] = Field(default=None, description="Output modalities") + messages: Optional[List[Dict[str, Any]]] = Field(default=None, description="Conversation messages") + params: Optional[Dict[str, Any]] = Field(default=None, description="Additional Azure OpenAI parameters") + turn_detection: MllmTurnDetectionConfig = Field(..., description="MLLM turn detection configuration") + failure_message: Optional[str] = Field(default=None, description="Message played on failure") + + def to_config(self) -> Dict[str, Any]: + inner_params: Dict[str, Any] = dict(self.params or {}) + if self.model is not None: + inner_params["model"] = self.model + if self.voice is not None: + inner_params["voice"] = self.voice + if self.instructions is not None: + inner_params["instructions"] = self.instructions + if self.input_audio_transcription is not None: + inner_params["input_audio_transcription"] = self.input_audio_transcription + + config: Dict[str, Any] = { + "vendor": "azure", + "api_key": self.api_key, + "url": self.url, + } + if inner_params: + config["params"] = inner_params + if self.max_history is not None: + config["max_history"] = self.max_history + if self.greeting_message is not None: + config["greeting_message"] = self.greeting_message + if self.output_modalities is not None: + config["output_modalities"] = self.output_modalities + if self.messages is not None: + config["messages"] = self.messages + if self.failure_message is not None: + config["failure_message"] = self.failure_message + config["turn_detection"] = self.turn_detection + return config + + # xAI MLLM: use XaiGrok (product name, mllm.vendor "xai"). Do not use XaiRealtime—that name # is deprecated and reserved naming for future XaiSTT / XaiTTS cascading vendors. diff --git a/src/agora_agent/agentkit/vendors/namespaces.py b/src/agora_agent/agentkit/vendors/namespaces.py index 1f85e09..001b374 100644 --- a/src/agora_agent/agentkit/vendors/namespaces.py +++ b/src/agora_agent/agentkit/vendors/namespaces.py @@ -13,6 +13,7 @@ OpenAI, VertexAILLM, ) +from .mllm import AzureOpenAIRealtime, GeminiLive, OpenAIRealtime, VertexAI, XaiGrok from .stt import ( AmazonSTT, AresSTT, @@ -42,6 +43,7 @@ GenericTTS, RimeTTS, SarvamTTS, + TypecastTTS, XaiTTS, ) @@ -87,6 +89,18 @@ class CNLLMVendors: tencent = cn_vendors.TencentLLM +class GlobalMLLMVendors: + openai = OpenAIRealtime + azure = AzureOpenAIRealtime + gemini = GeminiLive + vertexai = VertexAI + xai = XaiGrok + + +class CNMLLMVendors: + qwen_omni = cn_vendors.QwenOmni + + class GlobalTTSVendors: microsoft = MicrosoftTTS elevenlabs = ElevenLabsTTS @@ -105,6 +119,7 @@ class GlobalTTSVendors: deepgram = DeepgramTTS gradium = GradiumTTS mistral = MistralTTS + typecast = TypecastTTS class CNTTSVendors: @@ -134,12 +149,14 @@ class CNAvatarVendors: class GlobalVendors: stt: GlobalSTTVendors llm: GlobalLLMVendors + mllm: GlobalMLLMVendors tts: GlobalTTSVendors avatar: GlobalAvatarVendors def __init__(self) -> None: self.stt = GlobalSTTVendors() self.llm = GlobalLLMVendors() + self.mllm = GlobalMLLMVendors() self.tts = GlobalTTSVendors() self.avatar = GlobalAvatarVendors() @@ -147,11 +164,13 @@ def __init__(self) -> None: class CNVendors: stt: CNSTTVendors llm: CNLLMVendors + mllm: CNMLLMVendors tts: CNTTSVendors avatar: CNAvatarVendors def __init__(self) -> None: self.stt = CNSTTVendors() self.llm = CNLLMVendors() + self.mllm = CNMLLMVendors() self.tts = CNTTSVendors() self.avatar = CNAvatarVendors() diff --git a/src/agora_agent/agentkit/vendors/region.py b/src/agora_agent/agentkit/vendors/region.py index 1ade190..721f7e3 100644 --- a/src/agora_agent/agentkit/vendors/region.py +++ b/src/agora_agent/agentkit/vendors/region.py @@ -58,6 +58,16 @@ "deepgram", "gradium", "mistral", + "typecast", +) + +CN_MLLM_VENDORS: typing.Tuple[str, ...] = ("qwen_omni",) +GLOBAL_MLLM_VENDORS: typing.Tuple[str, ...] = ( + "openai", + "azure", + "gemini", + "vertexai", + "xai", ) CN_LLM_VENDORS: typing.Tuple[str, ...] = ( @@ -98,11 +108,13 @@ def allowed_vendors_for_scope(scope: AreaScope) -> typing.Dict[str, typing.Tuple "asr": CN_ASR_VENDORS, "tts": CN_TTS_VENDORS, "llm": CN_LLM_VENDORS, + "mllm": CN_MLLM_VENDORS, "avatar": CN_AVATAR_VENDORS, } return { "asr": GLOBAL_ASR_VENDORS, "tts": GLOBAL_TTS_VENDORS, "llm": GLOBAL_LLM_VENDORS, + "mllm": GLOBAL_MLLM_VENDORS, "avatar": GLOBAL_AVATAR_VENDORS, } diff --git a/src/agora_agent/agentkit/vendors/stt.py b/src/agora_agent/agentkit/vendors/stt.py index 4a79fb6..ab610a3 100644 --- a/src/agora_agent/agentkit/vendors/stt.py +++ b/src/agora_agent/agentkit/vendors/stt.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional +from typing import Any, Dict, List, Optional from pydantic import ConfigDict, Field, model_validator @@ -216,12 +216,17 @@ def to_config(self) -> Dict[str, Any]: class AresSTT(BaseSTT): model_config = ConfigDict(extra="forbid") + keywords: Optional[List[str]] = Field(default=None, description="Hotwords that improve ASR accuracy") additional_params: Optional[Dict[str, Any]] = Field(default=None) def to_config(self) -> Dict[str, Any]: + params: Dict[str, Any] = dict(self.additional_params or {}) + if self.keywords is not None: + params["keywords"] = self.keywords + config: Dict[str, Any] = {"vendor": "ares"} - if self.additional_params: - config["params"] = self.additional_params + if params: + config["params"] = params return config diff --git a/src/agora_agent/agentkit/vendors/tts.py b/src/agora_agent/agentkit/vendors/tts.py index bb99e04..f357ace 100644 --- a/src/agora_agent/agentkit/vendors/tts.py +++ b/src/agora_agent/agentkit/vendors/tts.py @@ -507,6 +507,31 @@ def to_config(self) -> Dict[str, Any]: return result +class TypecastTTS(BaseTTS): + model_config = ConfigDict(extra="forbid") + + api_key: str = Field(..., description="Typecast API key") + voice_id: str = Field(..., description="Typecast voice identifier") + model: str = Field(..., description="Typecast TTS model name") + additional_params: Optional[Dict[str, Any]] = Field(default=None, description="Additional Typecast TTS parameters") + skip_patterns: Optional[List[int]] = Field(default=None) + + def to_config(self) -> Dict[str, Any]: + params: Dict[str, Any] = dict(self.additional_params or {}) + params.update( + { + "api_key": self.api_key, + "voice_id": self.voice_id, + "model": self.model, + } + ) + + result: Dict[str, Any] = {"vendor": "typecast", "params": params} + if self.skip_patterns is not None: + result["skip_patterns"] = self.skip_patterns + return result + + class SarvamTTS(BaseTTS): model_config = ConfigDict(extra="forbid") diff --git a/src/agora_agent/cn.py b/src/agora_agent/cn.py index 9df1a0f..da91d7a 100644 --- a/src/agora_agent/cn.py +++ b/src/agora_agent/cn.py @@ -4,7 +4,7 @@ Import CN vendors explicitly from this module so they stay separate from the global top-level namespace:: - from agora_agent.cn import AliyunLLM, MiniMaxTTS, TencentSTT + from agora_agent.cn import AliyunLLM, MiniMaxTTS, QwenOmni, TencentSTT ``MicrosoftSTT`` / ``MicrosoftTTS`` / ``MiniMaxTTS`` here are the CN variants; the global variants of those names are available from the top-level package @@ -22,6 +22,7 @@ MicrosoftSTT, MicrosoftTTS, MiniMaxTTS, + QwenOmni, SenseTimeAvatar, SpatiusAvatar, StepFunTTS, @@ -54,6 +55,8 @@ "BytedanceLLM", "DeepSeekLLM", "TencentLLM", + # MLLM + "QwenOmni", # Avatar "SenseTimeAvatar", "SpatiusAvatar", diff --git a/tests/custom/test_request_body.py b/tests/custom/test_request_body.py index e3117e5..4034ba3 100644 --- a/tests/custom/test_request_body.py +++ b/tests/custom/test_request_body.py @@ -19,9 +19,11 @@ from types import SimpleNamespace import pytest +from pydantic import ValidationError from agora_agent import ( Agent, + Area, AmazonBedrock, AmazonSTT, AmazonTTS, @@ -29,6 +31,7 @@ AresSTT, AssemblyAISTT, AzureOpenAI, + AzureOpenAIRealtime, CartesiaTTS, CredentialMode, CustomLLM, @@ -61,6 +64,7 @@ ) from agora_agent.agentkit import AgentSession from agora_agent.agentkit.presets import resolve_session_presets +from agora_agent.cn import QwenOmni from test_helpers import test_client @@ -1270,6 +1274,80 @@ def test_byok_openai_realtime_mllm_params() -> None: assert props["mllm"]["params"]["voice"] == "coral" +def test_byok_azure_openai_realtime_mllm_params() -> None: + agent = Agent(test_client()).with_mllm( + AzureOpenAIRealtime( + url="AZURE_URL", + api_key="APIKEY", + messages=[ + {"role": "user", "content": "can you please count to 3 after greeting。"} + ], + params={ + "instructions": "You are a Conversational AI Agent, developed by Agora.", + "model": "gpt-realtime-2", + "voice": "alloy", + }, + output_modalities=["audio"], + max_history=20, + greeting_message="Hey There Sir", + turn_detection={"mode": "server_vad"}, + ) + ) + + props = build_properties(agent) + + assert props["mllm"] == { + "enable": True, + "url": "AZURE_URL", + "api_key": "APIKEY", + "messages": [ + {"role": "user", "content": "can you please count to 3 after greeting。"} + ], + "params": { + "instructions": "You are a Conversational AI Agent, developed by Agora.", + "model": "gpt-realtime-2", + "voice": "alloy", + }, + "output_modalities": ["audio"], + "max_history": 20, + "greeting_message": "Hey There Sir", + "vendor": "azure", + "turn_detection": {"mode": "server_vad"}, + } + + +def test_azure_openai_realtime_rejects_input_modalities() -> None: + with pytest.raises(ValidationError): + AzureOpenAIRealtime( + url="AZURE_URL", + api_key="APIKEY", + input_modalities=["audio"], + turn_detection={"mode": "server_vad"}, + ) + + +def test_azure_and_qwen_mllm_require_turn_detection() -> None: + with pytest.raises(ValidationError): + AzureOpenAIRealtime(url="AZURE_URL", api_key="APIKEY") # type: ignore[call-arg] + + with pytest.raises(ValidationError): + QwenOmni(api_key="APIKEY") # type: ignore[call-arg] + + +def test_byok_qwen_omni_mllm_requires_and_serializes_turn_detection() -> None: + agent = Agent(test_client(area=Area.CN)).with_mllm( + QwenOmni( + api_key="APIKEY", + turn_detection={"mode": "server_vad"}, + ) + ) + + props = build_properties(agent) + + assert props["mllm"]["vendor"] == "qwen_omni" + assert props["mllm"]["turn_detection"] == {"mode": "server_vad"} + + def test_byok_gemini_live_mllm_params() -> None: agent = Agent(test_client()).with_mllm( GeminiLive(api_key="gemini-key", model="gemini-live-2.5-flash")