From 8c31df23a4a6fb91037b5d2a6f6c07bbdae66a49 Mon Sep 17 00:00:00 2001 From: seymourtang Date: Tue, 1 Sep 2026 11:42:21 +0800 Subject: [PATCH] Enhance Gemini ASR support and remove preview endpoint - Added `GeminiSTT` to the standard AgentKit STT vendors, requiring `api_key` and `model`, with optional parameters for `language`, `word_timestamp`, and `sample_rate`. - Updated Gemini ASR to use the normal regional API endpoint and generated request validation. - Removed documentation and code related to the deprecated preview endpoint. - Preserved credential redaction for debug request output. --- changelog.md | 2 + docs/concepts/vendors.md | 3 +- docs/guides/preview-endpoint.md | 231 ---------- docs/guides/regional-routing.md | 2 +- docs/index.md | 1 - docs/reference/vendors.md | 24 +- src/agora_agent/__init__.py | 1 + src/agora_agent/agentkit/__init__.py | 2 + src/agora_agent/agentkit/agent.py | 34 +- src/agora_agent/agentkit/agent_session.py | 23 - src/agora_agent/agentkit/preview/__init__.py | 29 -- src/agora_agent/agentkit/preview/client.py | 118 ----- src/agora_agent/agentkit/preview/vendors.py | 110 ----- src/agora_agent/agentkit/regional_agent.py | 2 + src/agora_agent/agentkit/vendors/__init__.py | 2 + src/agora_agent/agentkit/vendors/catalog.py | 2 + .../agentkit/vendors/namespaces.py | 2 + src/agora_agent/agentkit/vendors/region.py | 1 + src/agora_agent/agentkit/vendors/stt.py | 36 ++ tests/custom/test_debug.py | 113 +++++ tests/custom/test_gemini_stt.py | 160 +++++++ tests/custom/test_preview.py | 403 ------------------ 22 files changed, 349 insertions(+), 952 deletions(-) delete mode 100644 docs/guides/preview-endpoint.md delete mode 100644 src/agora_agent/agentkit/preview/__init__.py delete mode 100644 src/agora_agent/agentkit/preview/client.py delete mode 100644 src/agora_agent/agentkit/preview/vendors.py create mode 100644 tests/custom/test_debug.py create mode 100644 tests/custom/test_gemini_stt.py delete mode 100644 tests/custom/test_preview.py diff --git a/changelog.md b/changelog.md index 907f7c7..cfa0438 100644 --- a/changelog.md +++ b/changelog.md @@ -9,10 +9,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). ### Added - **Inline REST LLM tools** — Global and CN LLM vendors now accept dictionary-based `tools` definitions and serialize them to `llm.tools`. Added public `LlmToolConfig`, `LlmToolExecutionConfig`, `LlmToolFunctionConfig`, `LlmToolFunctionParametersConfig`, and `LlmToolServerConfig` aliases. Tool execution must be enabled explicitly with `Agent.with_tools()`. +- **Gemini ASR** — Added `GeminiSTT` to the standard AgentKit STT vendors using the Fern-generated `GeminiAsrParams` schema. It requires `api_key` and `model`; `language`, `word_timestamp`, and `sample_rate` are optional. ### Changed - **Generated filler words configuration** — Generated filler word settings are now optional. The service can use default generator settings when `generated_config` is omitted, and `llm_provider`, `prompt`, and `fallback_strategy` may be omitted individually. +- **Gemini ASR routing** — Gemini ASR now uses the normal regional API endpoint and generated request validation. The temporary preview routing layer has been removed. ## [v2.7.2] — 2026-08-26 diff --git a/docs/concepts/vendors.md b/docs/concepts/vendors.md index 1fc868b..bed45b7 100644 --- a/docs/concepts/vendors.md +++ b/docs/concepts/vendors.md @@ -119,6 +119,7 @@ top-level `asr.keywords`. Both vendors also accept `additional_params`, serializ | `DeepgramSTT` | Deepgram | `model` for Agora-managed `nova-2`/`nova-3`; `api_key` for BYOK; `language?`, `keyterm?` | | `MicrosoftSTT` | Microsoft Azure | `key`, `region`, `language` | | `OpenAISTT` | OpenAI | `api_key` | +| `GeminiSTT` | Google Gemini | `api_key`, `model`; optional `language`, `word_timestamp`, `sample_rate` | | `GoogleSTT` | Google Cloud | `project_id`, `location`, `adc_credentials_string`, `language` | | `AmazonSTT` | Amazon Transcribe | `access_key`, `secret_key`, `region`, `language` | | `AssemblyAISTT` | AssemblyAI | `api_key`, `language` | @@ -146,8 +147,6 @@ from agora_agent import DeepgramSTT stt = DeepgramSTT(api_key='your-deepgram-key', language='en-US', model='nova-2') ``` -> **Preview providers** — `GeminiSTT` (ASR) lives in `agora_agent.agentkit.preview`. Sessions using it route to the preview gateway automatically. See [Preview Endpoint](../guides/preview-endpoint.md). - ## MLLM Vendors Used with `agent.with_mllm()` for the [MLLM flow](../guides/mllm-flow.md). These handle audio input and output end-to-end. diff --git a/docs/guides/preview-endpoint.md b/docs/guides/preview-endpoint.md deleted file mode 100644 index 8ea8859..0000000 --- a/docs/guides/preview-endpoint.md +++ /dev/null @@ -1,231 +0,0 @@ ---- -sidebar_position: 10 -title: Preview Endpoint -description: How AgentSession routes preview providers and pins the gateway's agora-feature gate header. ---- - -# Preview Endpoint - -Some providers ship on a preview gateway before they reach the production Conversational AI environment. Standard `Agora` and `AsyncAgora` clients detect these providers from the resolved start body and route that session automatically. - -Everything in `agentkit/preview/` is temporary by design. When these providers go GA, the package is deleted and the vendor classes move into `vendors/stt.py`. - -## Using a preview provider - -```python -import os - -from agora_agent import Agora, Area -from agora_agent.agentkit import Agent -from agora_agent.agentkit import Gemini, GoogleTTS -from agora_agent.agentkit.preview import GeminiSTT - -client = Agora( - area=Area.US, - app_id=os.environ["AGORA_APP_ID"], - app_certificate=os.environ["AGORA_APP_CERTIFICATE"], -) - -google_api_key = os.environ["GOOGLE_API_KEY"] -session = ( - Agent(client=client) - .with_stt(GeminiSTT(api_key=google_api_key, language_codes=["en-US"])) - .with_llm(Gemini(api_key=google_api_key, model="gemini-2.0-flash")) - .with_tts(GoogleTTS( - key=google_api_key, - voice_name="en-US-Chirp3-HD-Charon", - language_code="en-US", - )) - .create_session(channel="demo", agent_uid="1", remote_uids=["100"]) -) -agent_id = session.start() -``` - -`AsyncAgora` behaves the same, with `await session.start()`. - -Routing is session-scoped. Preview session calls use the preview host and pinned gate header; GA sessions created from the same client continue using the production regional endpoint. - -## The gate header - -The gateway routes preview traffic on a single request header: - -``` -agora-feature: gemini-live -``` - -| Constant | Value | -| ----------------------------- | --------------- | -| `PREVIEW_FEATURE_HEADER` | `agora-feature` | -| `PreviewFeatures.GEMINI_LIVE` | `gemini-live` | - -### The header is not overridable - -For preview sessions, the gate header is merged **after** caller-supplied client `headers`, so custom headers cannot drop or blank it: - -```python -client = Agora( - area=Area.US, - app_id=..., - app_certificate=..., - headers={"agora-feature": "", "x-custom": "kept"}, -) -# Requests still send agora-feature: gemini-live, and x-custom: kept -``` - -This ordering is deliberate and load-bearing. A preview request that loses the header is not rejected — it routes to the production environment, where the preview providers do not exist. - -The header rides every session verb, not just `start()` — `say`, `interrupt`, `think`, `update`, `get_history`, `get_info`, `get_turns`, and `stop`. The top-level `Agora.stop_agent()` and `AsyncAgora.stop_agent()` methods remain production-only because they do not carry session routing state. - -## Preview providers bypass request validation - -The generated request models mirror what production serves, so `asr.vendor = "gemini"` is not a member of the generated `Asr` union and pydantic rejects it. - -`_start_properties_from_mapping` catches that and, when `required_preview_features()` recognises the config, passes the mapping through unvalidated instead of raising. Production configs still get full validation — only configs the preview gateway understands take the bypass. - -This is worth knowing when adding a preview provider: if a new vendor is missing from the generated unions, register it in the detection sets in `preview/client.py` rather than editing generated code. - -## Intake node behavior - -The gateway decides where a request goes before it validates the body. That produces failure modes that look like outages but are routing problems. - -| Symptom | What it means | -| ---------------------------------------------------------- | -------------------------------------------------------------------------------------------------- | -| `503` `{"reason":"ServiceUnavailable"}` on `POST .../join` | The gate header was not recognised. The request died before validation, so the body is irrelevant. | -| `401` `Missing authorization header` | Routing worked; auth did not. | -| `404` `no Route matched with those values` | The base URL path is wrong. | -| `400` validation error | You are past the gate. The header is fine and the body is the problem. | - -The 503 is the one that misleads. It reads as a partner-side outage and invites waiting it out, when the fix is usually a one-line header change. - -Observed on the `gemini-live` rollout in August 2026, when the gateway had not yet been configured to route on `agora-feature` and every request fell through to a 503. That was fixed server-side on 2026-08-09, so the 503 is not currently reproducible — the mapping is recorded here because it is the failure signature a newly provisioned preview family is most likely to hit first. - -### Diagnosing without starting a billable agent - -Two probes, neither of which allocates an agent: - -1. **`GET .../v2/projects/{appId}/agents`** — a `200` proves host, auth, and routing are all healthy. If this succeeds while `join` fails, the problem is specific to the start path. -2. **A deliberately invalid start body** — send properties with no `llm` and no `mllm` at all. A `400` means you are past the gate; a `503` means you are not. This is what separates "my config is wrong" from "my header is wrong". - -### Known gap (as of 2026-08-09) - -A missing gate header is _intended_ to route to the production environment, where a preview config would fail. In practice an ungated start currently **succeeds** against the preview host, so the fallback is not observable from the client side. - -The SDK cannot control the intake node, and this does not affect SDK users because the header is pinned. It matters only for callers hitting the REST API directly, who may get a request that appears to succeed while silently landing in the wrong environment. Flag it to the endpoint owner rather than working around it in SDK code. - -## Session routing detection - -`required_preview_features()` reads the resolved request body rather than the vendor classes, so hand-written configs and preset resolution are covered too. It keys on `asr.vendor` — the vendor names served only by the preview endpoint, listed as `_PREVIEW_ASR_VENDORS` in `preview/client.py`. - -## Preview vendors - -| Class | Wire vendor | Model | -| ----------- | ----------------------- | ---------------------------- | -| `GeminiSTT` | `asr.vendor = "gemini"` | `gemini-3.5-transcribe-live` | - -`GeminiSTT` is an ASR stage, so it needs an LLM and a TTS vendor alongside it. The sample above uses Gemini LLM and Google TTS with the same Google API key. Mixing in other vendors is still valid; preview routing triggers only on `asr.vendor`. - -### ASR language selection - -Gemini Transcribe takes `params.language_codes`, an **array**, in place of the singular `params.language` other ASR vendors use. - -```python -# Auto-detect (the default) — language_codes is not sent at all -GeminiSTT(api_key=...) - -# Commit to one language -GeminiSTT(api_key=..., language_codes=["en-US"]) - -# Let the model choose between several -GeminiSTT(api_key=..., language_codes=["en-US", "es-ES"]) - -# Auto-detect, stated outright -GeminiSTT(api_key=..., language_codes=[]) -``` - -`language_codes` is omitted from the request unless you supply it, which is how the provider spells auto-detect. Omitting the field and sending `[]` mean the same thing. - -`GeminiSTT` takes **no `language` argument**. `Agent` always derives the top-level `asr.language` from the turn detection language — as it does for every STT vendor — so a vendor-level copy would be a no-op the builder overwrites. Set the interaction language on turn detection, and the transcription languages on `language_codes`; they are separate settings and neither feeds the other. The model forbids extra fields, so a stale `language=` argument raises rather than being silently dropped. - -| Setting | Where it belongs | What it controls | -| ----------------------- | ------------------------------ | --------------------------- | -| interaction language | turn detection `language` | top-level `asr.language` | -| transcription languages | `language_codes` on the vendor | `asr.params.language_codes` | - -`custom_vocabulary` biases recognition toward words the model would otherwise mis-hear — product names, jargon, proper nouns. It is omitted from the request entirely when unset. - -```python -GeminiSTT(api_key=..., custom_vocabulary=["Agora", "Kubernetes"]) -``` - -`word_timestamp` is also omitted unless you set it explicitly. Gemini does not support enabled word timestamps together with `custom_vocabulary`, so `to_config()` raises `ValueError` if both are requested. Explicit `word_timestamp=False` remains compatible with custom vocabulary. - -```python -GeminiSTT(api_key=..., word_timestamp=True) -``` - -## The vendor class is not the whole wire shape - -A vendor class emitting the right dict is not proof of what ships, because `Agent.to_properties` **also writes into the vendor config** after the vendor is done with it — using the Agora schema spellings, which are correct for every GA provider but need not match what a preview route reads. - -Every field the shared builder injects is listed below. Each one is a candidate for a silent mismatch on a preview route: - -| Category | Field | Written from | When | -| -------- | ------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------- | ------------------------------------------------- | -| `mllm` | `greeting_message` | agent-level `greeting` | only when the vendor left it unset (`setdefault`) | -| `mllm` | `failure_message` | agent-level `failure_message` | only when the vendor left it unset (`setdefault`) | -| `mllm` | `enable` | `with_mllm()` | always | -| `asr` | `language` | turn detection `language` | **always — overwrites any vendor value** | -| `llm` | `system_messages`, `greeting_message`, `greeting_configs`, `failure_message`, `max_history` | agent-level `instructions`, `greeting`, `greeting_configs`, `failure_message`, `max_history` | only when the vendor left them unset | - -So when a preview provider documents a field that appears in that table, putting it on the vendor class is not the whole fix. One of three things applies: - -- **The builder always overwrites it** (`asr.language`) — do not expose it on the vendor class at all; it would be an argument the builder silently discards. `GeminiSTT` takes no `language` for exactly this reason, and `extra="forbid"` turns a stale `language=` into an error rather than a no-op. -- **It is an Agora engine field rather than the provider's** (`failure_message`) — leave it in the schema spelling. -- **The preview route spells it differently** — the translation belongs in `preview/client.py`, applied at session start, so it disappears with `agentkit/preview/` at GA rather than leaving a vestigial hook in the shared builder. Nothing in this release needs one, but a future preview family may. - -### Verify against the request body, not the vendor output - -`to_config()` returning the right dict proves nothing about what ships, because the builder runs after it. Both checks are needed: - -1. A unit test on the vendor class, for the fields the vendor owns. -2. An **end-to-end test that starts a session against a mock transport and asserts on the captured request body** — the only check that sees the builder's injections. Every preview vendor has one in `tests/custom/test_preview.py`. - -The manual version is `debug=True`, which logs the fully resolved body. Diff it against the payload the provider documented, key by key. A value sitting under a name the route ignores fails **silently** — no error, no validation complaint, the agent simply never greets. That is the failure mode this whole section exists to catch, and it is invisible to type checking, to pydantic validation, and to any test that stops at the vendor class. - -Wire parity across the three SDKs is a hard requirement, so a change here lands in Python, TypeScript, and Go together, verified by diffing the serialized bodies. - -## Adding a future preview family - -Everything preview-only lives under `agentkit/preview/` so it can be deleted wholesale at GA. To add a family: - -1. Add an attribute to `PreviewFeatures` in `preview/client.py`. The value is what goes in the `agora-feature` header. -2. Add the vendor classes to `preview/vendors.py`, extending the same `BaseSTT` / `BaseMLLM` / `BaseLLM` bases as production vendors so the builder accepts them unchanged. -3. Register the detection keys — for an ASR family, the vendor name in `_PREVIEW_ASR_VENDORS` — so `required_preview_features()` recognises configs that need the new family. This is also what lets the config through the validation bypass described above. -4. Export from `preview/__init__.py`. Preview symbols stay in that subpackage rather than being re-exported from `agentkit/__init__.py`, which is what makes the GA deletion a single-directory change. -5. **Diff the resolved request body against the payload the provider documented**, not the vendor class output — see [The vendor class is not the whole wire shape](#the-vendor-class-is-not-the-whole-wire-shape). -6. Add an end-to-end test that starts a session and asserts on the captured body, alongside the vendor-class unit test. - -If the generated request models do not cover the new provider, rely on the bypass in `_start_properties_from_mapping` rather than editing generated code. Generated files are overwritten on the next Fern run; `.fernignore` protects `src/agora_agent/agentkit/`. - -At GA, delete `preview/` and move the vendor classes into `vendors/stt.py`. - -## Base URL - -``` -https://partner.ai.agora.io/preview/api/conversational-ai-agent -``` - -Request paths append to it exactly as they do in production — `POST {base}/v2/projects/{appId}/join`. The service path segment is part of the base: `https://partner.ai.agora.io/preview/api/` alone returns `404 no Route matched with those values`. - -The preview host is a single partner endpoint with no regional replicas. The parent client's regional selection is not mutated; it remains available for GA sessions. - -## Debug output - -`debug=True` prints the resolved start request. The body is passed through `redact_secrets` first, which replaces vendor API keys, the RTC token, and the App ID with `[REDACTED]` while leaving model names, voices, and instructions readable. The same redaction is applied to the httpx request log. - -Empty strings are left visible on purpose: `""` is the signature of an unset environment variable, and hiding it would disguise the exact misconfiguration the debug output exists to surface. - -## Related - -- [Regional Routing](./regional-routing.md) — the production domain pool the preview client bypasses -- [Error Handling](./error-handling.md) — `ApiError` and API error handling diff --git a/docs/guides/regional-routing.md b/docs/guides/regional-routing.md index a6051bd..ff6e4cc 100644 --- a/docs/guides/regional-routing.md +++ b/docs/guides/regional-routing.md @@ -40,7 +40,7 @@ If you omit `with_stt()`, AgentKit uses `FengmingSTT` by default for `Area.CN` c | Client area | STT classes | LLM classes | MLLM classes | TTS classes | Avatar classes | |---|---|---|---|---|---| -| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | +| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GeminiSTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | | `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `QwenOmni` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` | Global client example: diff --git a/docs/index.md b/docs/index.md index e2c2028..11f3196 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,7 +53,6 @@ The Agora Conversational AI Python SDK lets you build voice-powered AI agents on | [Pagination](./guides/pagination.md) | Iterate over paginated list endpoints | | [Advanced](./guides/advanced.md) | Raw response, retries, timeouts, custom httpx client | | [Low-Level API](./guides/low-level-api.md) | Generated REST APIs | -| [Preview Endpoint](./guides/preview-endpoint.md) | Session-scoped preview routing and the `agora-feature` gate header | | [Client Reference](./reference/client.md) | Full `Agora` / `AsyncAgora` API | | [Agent Reference](./reference/agent.md) | Full `Agent` builder API | | [Session Reference](./reference/session.md) | Full `AgentSession` / `AsyncAgentSession` API | diff --git a/docs/reference/vendors.md b/docs/reference/vendors.md index 38fee0e..960a3ad 100644 --- a/docs/reference/vendors.md +++ b/docs/reference/vendors.md @@ -20,7 +20,7 @@ Construct vendors directly from `agora_agent`, then bind a client with `Agent(cl | Area | STT classes | LLM classes | MLLM classes | TTS classes | Avatar classes | |---|---|---|---|---|---| -| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | +| `Area.US`, `Area.EU`, `Area.AP` | `DeepgramSTT`, `SpeechmaticsSTT`, `MicrosoftSTT`, `OpenAISTT`, `GeminiSTT`, `GoogleSTT`, `AmazonSTT`, `AssemblyAISTT`, `AresSTT`, `SarvamSTT`, `XaiSTT` | `OpenAI`, `AzureOpenAI`, `Anthropic`, `Gemini`, `Groq`, `VertexAILLM`, `AmazonBedrock`, `Dify`, `CustomLLM` | `OpenAIRealtime`, `AzureOpenAIRealtime`, `GeminiLive`, `VertexAI`, `XaiGrok` | `ElevenLabsTTS`, `MicrosoftTTS`, `OpenAITTS`, `CartesiaTTS`, `GoogleTTS`, `AmazonTTS`, `DeepgramTTS`, `GradiumTTS`, `MistralTTS`, `TypecastTTS`, `HumeAITTS`, `RimeTTS`, `FishAudioTTS`, `MiniMaxTTS`, `MurfTTS`, `SarvamTTS`, `GenericTTS`, `XaiTTS` | `LiveAvatarAvatar`, `HeyGenAvatar`, `AkoolAvatar`, `AnamAvatar`, `GenericAvatar` | | `Area.CN` | `FengmingSTT`, `TencentSTT`, `MicrosoftCNSTT`, `XfyunSTT`, `XfyunBigModelSTT`, `XfyunDialectSTT` | `AliyunLLM`, `BytedanceLLM`, `DeepSeekLLM`, `TencentLLM` | `QwenOmni` | `MiniMaxCNTTS`, `TencentTTS`, `BytedanceTTS`, `MicrosoftCNTTS`, `CosyVoiceTTS`, `BytedanceDuplexTTS`, `StepFunTTS`, `GenericTTS` | `SenseTimeAvatar`, `SpatiusAvatar` | Global example: @@ -541,6 +541,28 @@ For `nova-2` and `nova-3`, omit `api_key` to use Agora-managed credentials. For | `model` | `str` | No | `None` | Recognition model | | `additional_params` | `Dict[str, Any]` | No | `None` | Additional parameters | +### `GeminiSTT` + +| Parameter | Type | Required | Default | Description | +|---|---|---|---|---| +| `api_key` | `str` | Yes | — | Google Gemini API key | +| `model` | `str` | Yes | — | Gemini transcription model, such as `gemini-3.7-transcribe-live` | +| `language` | `str` | No | `None` | Language code for speech recognition. Takes precedence over top-level `asr.language`. | +| `word_timestamp` | `bool` | No | `None` | Include word-level timestamps in transcription results | +| `sample_rate` | `int` | No | `None` | Audio sample rate in Hz | +| `additional_params` | `Dict[str, Any]` | No | `None` | Additional Gemini ASR parameters | + +```python +from agora_agent import GeminiSTT + +stt = GeminiSTT( + api_key="your-google-api-key", + model="gemini-3.7-transcribe-live", + language="en-US", + word_timestamp=True, +) +``` + ### `AmazonSTT` | Parameter | Type | Required | Default | Description | diff --git a/src/agora_agent/__init__.py b/src/agora_agent/__init__.py index b132d4b..a7ddea1 100644 --- a/src/agora_agent/__init__.py +++ b/src/agora_agent/__init__.py @@ -45,6 +45,7 @@ ElevenLabsTTS, FishAudioTTS, Gemini, + GeminiSTT, GeminiLive, GenericAvatar, GenericTTS, diff --git a/src/agora_agent/agentkit/__init__.py b/src/agora_agent/agentkit/__init__.py index 934dbd9..9d24cc3 100644 --- a/src/agora_agent/agentkit/__init__.py +++ b/src/agora_agent/agentkit/__init__.py @@ -180,6 +180,7 @@ Dify, FishAudioTTS, Gemini, + GeminiSTT, GeminiLive, GenericAvatar, GoogleSTT, @@ -436,6 +437,7 @@ "MicrosoftSTT", "MicrosoftCNSTT", "OpenAISTT", + "GeminiSTT", "GoogleSTT", "AmazonSTT", "AssemblyAISTT", diff --git a/src/agora_agent/agentkit/agent.py b/src/agora_agent/agentkit/agent.py index f1d4587..f743d4c 100644 --- a/src/agora_agent/agentkit/agent.py +++ b/src/agora_agent/agentkit/agent.py @@ -202,34 +202,10 @@ class SessionOptions(typing_extensions.TypedDict, total=False): warn: typing.Callable[[str], None] - -def _drop_none(value: typing.Any) -> typing.Any: - """Recursively remove None-valued mapping entries.""" - if isinstance(value, dict): - return {k: _drop_none(v) for k, v in value.items() if v is not None} - if isinstance(value, list): - return [_drop_none(item) for item in value] - return value - - def _start_properties_from_mapping( properties: typing.Mapping[str, typing.Any], ) -> StartAgentsRequestProperties: - try: - return parse_obj_as(StartAgentsRequestProperties, dict(properties)) - except Exception: - # Preview providers are absent from the generated unions by design — the - # schema models what production serves. A config the preview gateway - # does understand is passed through unvalidated rather than rejected. - # Imported lazily: preview.client imports the pool client, which imports - # this module. - from .preview.client import required_preview_features - - if required_preview_features(properties): - # The typed path serializes with exclude_none; strip None here so the - # bypass puts the same bytes on the wire instead of explicit nulls. - return typing.cast(StartAgentsRequestProperties, _drop_none(dict(properties))) - raise + return parse_obj_as(StartAgentsRequestProperties, dict(properties)) # LLM sub-type aliases @@ -1051,10 +1027,6 @@ def to_properties( if is_mllm_mode: if self._mllm is not None: mllm_config = dict(self._mllm) - # These are production wire spellings. A route that spells one of them - # differently needs a rename entry in `preview/client.py`, or the value - # lands in a field the provider ignores and fails silently. See - # docs/guides/preview-endpoint.md#the-vendor-class-is-not-the-whole-wire-shape. if self._greeting is not None: mllm_config.setdefault("greeting_message", self._greeting) if self._failure_message is not None: @@ -1123,10 +1095,6 @@ def _resolve_asr_config(self, turn_detection_config: TurnDetectionInput) -> typi if not asr_config: area_scope = getattr(self._client, "area_scope", None) asr_config["vendor"] = "fengming" if area_scope == "cn" else "ares" - # Unconditional: turn detection is the single source of truth for the - # interaction language, so a vendor-level ``language`` would be silently - # discarded here. Do not add one to a vendor class — see - # docs/guides/preview-endpoint.md#the-vendor-class-is-not-the-whole-wire-shape. asr_config["language"] = self._field_value(turn_detection_config, "language") return asr_config diff --git a/src/agora_agent/agentkit/agent_session.py b/src/agora_agent/agentkit/agent_session.py index b5756e1..e6cfbba 100644 --- a/src/agora_agent/agentkit/agent_session.py +++ b/src/agora_agent/agentkit/agent_session.py @@ -43,7 +43,6 @@ normalize_preset_input, resolve_session_presets, ) -from .preview.client import create_preview_session_clients, required_preview_features from .token import _parse_numeric_uid, generate_convo_ai_token @@ -182,24 +181,6 @@ def _require_agent_management(self) -> typing.Any: ) return self._agent_management - def _bind_session_clients(self, features: typing.Sequence[str]) -> None: - """Pin this session to production or preview without mutating its client.""" - if features: - self._agents, self._agent_management = create_preview_session_clients( - self._client, features - ) - from .preview.client import PREVIEW_API_BASE_URL - - self._api_base_url = PREVIEW_API_BASE_URL - return - self._agents = self._client.agents - self._agent_management = getattr(self._client, "agent_management", None) - self._api_base_url = ( - self._client.get_current_url() - if hasattr(self._client, "get_current_url") - else None - ) - # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ @@ -636,8 +617,6 @@ def start(self) -> str: properties, ) - self._bind_session_clients(required_preview_features(resolved_properties)) - if self._debug: print("[Agora Debug] Starting agent session...") if hasattr(self._client, "get_current_url"): @@ -1003,8 +982,6 @@ async def start(self) -> str: properties, ) - self._bind_session_clients(required_preview_features(resolved_properties)) - if self._debug: print("[Agora Debug] Starting agent session...") if hasattr(self._client, "get_current_url"): diff --git a/src/agora_agent/agentkit/preview/__init__.py b/src/agora_agent/agentkit/preview/__init__.py deleted file mode 100644 index 984eff7..0000000 --- a/src/agora_agent/agentkit/preview/__init__.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Preview endpoint support. - -Temporary package: delete it when these providers ship on the production -gateway. See ``client.py`` for the routing and gate header. -""" - -from .client import ( - PREVIEW_API_BASE_URL, - PREVIEW_FEATURE_HEADER, - PreviewFeature, - PreviewFeatures, - create_preview_session_clients, - required_preview_features, -) -from .vendors import ( - GeminiSTT, - GeminiSTTModels, -) - -__all__ = [ - "PREVIEW_API_BASE_URL", - "PREVIEW_FEATURE_HEADER", - "GeminiSTTModels", - "GeminiSTT", - "PreviewFeature", - "PreviewFeatures", - "create_preview_session_clients", - "required_preview_features", -] diff --git a/src/agora_agent/agentkit/preview/client.py b/src/agora_agent/agentkit/preview/client.py deleted file mode 100644 index 2203c2d..0000000 --- a/src/agora_agent/agentkit/preview/client.py +++ /dev/null @@ -1,118 +0,0 @@ -"""Preview endpoint routing helpers. - -Preview providers are served by a partner host gated by an ``agora-feature`` -header. Agent sessions use these helpers to create private generated clients -for preview traffic while the caller's ``Agora`` / ``AsyncAgora`` client stays -on its production endpoint. - -Everything under ``agentkit/preview/`` is temporary. When these providers ship -on the production gateway, delete this package and move the vendor classes into -``vendors/stt.py``. -""" - -from __future__ import annotations - -import typing - -from ...agent_management.client import AgentManagementClient, AsyncAgentManagementClient -from ...agents.client import AgentsClient, AsyncAgentsClient -from ...core.client_wrapper import AsyncClientWrapper, SyncClientWrapper - -#: Base URL that serves the preview providers. -PREVIEW_API_BASE_URL = "https://partner.ai.agora.io/preview/api/conversational-ai-agent" - -#: Request header that opts a request into a preview provider family. -#: -#: This is the header the preview gateway routes on. A request that reaches the -#: gateway without it is not rejected — it is routed to the production -#: environment, where the preview providers do not exist. -PREVIEW_FEATURE_HEADER = "agora-feature" - - -class PreviewFeatures: - """Preview provider families. - - Each value is one entry in the ``agora-feature`` header and gates a set of - vendors on the preview endpoint. - """ - - #: Gemini 3.5 Transcribe ASR. - GEMINI_LIVE = "gemini-live" - - -PreviewFeature = str - - -def _preview_headers( - features: typing.Sequence[str], - headers: typing.Optional[typing.Dict[str, str]], -) -> typing.Dict[str, str]: - """Merge the gate header over caller headers. - - The gate goes last on purpose: caller-supplied headers must not be able to - drop or blank it. A preview request that loses the header is not rejected — - it routes to the production environment, where the preview providers do not - exist. Use ``features`` to change the value. - """ - merged: typing.Dict[str, str] = dict(headers or {}) - merged[PREVIEW_FEATURE_HEADER] = ",".join(features) - return merged - - -def create_preview_session_clients( - client: typing.Any, - features: typing.Sequence[str], -) -> typing.Tuple[typing.Any, typing.Any]: - """Create generated clients pinned to the preview host and feature gate.""" - source = client._client_wrapper - kwargs = { - "authorization": source._authorization, - "username": source._username, - "password": source._password, - "headers": _preview_headers(features, source.get_custom_headers()), - "base_url": PREVIEW_API_BASE_URL, - "timeout": source.get_timeout(), - "httpx_client": source.httpx_client.httpx_client, - } - if isinstance(source, AsyncClientWrapper): - async_wrapper = AsyncClientWrapper(**kwargs) - return ( - AsyncAgentsClient(client_wrapper=async_wrapper), - AsyncAgentManagementClient(client_wrapper=async_wrapper), - ) - if isinstance(source, SyncClientWrapper): - sync_wrapper = SyncClientWrapper(**kwargs) - return ( - AgentsClient(client_wrapper=sync_wrapper), - AgentManagementClient(client_wrapper=sync_wrapper), - ) - raise TypeError("Unsupported Agora client wrapper") - - -#: ASR vendors served only by the preview endpoint. -_PREVIEW_ASR_VENDORS = frozenset({"gemini"}) - - -def required_preview_features(properties: typing.Mapping[str, typing.Any]) -> typing.List[str]: - """Return the preview features a start request needs. - - Derived from the request body rather than from the vendor classes, so - hand-written configs are covered too. - """ - features: typing.List[str] = [] - - asr = properties.get("asr") - if isinstance(asr, dict) and asr.get("vendor") in _PREVIEW_ASR_VENDORS: - features.append(PreviewFeatures.GEMINI_LIVE) - - return features - - -__all__ = [ - "PREVIEW_API_BASE_URL", - "PREVIEW_FEATURE_HEADER", - "PreviewFeature", - "PreviewFeatures", - "create_preview_session_clients", - "required_preview_features", -] diff --git a/src/agora_agent/agentkit/preview/vendors.py b/src/agora_agent/agentkit/preview/vendors.py deleted file mode 100644 index d4ef4b7..0000000 --- a/src/agora_agent/agentkit/preview/vendors.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Preview provider vendor classes. - -These follow the same shape as the GA vendor classes in ``vendors/`` — snake_case -constructor options in, snake_case wire config out — so they drop into -``agent.with_stt()`` unchanged. Sessions that use them route to the preview -endpoint automatically. -""" - -from __future__ import annotations - -from typing import Any, Dict, List, Optional - -from ..vendors.base import BaseSTT -from pydantic import ConfigDict, Field - -#: Rejects ``api_key=""``. ``Field(...)`` alone makes the key required but still -#: accepts the empty string, which would reach the provider as a blank -#: credential; the TypeScript and Go vendors both refuse it at construction. -_ApiKey = Field(..., min_length=1, description="Google API key") - - -class GeminiSTTModels: - """Gemini preview transcription models.""" - - TRANSCRIBE_35_LIVE = "gemini-3.5-transcribe-live" - - -class GeminiSTT(BaseSTT): - """Gemini 3.5 Transcribe ASR vendor (preview). - - Example:: - - agent = Agent(client=client).with_stt( - GeminiSTT(api_key=..., language_codes=["en-US"]) - ) - """ - - model_config = ConfigDict(extra="forbid") - - api_key: str = _ApiKey - model: Optional[str] = Field( - default=None, - description="Model name. Defaults to `gemini-3.5-transcribe-live`.", - ) - language_codes: Optional[List[str]] = Field( - default=None, - description=( - "Languages the model should transcribe, sent as `params.language_codes`. " - "Omitted from the request when unset, which is how the provider sets " - "auto-detect — the SDK does not pin a language the caller never asked for. " - "Pass one code to commit to a language, several to let the model choose " - "between them, or an explicit empty list to request auto-detect outright. " - "This is the only language setting on this vendor. The top-level " - "`asr.language` is supplied by `Agent` from turn detection, as it is for " - "every STT vendor." - ), - ) - custom_vocabulary: Optional[List[str]] = Field( - default=None, - description=( - "Words and phrases to bias recognition toward — product names, jargon, " - "proper nouns the model would otherwise mis-hear." - ), - ) - sample_rate: Optional[int] = Field( - default=None, - description="Audio sample rate in Hz. Defaults to 16000.", - ) - word_timestamp: Optional[bool] = Field( - default=None, - description=( - "Emit per-word timestamps in transcription results. Omitted unless explicitly set; " - "cannot be `true` when `custom_vocabulary` is set." - ), - ) - additional_params: Optional[Dict[str, Any]] = Field( - default=None, - description="Additional vendor-specific parameters.", - ) - - def to_config(self) -> Dict[str, Any]: - model = self.model if self.model is not None else GeminiSTTModels.TRANSCRIBE_35_LIVE - sample_rate = self.sample_rate if self.sample_rate is not None else 16000 - - # additional_params first so that explicit fields always win. - params: Dict[str, Any] = dict(self.additional_params or {}) - params["api_key"] = self.api_key - params["model"] = model - params["sample_rate"] = sample_rate - # Omitted unless the caller asked for it: no language_codes is how the - # provider spells auto-detect, and seeding it from ``language`` would pin - # every request to a language the caller never chose. - if self.language_codes is not None: - params["language_codes"] = list(self.language_codes) - if self.custom_vocabulary is not None: - params["custom_vocabulary"] = list(self.custom_vocabulary) - if self.word_timestamp is not None: - params["word_timestamp"] = self.word_timestamp - if "custom_vocabulary" in params and params.get("word_timestamp") is True: - raise ValueError("custom_vocabulary cannot be used with word_timestamp=true") - - # No top-level `language`: `Agent` sets it from turn detection, - # the same as every other STT vendor. - return {"vendor": "gemini", "params": params} - - -__all__ = [ - "GeminiSTTModels", - "GeminiSTT", -] diff --git a/src/agora_agent/agentkit/regional_agent.py b/src/agora_agent/agentkit/regional_agent.py index 0a91dd2..8a9b3f1 100644 --- a/src/agora_agent/agentkit/regional_agent.py +++ b/src/agora_agent/agentkit/regional_agent.py @@ -30,6 +30,7 @@ AresSTT, AssemblyAISTT, DeepgramSTT, + GeminiSTT, GoogleSTT, MicrosoftSTT, OpenAISTT, @@ -82,6 +83,7 @@ DeepgramSTT, MicrosoftSTT, OpenAISTT, + GeminiSTT, GoogleSTT, AmazonSTT, AssemblyAISTT, diff --git a/src/agora_agent/agentkit/vendors/__init__.py b/src/agora_agent/agentkit/vendors/__init__.py index c69669d..fb0ac94 100644 --- a/src/agora_agent/agentkit/vendors/__init__.py +++ b/src/agora_agent/agentkit/vendors/__init__.py @@ -41,6 +41,7 @@ AresSTT, AssemblyAISTT, DeepgramSTT, + GeminiSTT, GoogleSTT, MicrosoftSTT, OpenAISTT, @@ -120,6 +121,7 @@ "MicrosoftSTT", "MicrosoftCNSTT", "OpenAISTT", + "GeminiSTT", "GoogleSTT", "AmazonSTT", "AssemblyAISTT", diff --git a/src/agora_agent/agentkit/vendors/catalog.py b/src/agora_agent/agentkit/vendors/catalog.py index 25c99df..6c63d08 100644 --- a/src/agora_agent/agentkit/vendors/catalog.py +++ b/src/agora_agent/agentkit/vendors/catalog.py @@ -22,6 +22,7 @@ AresSTT, AssemblyAISTT, DeepgramSTT, + GeminiSTT, GoogleSTT, MicrosoftSTT, OpenAISTT, @@ -74,6 +75,7 @@ def __init__( "deepgram": DeepgramSTT, "microsoft": MicrosoftSTT, "openai": OpenAISTT, + "gemini": GeminiSTT, "google": GoogleSTT, "amazon": AmazonSTT, "assemblyai": AssemblyAISTT, diff --git a/src/agora_agent/agentkit/vendors/namespaces.py b/src/agora_agent/agentkit/vendors/namespaces.py index 001b374..c5d7c89 100644 --- a/src/agora_agent/agentkit/vendors/namespaces.py +++ b/src/agora_agent/agentkit/vendors/namespaces.py @@ -19,6 +19,7 @@ AresSTT, AssemblyAISTT, DeepgramSTT, + GeminiSTT, GoogleSTT, MicrosoftSTT, OpenAISTT, @@ -53,6 +54,7 @@ class GlobalSTTVendors: deepgram = DeepgramSTT microsoft = MicrosoftSTT openai = OpenAISTT + gemini = GeminiSTT google = GoogleSTT amazon = AmazonSTT assemblyai = AssemblyAISTT diff --git a/src/agora_agent/agentkit/vendors/region.py b/src/agora_agent/agentkit/vendors/region.py index 721f7e3..0e5b473 100644 --- a/src/agora_agent/agentkit/vendors/region.py +++ b/src/agora_agent/agentkit/vendors/region.py @@ -22,6 +22,7 @@ "deepgram", "microsoft", "openai", + "gemini", "google", "amazon", "assemblyai", diff --git a/src/agora_agent/agentkit/vendors/stt.py b/src/agora_agent/agentkit/vendors/stt.py index 377d1ba..29b3dbb 100644 --- a/src/agora_agent/agentkit/vendors/stt.py +++ b/src/agora_agent/agentkit/vendors/stt.py @@ -196,6 +196,42 @@ def to_config(self) -> Dict[str, Any]: return config +class GeminiSTTOptions(BaseModel): + model_config = ConfigDict(extra="forbid") + + api_key: str = Field(..., description="Google Gemini API key") + model: str = Field(..., description="Google Gemini transcription model") + language: Optional[str] = Field(default=None, description="Language code for speech recognition") + word_timestamp: Optional[bool] = Field( + default=None, + description="Include word-level timestamps in transcription results", + ) + sample_rate: Optional[int] = Field(default=None, description="Audio sample rate in Hz") + additional_params: Optional[Dict[str, Any]] = Field(default=None) + + +class GeminiSTT(GeminiSTTOptions, BaseSTT): + def to_config(self) -> Dict[str, Any]: + params: Dict[str, Any] = dict(self.additional_params or {}) + params.update( + { + "api_key": self.api_key, + "model": self.model, + } + ) + if self.sample_rate is not None: + params["sample_rate"] = self.sample_rate + if self.language is not None: + params["language"] = self.language + if self.word_timestamp is not None: + params["word_timestamp"] = self.word_timestamp + + return { + "vendor": "gemini", + "params": params, + } + + class AmazonSTTOptions(BaseModel): model_config = ConfigDict(extra="forbid") diff --git a/tests/custom/test_debug.py b/tests/custom/test_debug.py new file mode 100644 index 0000000..aa91d23 --- /dev/null +++ b/tests/custom/test_debug.py @@ -0,0 +1,113 @@ +import json +from typing import List + +import httpx + +from agora_agent import Agent, Agora, Area, Gemini, GeminiSTT, GoogleTTS +from agora_agent.agentkit.debug import REDACTED, redact_secrets + +API_KEY = "test-google-api-key" +APP_ID = "0" * 32 +APP_CERTIFICATE = "1" * 32 + + +class _Recorder(httpx.MockTransport): + def __init__(self) -> None: + self.requests: List[httpx.Request] = [] + super().__init__(self._handle) + + def _handle(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + return httpx.Response(200, json={"agent_id": "agent-1"}) + + +def test_redact_secrets_walks_nested_structures() -> None: + redacted = redact_secrets( + { + "appid": APP_ID, + "properties": { + "token": "007eJxTYKhsrH10", + "asr": {"vendor": "gemini", "params": {"api_key": API_KEY, "model": "m"}}, + "mcp_servers": [{"name": "a", "headers": {"authorization": "Bearer x"}}], + }, + } + ) + + assert redacted == { + "appid": REDACTED, + "properties": { + "token": REDACTED, + "asr": {"vendor": "gemini", "params": {"api_key": REDACTED, "model": "m"}}, + "mcp_servers": [{"name": "a", "headers": {"authorization": REDACTED}}], + }, + } + + +def test_redact_secrets_covers_query_keys_and_google_credentials() -> None: + redacted = redact_secrets( + { + "llm": { + "url": ( + "https://generativelanguage.googleapis.com/v1beta/models/" + f"gemini-2.0-flash:streamGenerateContent?alt=sse&key={API_KEY}" + ), + }, + "tts": {"params": {"credentials": API_KEY}}, + } + ) + + assert API_KEY not in json.dumps(redacted) + + +def test_redact_secrets_leaves_empty_values_visible() -> None: + assert redact_secrets({"params": {"api_key": ""}}) == {"params": {"api_key": ""}} + + +def test_redact_secrets_does_not_mutate_input() -> None: + original = {"properties": {"asr": {"params": {"api_key": API_KEY}}}} + redact_secrets(original) + + assert original["properties"]["asr"]["params"]["api_key"] == API_KEY + + +def test_debug_output_redacts_credentials_without_changing_request(capsys) -> None: + recorder = _Recorder() + client = Agora( + area=Area.US, + app_id=APP_ID, + app_certificate=APP_CERTIFICATE, + httpx_client=httpx.Client(transport=recorder), + ) + agent = ( + Agent(client=client) + .with_stt( + GeminiSTT( + api_key=API_KEY, + model="gemini-3.7-transcribe-live", + language="en-US", + word_timestamp=True, + ) + ) + .with_llm(Gemini(api_key=API_KEY, model="gemini-2.0-flash")) + .with_tts( + GoogleTTS( + key=API_KEY, + voice_name="en-US-Chirp3-HD-Charon", + language_code="en-US", + ) + ) + ) + + agent.create_session( + channel="debug-channel", + agent_uid="1", + remote_uids=["100"], + debug=True, + ).start() + + output = capsys.readouterr().out + assert API_KEY not in output + assert "gemini-3.7-transcribe-live" in output + + sent = json.loads(recorder.requests[0].content) + assert sent["properties"]["asr"]["params"]["api_key"] == API_KEY diff --git a/tests/custom/test_gemini_stt.py b/tests/custom/test_gemini_stt.py new file mode 100644 index 0000000..8fc1d06 --- /dev/null +++ b/tests/custom/test_gemini_stt.py @@ -0,0 +1,160 @@ +import json +from typing import List + +import httpx +import pytest +from pydantic import ValidationError + +import agora_agent +from agora_agent import Agent, Agora, Area, Gemini, GeminiSTT, GoogleTTS +from agora_agent.agentkit.vendors.catalog import GLOBAL_VENDOR_NAMESPACE +from agora_agent.agentkit.vendors.namespaces import GlobalSTTVendors +from agora_agent.agentkit.vendors.region import GLOBAL_ASR_VENDORS + +API_KEY = "test-google-api-key" +MODEL = "gemini-3.7-transcribe-live" +APP_ID = "0" * 32 +APP_CERTIFICATE = "1" * 32 + + +class _Recorder(httpx.MockTransport): + def __init__(self) -> None: + self.requests: List[httpx.Request] = [] + super().__init__(self._handle) + + def _handle(self, request: httpx.Request) -> httpx.Response: + self.requests.append(request) + return httpx.Response(200, json={"agent_id": "agent-1"}) + + +def _gemini_stt(**kwargs) -> GeminiSTT: + options = { + "api_key": API_KEY, + "model": MODEL, + "language": "en-US", + "word_timestamp": True, + } + options.update(kwargs) + return GeminiSTT(**options) + + +def _complete_agent(client: Agora) -> Agent: + return ( + Agent(client=client) + .with_stt(_gemini_stt()) + .with_llm(Gemini(api_key=API_KEY, model="gemini-2.0-flash")) + .with_tts( + GoogleTTS( + key=API_KEY, + voice_name="en-US-Chirp3-HD-Charon", + language_code="en-US", + ) + ) + ) + + +def test_gemini_stt_serializes_fern_schema() -> None: + assert _gemini_stt().to_config() == { + "vendor": "gemini", + "params": { + "api_key": API_KEY, + "model": MODEL, + "language": "en-US", + "word_timestamp": True, + }, + } + + +def test_gemini_stt_serializes_optional_sample_rate() -> None: + config = _gemini_stt( + sample_rate=24000, + additional_params={"provider_option": "kept", "model": "overridden"}, + ).to_config() + + assert config["params"] == { + "provider_option": "kept", + "api_key": API_KEY, + "model": MODEL, + "language": "en-US", + "word_timestamp": True, + "sample_rate": 24000, + } + + +@pytest.mark.parametrize("field", ["api_key", "model"]) +def test_gemini_stt_requires_fern_fields(field: str) -> None: + options = { + "api_key": API_KEY, + "model": MODEL, + "language": "en-US", + "word_timestamp": True, + } + del options[field] + + with pytest.raises(ValidationError): + GeminiSTT(**options) + + +def test_gemini_stt_omits_optional_fields_when_unset() -> None: + config = GeminiSTT(api_key=API_KEY, model=MODEL).to_config() + + assert config == { + "vendor": "gemini", + "params": { + "api_key": API_KEY, + "model": MODEL, + }, + } + + +def test_gemini_stt_preserves_explicit_false_word_timestamp() -> None: + config = GeminiSTT( + api_key=API_KEY, + model=MODEL, + word_timestamp=False, + ).to_config() + + assert config["params"]["word_timestamp"] is False + + +def test_gemini_stt_rejects_unknown_fields() -> None: + with pytest.raises(ValidationError): + _gemini_stt(unknown_option=True) + + +def test_gemini_stt_is_available_from_standard_vendor_surfaces() -> None: + assert agora_agent.GeminiSTT is GeminiSTT + assert "GeminiSTT" in agora_agent.__all__ + assert "gemini" in GLOBAL_ASR_VENDORS + assert GLOBAL_VENDOR_NAMESPACE.asr["gemini"] is GeminiSTT + assert GlobalSTTVendors.gemini is GeminiSTT + + +def test_gemini_stt_uses_production_endpoint_and_fern_request_model() -> None: + recorder = _Recorder() + client = Agora( + area=Area.US, + app_id=APP_ID, + app_certificate=APP_CERTIFICATE, + httpx_client=httpx.Client(transport=recorder), + ) + + _complete_agent(client).create_session( + channel="gemini-channel", + agent_uid="1", + remote_uids=["100"], + ).start() + + request = recorder.requests[0] + body = json.loads(request.content) + assert str(request.url).startswith(client.get_current_url()) + assert body["properties"]["asr"] == { + "vendor": "gemini", + "language": "en-US", + "params": { + "api_key": API_KEY, + "model": MODEL, + "language": "en-US", + "word_timestamp": True, + }, + } diff --git a/tests/custom/test_preview.py b/tests/custom/test_preview.py deleted file mode 100644 index 2fba9c4..0000000 --- a/tests/custom/test_preview.py +++ /dev/null @@ -1,403 +0,0 @@ -"""Preview endpoint tests. - -Wire-shape expectations are copied from the TypeScript suite so the three SDKs -stay byte-identical on the wire. -""" - -import json - -import httpx -import pytest - -from agora_agent import Agora, Area, AsyncAgora -from agora_agent.agentkit import Agent -from agora_agent.agentkit.debug import REDACTED, redact_secrets -from agora_agent.agentkit.preview import ( - PREVIEW_API_BASE_URL, - GeminiSTT, - required_preview_features, -) -from agora_agent.agentkit.vendors import Gemini, GoogleTTS - -API_KEY = "test-google-api-key" -APP_ID = "test-app-id-0123456789abcdefghij" -APP_CERTIFICATE = "test-app-certificate-01234567890" - - -def _client(transport=None, **kwargs): - httpx_client = httpx.Client(transport=transport) if transport is not None else None - return Agora( - area=Area.US, - app_id=APP_ID, - app_certificate=APP_CERTIFICATE, - httpx_client=httpx_client, - **kwargs, - ) - - -def _with_preview_asr(agent): - """Complete an agent with the preview ASR plus Gemini LLM and Google TTS. - - The preview ASR only reaches its provider through the preview endpoint, so - this is what the routing and gating tests need to have configured. - """ - return ( - agent.with_stt(GeminiSTT(api_key=API_KEY, language_codes=["en-US"])) - .with_llm(Gemini(api_key=API_KEY, model="gemini-2.0-flash")) - .with_tts( - GoogleTTS( - key=API_KEY, - voice_name="en-US-Chirp3-HD-Charon", - language_code="en-US", - ) - ) - ) - - -class _Recorder(httpx.MockTransport): - """Captures every outgoing request and answers with a generic success body.""" - - def __init__(self): - self.requests = [] - super().__init__(self._handle) - - def _handle(self, request: httpx.Request) -> httpx.Response: - self.requests.append(request) - return httpx.Response(200, json={"agent_id": "agent-1", "data": {"list": []}}) - - -# --- Vendor wire shapes ----------------------------------------------------- - - -def test_gemini_transcribe_serialises_to_documented_asr_shape(): - config = GeminiSTT(api_key=API_KEY, language_codes=["en-US"]).to_config() - - assert config == { - "vendor": "gemini", - "params": { - "api_key": API_KEY, - "model": "gemini-3.5-transcribe-live", - "sample_rate": 16000, - "language_codes": ["en-US"], - }, - } - - -def test_gemini_transcribe_allows_overrides(): - config = GeminiSTT( - api_key=API_KEY, - model="not-a-real-model", - sample_rate=24000, - word_timestamp=False, - additional_params={"hotwords": ["Agora"]}, - ).to_config() - - assert config == { - "vendor": "gemini", - "params": { - "hotwords": ["Agora"], - "api_key": API_KEY, - "model": "not-a-real-model", - "sample_rate": 24000, - "word_timestamp": False, - }, - } - - -def test_emits_no_top_level_language_of_its_own(): - # Every STT vendor leaves asr.language to the Agent, which derives it from - # the turn detection language. A vendor-level copy would be a no-op the - # builder overwrites, so this one does not offer the option at all. - config = GeminiSTT(api_key=API_KEY).to_config() - - assert "language" not in config - assert "language" not in config["params"] - # Nor does it invent language_codes — absent means auto-detect. - assert "language_codes" not in config["params"] - - -def test_language_is_not_an_accepted_option(): - # extra="forbid", so a stale `language=` argument fails loudly rather than - # being silently dropped. - with pytest.raises(Exception): - GeminiSTT(api_key=API_KEY, language="en-US") - - -def test_language_codes_is_sent_verbatim_when_supplied(): - single = GeminiSTT(api_key=API_KEY, language_codes=["es-ES"]).to_config() - assert single["params"]["language_codes"] == ["es-ES"] - - multiple = GeminiSTT(api_key=API_KEY, language_codes=["en-US", "es-ES"]).to_config() - assert multiple["params"]["language_codes"] == ["en-US", "es-ES"] - - -def test_explicit_empty_language_codes_still_reaches_the_wire(): - # `[]` is the caller spelling auto-detect outright; both that and omitting - # the field mean the same thing to the provider. - config = GeminiSTT(api_key=API_KEY, language_codes=[]).to_config() - - assert config["params"]["language_codes"] == [] - - -def test_custom_vocabulary_is_sent_only_when_supplied(): - with_vocab = GeminiSTT(api_key=API_KEY, custom_vocabulary=["Agora", "Kubernetes"]).to_config() - assert with_vocab["params"]["custom_vocabulary"] == ["Agora", "Kubernetes"] - assert "word_timestamp" not in with_vocab["params"] - - without_vocab = GeminiSTT(api_key=API_KEY).to_config() - assert "custom_vocabulary" not in without_vocab["params"] - - -def test_word_timestamp_is_sent_only_when_explicitly_supplied(): - without_timestamp = GeminiSTT(api_key=API_KEY).to_config() - assert "word_timestamp" not in without_timestamp["params"] - - with_timestamp = GeminiSTT(api_key=API_KEY, word_timestamp=True).to_config() - assert with_timestamp["params"]["word_timestamp"] is True - - -def test_custom_vocabulary_rejects_enabled_word_timestamps(): - incompatible_options = [ - {"custom_vocabulary": ["Agora"], "word_timestamp": True}, - {"custom_vocabulary": [], "word_timestamp": True}, - { - "additional_params": { - "custom_vocabulary": ["Agora"], - "word_timestamp": True, - } - }, - ] - - for options in incompatible_options: - with pytest.raises( - ValueError, - match="custom_vocabulary cannot be used with word_timestamp=true", - ): - GeminiSTT(api_key=API_KEY, **options).to_config() - - -def test_custom_vocabulary_allows_explicitly_disabled_word_timestamps(): - config = GeminiSTT( - api_key=API_KEY, - custom_vocabulary=["Agora"], - word_timestamp=False, - ).to_config() - - assert config["params"]["custom_vocabulary"] == ["Agora"] - assert config["params"]["word_timestamp"] is False - - -# --- Routing ---------------------------------------------------------------- - - -def test_sends_requests_to_the_preview_endpoint_with_the_gate_header(): - recorder = _Recorder() - client = _client(transport=recorder) - production_url = client.get_current_url() - - ( - _with_preview_asr(Agent(client=client)) - .create_session(channel="preview-channel", agent_uid="1", remote_uids=["100"]) - .start() - ) - - request = recorder.requests[0] - assert str(request.url) == f"{PREVIEW_API_BASE_URL}/v2/projects/{APP_ID}/join" - assert request.headers["agora-feature"] == "gemini-live" - # Exactly one header may carry the preview value — the gateway accepts other - # spellings that are not part of the public contract, and none may ship here. - carriers = [name for name, value in request.headers.items() if "gemini-live" in value] - assert carriers == ["agora-feature"] - assert client.get_current_url() == production_url - assert client._client_wrapper.get_base_url() == production_url - assert "agora-feature" not in client._client_wrapper.get_headers() - - -def test_ga_session_stays_on_production_endpoint(): - recorder = _Recorder() - client = _client(transport=recorder) - production_url = client.get_current_url() - - Agent(client=client).create_session( - channel="ga-channel", - agent_uid="1", - remote_uids=["100"], - pipeline_id="pipeline-id", - ).start() - - assert str(recorder.requests[0].url).startswith(production_url) - assert "agora-feature" not in recorder.requests[0].headers - - -def test_keeps_the_gate_header_on_every_request(): - # A request that loses the header is routed to the production environment, - # where the preview providers do not exist — so each verb must carry it. - recorder = _Recorder() - client = _client(transport=recorder) - - session = _with_preview_asr(Agent(client=client)).create_session( - channel="preview-channel", agent_uid="1", remote_uids=["100"] - ) - session.start() - session.say("hello") - session.interrupt() - session.stop() - - assert len(recorder.requests) >= 4 - for request in recorder.requests: - assert request.headers.get("agora-feature") == "gemini-live", request.url - assert str(request.url).startswith(PREVIEW_API_BASE_URL) - - -def test_stop_agent_remains_production_only(): - recorder = _Recorder() - client = _client(transport=recorder) - production_url = client.get_current_url() - - client.stop_agent("agent-2") - - request = recorder.requests[0] - assert str(request.url).startswith(production_url) - assert "agora-feature" not in request.headers - - -def test_caller_headers_cannot_drop_the_gate(): - recorder = _Recorder() - client = _client(transport=recorder, headers={"agora-feature": "", "x-custom": "kept"}) - - ( - _with_preview_asr(Agent(client=client)) - .create_session(channel="preview-channel", agent_uid="1", remote_uids=["100"]) - .start() - ) - - request = recorder.requests[0] - assert request.headers["agora-feature"] == "gemini-live" - assert request.headers["x-custom"] == "kept" - - -@pytest.mark.asyncio -async def test_async_session_pins_the_preview_host_and_gate(): - recorder = _Recorder() - client = AsyncAgora( - area=Area.US, - app_id=APP_ID, - app_certificate=APP_CERTIFICATE, - httpx_client=httpx.AsyncClient(transport=recorder), - ) - session = _with_preview_asr(Agent(client=client)).create_async_session( - channel="preview-channel", agent_uid="1", remote_uids=["100"] - ) - - await session.start() - await session.say("hello") - await session.interrupt() - await session.stop() - - assert len(recorder.requests) == 4 - assert all(str(request.url).startswith(PREVIEW_API_BASE_URL) for request in recorder.requests) - assert all(request.headers["agora-feature"] == "gemini-live" for request in recorder.requests) - assert "agora-feature" not in client._client_wrapper.get_headers() - - -# --- Preview support guard -------------------------------------------------- - - -def test_required_preview_features_flags_the_gemini_asr_vendor(): - assert required_preview_features({"asr": {"vendor": "gemini"}}) == ["gemini-live"] - - -def test_required_preview_features_leaves_a_ga_pipeline_alone(): - assert required_preview_features({"asr": {"vendor": "microsoft"}}) == [] - - -# --- Debug redaction -------------------------------------------------------- - - -def test_redact_secrets_walks_nested_structures(): - redacted = redact_secrets( - { - "appid": "81190c52971d4004b7244bdcd93e2f34", - "properties": { - "token": "007eJxTYKhsrH10", - "asr": {"vendor": "gemini", "params": {"api_key": API_KEY, "model": "m"}}, - "mcp_servers": [{"name": "a", "headers": {"authorization": "Bearer x"}}], - }, - } - ) - - assert redacted == { - "appid": REDACTED, - "properties": { - "token": REDACTED, - "asr": {"vendor": "gemini", "params": {"api_key": REDACTED, "model": "m"}}, - "mcp_servers": [{"name": "a", "headers": {"authorization": REDACTED}}], - }, - } - - -def test_redact_secrets_covers_gemini_url_and_google_tts_credentials(): - redacted = redact_secrets( - { - "llm": { - "url": ( - "https://generativelanguage.googleapis.com/v1beta/models/" - f"gemini-2.0-flash:streamGenerateContent?alt=sse&key={API_KEY}" - ), - }, - "tts": {"params": {"credentials": API_KEY}}, - } - ) - - assert API_KEY not in json.dumps(redacted) - - -def test_redact_secrets_leaves_empty_values_visible(): - # "" is the signature of an unset env var and must stay diagnosable. - assert redact_secrets({"params": {"api_key": ""}}) == {"params": {"api_key": ""}} - - -def test_redact_secrets_does_not_mutate_input(): - original = {"properties": {"asr": {"params": {"api_key": API_KEY}}}} - redact_secrets(original) - - assert original["properties"]["asr"]["params"]["api_key"] == API_KEY - - -def test_debug_output_never_prints_a_live_credential(capsys): - recorder = _Recorder() - client = _client(transport=recorder) - - ( - Agent(client=client) - .with_stt(GeminiSTT(api_key=API_KEY)) - .with_llm(Gemini(api_key=API_KEY, model="gemini-2.0-flash")) - .with_tts( - GoogleTTS( - key=API_KEY, - voice_name="en-US-Chirp3-HD-Charon", - language_code="en-US", - ) - ) - .create_session(channel="c", agent_uid="1", remote_uids=["100"], debug=True) - .start() - ) - - output = capsys.readouterr().out - assert API_KEY not in output - assert "gemini" in output # non-secret config stays readable - - # The request itself still carries the real key. - sent = json.loads(recorder.requests[0].content) - assert sent["properties"]["asr"]["params"]["api_key"] == API_KEY - - -# --- Vendor construction ---------------------------------------------------- - - -def test_rejects_an_empty_api_key(): - # Field(...) alone makes the key required but still accepts "", which would - # reach the provider as a blank credential. The TypeScript and Go vendors - # both refuse it at construction, so this one does too. - with pytest.raises(Exception): - GeminiSTT(api_key="")