From b014cd7c75563a86400295a93bd8bbc3ad0d73dc Mon Sep 17 00:00:00 2001 From: seymourtang Date: Mon, 31 Aug 2026 17:02:32 +0800 Subject: [PATCH] Implement inline REST LLM tools support and update documentation - Added support for inline REST LLM tools, allowing global and CN LLM vendors to accept dictionary-based `tools` definitions. - Introduced new configuration classes: `LlmToolConfig`, `LlmToolExecutionConfig`, `LlmToolFunctionConfig`, `LlmToolFunctionParametersConfig`, and `LlmToolServerConfig`. - Updated documentation to reflect the new tools feature and provided examples for usage. - Enhanced agent functionality to enable tool execution explicitly with `Agent.with_tools()`. - Added tests to verify the correct configuration and functionality of the new tools feature. --- changelog.md | 4 ++ docs/guides/agent-builder-features.md | 2 +- docs/reference/vendors.md | 35 +++++++++++++++ src/agora_agent/agentkit/__init__.py | 10 +++++ src/agora_agent/agentkit/agent.py | 10 +++++ src/agora_agent/agentkit/vendors/cn.py | 12 ++++++ src/agora_agent/agentkit/vendors/llm.py | 27 ++++++++++++ tests/custom/test_llm_tools.py | 57 +++++++++++++++++++++++++ 8 files changed, 156 insertions(+), 1 deletion(-) create mode 100644 tests/custom/test_llm_tools.py diff --git a/changelog.md b/changelog.md index 2ccc200..907f7c7 100644 --- a/changelog.md +++ b/changelog.md @@ -6,6 +6,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/). ## [Unreleased] +### Added + +- **Inline REST LLM tools** — Global and CN LLM vendors now accept dictionary-based `tools` definitions and serialize them to `llm.tools`. Added public `LlmToolConfig`, `LlmToolExecutionConfig`, `LlmToolFunctionConfig`, `LlmToolFunctionParametersConfig`, and `LlmToolServerConfig` aliases. Tool execution must be enabled explicitly with `Agent.with_tools()`. + ### Changed - **Generated filler words configuration** — Generated filler word settings are now optional. The service can use default generator settings when `generated_config` is omitted, and `llm_provider`, `prompt`, and `fallback_strategy` may be omitted individually. diff --git a/docs/guides/agent-builder-features.md b/docs/guides/agent-builder-features.md index cb910e0..8c6d5ea 100644 --- a/docs/guides/agent-builder-features.md +++ b/docs/guides/agent-builder-features.md @@ -26,7 +26,7 @@ Pass `client=client` to every `Agent(...)` builder. `create_session()` and `crea |---|---|---| | `sal` | `with_sal(config)` | Selective Attention Locking — speaker recognition and noise suppression | | `advanced_features` | `with_advanced_features(features)` | Enable MLLM, RTM, SAL, tools | -| `tools` | `with_tools(enabled=True)` | Enable MCP tool invocation | +| `tools` | `with_tools(enabled=True)` | Enable MCP and inline LLM tool invocation | | `parameters` | `with_parameters(params)` | Silence config, farewell config, data channel | | `failure_message` | LLM/MLLM vendor option | Message spoken when LLM fails | | `max_history` | LLM vendor option | Max conversation turns in LLM context | diff --git a/docs/reference/vendors.md b/docs/reference/vendors.md index 1a3ed52..38fee0e 100644 --- a/docs/reference/vendors.md +++ b/docs/reference/vendors.md @@ -807,6 +807,41 @@ All CN TTS vendor classes support `skip_patterns` and `additional_params`. | `params` | `Dict[str, Any]` | No | `None` | Additional parameters | | `turn_detection` | `MllmTurnDetectionConfig` | No | `None` | MLLM turn detection configuration; overrides top-level `turn_detection` | +### Inline REST LLM tools + +`LlmConfig.tools` accepts a list of dictionaries for synchronous pass-through REST tools, using the same public shape as `mcp_servers`. This is the inline equivalent of `mcp_servers`, and it requires `advanced_features.enable_tools=True` to execute. Enable it explicitly with `Agent.with_tools()`. + +Each tool requires `function.name`, an object-shaped `function.parameters`, `server.method` (`GET` or `POST`), and `server.url`. + +```python +from agora_agent import Agent, OpenAI + +llm = OpenAI( + api_key='your-openai-key', + base_url='https://api.openai.com/v1/chat/completions', + model='gpt-4o-mini', + tools=[{ + 'type': 'function', + 'function': { + 'name': 'lookup_order', + 'description': 'Look up an order by ID.', + 'parameters': { + 'type': 'object', + 'properties': {'order_id': {'type': 'string'}}, + 'required': ['order_id'], + }, + }, + 'server': { + 'method': 'GET', + 'url': 'https://api.example.com/orders/{{args.order_id}}', + }, + }], +) +agent = Agent(client=client).with_llm(llm).with_tools() +``` + +`server.body` is only valid for `POST`. Template values may use `{{args.name}}` in URLs and bodies, and `{{template_variables.name}}` or `{{tool_call_id}}` in URLs, headers, and bodies. `execution.mode` currently supports only `sync`; `timeout_ms` must be between `1000` and `100000`. + ### `AzureOpenAIRealtime` Global Azure OpenAI Realtime vendor (`mllm.vendor`: `"azure"`). diff --git a/src/agora_agent/agentkit/__init__.py b/src/agora_agent/agentkit/__init__.py index e2935e4..934dbd9 100644 --- a/src/agora_agent/agentkit/__init__.py +++ b/src/agora_agent/agentkit/__init__.py @@ -62,6 +62,11 @@ Labels, LlmGreetingConfigs, LlmGreetingConfigsMode, + LlmToolConfig, + LlmToolExecutionConfig, + LlmToolFunctionConfig, + LlmToolFunctionParametersConfig, + LlmToolServerConfig, McpServersItem, SessionInfo, SessionListResponse, @@ -332,6 +337,11 @@ # LLM sub-types "LlmGreetingConfigs", "LlmGreetingConfigsMode", + "LlmToolConfig", + "LlmToolExecutionConfig", + "LlmToolFunctionConfig", + "LlmToolFunctionParametersConfig", + "LlmToolServerConfig", "McpServersItem", "AgentSession", "AsyncAgentSession", diff --git a/src/agora_agent/agentkit/agent.py b/src/agora_agent/agentkit/agent.py index 8821267..f1d4587 100644 --- a/src/agora_agent/agentkit/agent.py +++ b/src/agora_agent/agentkit/agent.py @@ -68,6 +68,11 @@ from ..types.mllm_turn_detection import MllmTurnDetection from ..types.mllm_turn_detection_mode import MllmTurnDetectionMode as GeneratedMllmTurnDetectionMode from ..types.mllm_vendor import MllmVendor as GeneratedMllmVendor +from ..types.llm_tool import LlmTool +from ..types.llm_tool_execution import LlmToolExecution +from ..types.llm_tool_function import LlmToolFunction +from ..types.llm_tool_function_parameters import LlmToolFunctionParameters +from ..types.llm_tool_server import LlmToolServer from ..agent_management.types.agent_think_agent_management_request_on_listening_action import ( AgentThinkAgentManagementRequestOnListeningAction, ) @@ -244,6 +249,11 @@ def _start_properties_from_mapping( FillerWordsContentGeneratedConfig = StartAgentsRequestPropertiesFillerWordsContentGeneratedConfig FillerWordsContentGeneratedConfigLlmProvider = StartAgentsRequestPropertiesFillerWordsContentGeneratedConfigLlmProvider FillerWordsGeneratedLlmProvider = StartAgentsRequestPropertiesFillerWordsContentGeneratedConfigLlmProvider +LlmToolConfig = LlmTool +LlmToolExecutionConfig = LlmToolExecution +LlmToolFunctionConfig = LlmToolFunction +LlmToolFunctionParametersConfig = LlmToolFunctionParameters +LlmToolServerConfig = LlmToolServer # Think type aliases and response ThinkOnListeningAction = AgentThinkAgentManagementRequestOnListeningAction diff --git a/src/agora_agent/agentkit/vendors/cn.py b/src/agora_agent/agentkit/vendors/cn.py index 020a98a..410b5b0 100644 --- a/src/agora_agent/agentkit/vendors/cn.py +++ b/src/agora_agent/agentkit/vendors/cn.py @@ -515,6 +515,7 @@ class AliyunLLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="aliyun") mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -572,6 +573,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -599,6 +602,7 @@ class BytedanceLLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="bytedance") mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -656,6 +660,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -683,6 +689,7 @@ class DeepSeekLLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="deepseek") mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -740,6 +747,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -767,6 +776,7 @@ class TencentLLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default="tencent") mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -824,6 +834,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history diff --git a/src/agora_agent/agentkit/vendors/llm.py b/src/agora_agent/agentkit/vendors/llm.py index 1ba29f6..24109e3 100644 --- a/src/agora_agent/agentkit/vendors/llm.py +++ b/src/agora_agent/agentkit/vendors/llm.py @@ -47,6 +47,7 @@ class OpenAIOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -107,6 +108,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -136,6 +139,7 @@ class AzureOpenAIOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -183,6 +187,8 @@ def to_config(self) -> Dict[str, Any]: config["template_variables"] = self.template_variables if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -210,6 +216,7 @@ class AnthropicOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -251,6 +258,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -279,6 +288,7 @@ class GeminiOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -325,6 +335,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -352,6 +364,7 @@ class GroqOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -401,6 +414,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -428,6 +443,7 @@ class CustomLLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @model_validator(mode="after") @@ -477,6 +493,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -508,6 +526,7 @@ class VertexAILLMOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -557,6 +576,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history @@ -587,6 +608,7 @@ class AmazonBedrockOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0, description="Maximum number of conversation history messages to cache") @@ -630,6 +652,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history return config @@ -655,6 +679,7 @@ class DifyOptions(BaseModel): template_variables: Optional[Dict[str, str]] = Field(default=None) vendor: Optional[str] = Field(default=None) mcp_servers: Optional[List[Dict[str, Any]]] = Field(default=None) + tools: Optional[List[Dict[str, Any]]] = Field(default=None) max_history: Optional[int] = Field(default=None, gt=0) @@ -693,6 +718,8 @@ def to_config(self) -> Dict[str, Any]: config["vendor"] = self.vendor if self.mcp_servers is not None: config["mcp_servers"] = _ensure_mcp_transport(self.mcp_servers) + if self.tools is not None: + config["tools"] = _dump_optional_model(self.tools) if self.max_history is not None: config["max_history"] = self.max_history return config diff --git a/tests/custom/test_llm_tools.py b/tests/custom/test_llm_tools.py new file mode 100644 index 0000000..879c241 --- /dev/null +++ b/tests/custom/test_llm_tools.py @@ -0,0 +1,57 @@ +from test_helpers import test_client + +from agora_agent import AdvancedFeatures, Agent, OpenAI +from agora_agent.agentkit.vendors.cn import AliyunLLM + + +def _tool() -> dict: + return { + "type": "function", + "function": { + "name": "lookup_order", + "parameters": {"type": "object", "properties": {}}, + }, + "server": {"method": "GET", "url": "https://example.com/orders"}, + } + + +def test_global_llm_tools_use_dict_shape_and_require_explicit_enablement() -> None: + tool = _tool() + vendor = OpenAI( + api_key="openai-key", + base_url="https://api.openai.com/v1/chat/completions", + model="gpt-4o-mini", + tools=[tool], + ) + + assert vendor.to_config()["tools"] == [tool] + agent = Agent(test_client()).with_llm(vendor) + assert agent.advanced_features is None + enabled_agent = agent.with_tools() + assert enabled_agent.advanced_features is not None + assert enabled_agent.advanced_features.enable_tools is True + disabled_agent = agent.with_tools(False) + assert disabled_agent.advanced_features is not None + assert disabled_agent.advanced_features.enable_tools is False + + +def test_cn_llm_uses_the_same_tools_shape_as_global_llm() -> None: + tool = _tool() + config = AliyunLLM( + api_key="aliyun-key", + base_url="https://example.com/v1/chat/completions", + model="qwen-plus", + tools=[tool], + ).to_config() + + assert config["tools"] == [tool] + + +def test_with_tools_preserves_other_advanced_features() -> None: + agent = Agent( + test_client(), advanced_features=AdvancedFeatures(enable_sal=True) + ).with_tools() + + assert agent.advanced_features is not None + assert agent.advanced_features.enable_sal is True + assert agent.advanced_features.enable_tools is True