From 156b459eecc310a8b0f9833c78a14e9aba7f4dc7 Mon Sep 17 00:00:00 2001 From: dogeONE-bit Date: Thu, 12 Nov 2026 21:54:30 +0800 Subject: [PATCH] perf(function_call): skip DeepSeek V3 regex scans until the JSON fence closes The streaming parser ran its DOTALL regex over the whole buffer on every delta. A JSON argument cannot match until a closing fence follows the json header, so the scan now waits for that. Output is unchanged. AI assistance (Cursor) was used for this change. Co-authored-by: Cursor --- lightllm/server/function_call_parser.py | 15 ++++-- .../test_deepseek_tool_parser_stream.py | 47 +++++++++++++++++++ 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/lightllm/server/function_call_parser.py b/lightllm/server/function_call_parser.py index ad810bc857..7140d3209c 100644 --- a/lightllm/server/function_call_parser.py +++ b/lightllm/server/function_call_parser.py @@ -1107,11 +1107,16 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami calls: list[ToolCallItem] = [] try: - partial_match = re.search( - pattern=r"<｜tool▁call▁begin｜>(.*)<｜tool▁sep｜>(.*)\n```json\n(.*)\n```.*", - string=current_text, - flags=re.DOTALL, - ) + # The pattern needs a closing ``` fence after ```json, so skip the scan until then. + partial_match = None + json_header = "\n```json\n" + header_at = current_text.find(json_header) + if header_at != -1 and current_text.find("\n```", header_at + len(json_header)) != -1: + partial_match = re.search( + pattern=r"<｜tool▁call▁begin｜>(.*)<｜tool▁sep｜>(.*)\n```json\n(.*)\n```.*", + string=current_text, + flags=re.DOTALL, + ) if partial_match: func_name = partial_match.group(2).strip() func_args_raw = partial_match.group(3).strip() diff --git a/unit_tests/server/test_deepseek_tool_parser_stream.py b/unit_tests/server/test_deepseek_tool_parser_stream.py index 32c59e300c..e18eb9a005 100644 --- a/unit_tests/server/test_deepseek_tool_parser_stream.py +++ b/unit_tests/server/test_deepseek_tool_parser_stream.py @@ -82,6 +82,53 @@ def test_dsml_emits_buffered_invoke_on_next_delta(): assert json.loads(arguments[1]) == {"location": "Paris"} +def _v3_call(name, payload): + return "<｜tool▁call▁begin｜>function<｜tool▁sep｜>" + name + "\n```json\n" + payload + "\n```<｜tool▁call▁end｜>" + + +def test_deepseekv3_streams_long_arguments_in_small_deltas(): + arguments = {"content": "print(1)\n" * 500} + payload = json.dumps(arguments) + call = _v3_call("write_file", payload) + deltas = ["<｜tool▁calls▁begin｜>"] + [call[i : i + 4] for i in range(0, len(call), 4)] + + parser = FunctionCallParser(TOOLS, "deepseekv3") + names, streamed = _stream(parser, deltas) + + assert names == ["write_file"] + assert json.loads(streamed[0]) == arguments + + +def test_deepseekv3_holds_arguments_until_the_json_fence_closes(): + payload = json.dumps({"content": "x" * 200}) + head = "<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>function<｜tool▁sep｜>write_file\n```json\n" + parser = FunctionCallParser(TOOLS, "deepseekv3") + names, streamed = _stream(parser, [head] + [payload[i : i + 4] for i in range(0, len(payload), 4)]) + assert names == [] + assert streamed == {} + + # The closing fence is split across deltas. The name is emitted on the delta that + # completes it, and the arguments follow on the next delta, same as one regex search. + names, streamed = _stream(parser, ["\n", "``", "`<｜tool▁call▁end｜>", "<｜tool▁calls▁end｜>"]) + assert names == ["write_file"] + assert json.loads(streamed[0]) == {"content": "x" * 200} + + +def test_deepseekv3_keeps_a_fence_inside_the_argument(): + content = "line1\n```\nline2\n```python\ncode" + payload = json.dumps({"content": content}) + text = "<｜tool▁calls▁begin｜>" + _v3_call("write_file", payload) + deltas = [text[: len("<｜tool▁calls▁begin｜>")]] + [ + text[i : i + 4] for i in range(len("<｜tool▁calls▁begin｜>"), len(text), 4) + ] + + parser = FunctionCallParser(TOOLS, "deepseekv3") + names, streamed = _stream(parser, deltas) + + assert names == ["write_file"] + assert json.loads(streamed[0]) == {"content": content} + + def test_deepseekv31_streams_long_arguments_in_small_deltas(): arguments = {"content": "print(1)\n" * 500} payload = json.dumps(arguments)