From 809120edc0d30b10dce198d4d39aee435f0ed264 Mon Sep 17 00:00:00 2001 From: huahua Date: Wed, 26 Aug 2026 10:57:57 +0800 Subject: [PATCH 1/3] test(m3): add stream case guarding against U+FFFD emoji truncation Add test_02_08_stream_no_utf8_replacement_char_in_emoji to TestSSEStream. It streams an emoji-dense prompt 20 times and fails if any delta.content contains U+FFFD, which appears when a provider truncates a 4-byte emoji mid-UTF-8-frame and replaces the invalid bytes with EF BF BD. Also document the case in docs/m3_stream_cases.md and m3_stream_cases_en.md. --- m3_format_check/docs/m3_stream_cases.md | 3 +- m3_format_check/docs/m3_stream_cases_en.md | 3 +- m3_format_check/m3_stream_tests.py | 64 ++++++++++++++++++++++ 3 files changed, 68 insertions(+), 2 deletions(-) diff --git a/m3_format_check/docs/m3_stream_cases.md b/m3_format_check/docs/m3_stream_cases.md index d0cab18..2171f05 100644 --- a/m3_format_check/docs/m3_stream_cases.md +++ b/m3_format_check/docs/m3_stream_cases.md @@ -4,7 +4,7 @@ ## 测试范围 -- `TestSSEStream`：校验 `stream_options.include_usage=true` 时 usage 仅出现在最后一个 chunk，以及连续 20 次请求中任一 `delta` 不会同时包含非空 `content` 和 `reasoning_content`。 +- `TestSSEStream`：校验 `stream_options.include_usage=true` 时 usage 仅出现在最后一个 chunk，连续 20 次请求中任一 `delta` 不会同时包含非空 `content` 和 `reasoning_content`，以及高密度 emoji 请求连续 20 次中 `delta.content` 都不出现 `U+FFFD`（`�`）替换字符。 - `TestContentStreamPacketLengthDistribution`：纯 content 场景默认执行 5 次，将 5 次的所有非空 `delta.content` 分包合并后统一判定。可通过 `M3_STREAM_CONTENT_RUNS` 调整次数。 - `TestToolCallStreamPacketLengthDistribution`：每个 tool call 场景执行 1 次，独立使用自己的阈值判定非空 `delta.tool_calls[].function.arguments` 分包。并行工具场景同时校验整体分布和每个已观察到的 `tool_call.index` 分布。 - 分包质量场景只关注分包质量，不校验正文长度、JSON 合法性、工具名称、参数内容、工具数量等功能正确性。 @@ -49,6 +49,7 @@ |:---:|:---|:---|:---| | 02_06 | `test_02_06_stream_usage_only_in_last_chunk` | `stream_options.include_usage=true` 文本场景 | 独立 SSE 协议校验，不属于分包质量规则 | | 02_07 | `test_02_07_content_and_reasoning_content_not_coexist_in_chunk` | 使用长推理和长回答请求连续执行 20 次，检查每个 `delta` 中非空 `content` 与 `reasoning_content` 不会同时出现 | 独立 SSE 协议校验，不属于分包质量规则 | +| 02_08 | `test_02_08_stream_no_utf8_replacement_char_in_emoji` | 使用高密度 emoji 请求连续执行 20 次，检查任意 `delta.content` 中都不出现 `U+FFFD`（`�`）替换字符 | 防止 provider 把四字节 emoji 从 UTF-8 分帧中间截断（0823 innomatrix emoji 截断问题） | | 01_01 | `01_01_essay_500_chars` | 直接返回约 500 字中文作文，默认执行 5 次 | 合并 5 次 `content` 分包后判定 | | 01_02 | `01_02_structured_json_1k` | 返回不少于 1KB 的结构化 JSON，默认执行 5 次 | 合并 5 次 `content` 分包后判定 | | 01_03 | `01_03_tool_content_string_500_chars` | 调用仅有 `content: string` 参数的 `save_content`，保存约 500 字作文 | `arguments` 分包 | diff --git a/m3_format_check/docs/m3_stream_cases_en.md b/m3_format_check/docs/m3_stream_cases_en.md index cc0bb06..6aeb476 100644 --- a/m3_format_check/docs/m3_stream_cases_en.md +++ b/m3_format_check/docs/m3_stream_cases_en.md @@ -4,7 +4,7 @@ ## Scope -- `TestSSEStream`: verifies that usage appears only in the final chunk when `stream_options.include_usage=true`, and that non-empty `content` and `reasoning_content` never coexist in a `delta` across 20 requests. +- `TestSSEStream`: verifies that usage appears only in the final chunk when `stream_options.include_usage=true`, that non-empty `content` and `reasoning_content` never coexist in a `delta` across 20 requests, and that no `U+FFFD` (`�`) replacement char appears in `delta.content` for an emoji-dense request across 20 requests. - `TestContentStreamPacketLengthDistribution`: each pure-content scenario runs five times by default. All non-empty `delta.content` fragments from the five runs are merged and evaluated once. Override the run count with `M3_STREAM_CONTENT_RUNS`. - `TestToolCallStreamPacketLengthDistribution`: each tool-call scenario runs once and uses its own thresholds for non-empty `delta.tool_calls[].function.arguments` fragments. The parallel-tool scenario evaluates both the combined distribution and every observed `tool_call.index` distribution. - The packet-quality scenarios evaluate packet-size quality only. They do not validate output length, JSON validity, tool names, argument contents, or tool-call counts. @@ -49,6 +49,7 @@ A target field with no non-empty fragments is an immediate `FAIL`. The decision |:---:|:---|:---|:---| | 02_06 | `test_02_06_stream_usage_only_in_last_chunk` | Text request with `stream_options.include_usage=true` | Independent SSE protocol check; not part of packet-quality rules | | 02_07 | `test_02_07_content_and_reasoning_content_not_coexist_in_chunk` | Run a long-reasoning, long-answer request 20 times and verify that non-empty `content` and `reasoning_content` never coexist in one `delta` | Independent SSE protocol check; not part of packet-quality rules | +| 02_08 | `test_02_08_stream_no_utf8_replacement_char_in_emoji` | Run an emoji-dense request 20 times and verify no `U+FFFD` (`�`) replacement char appears in any `delta.content` | Guards against providers truncating a 4-byte emoji mid-UTF-8-frame (0823 innomatrix emoji truncation issue) | | 01_01 | `01_01_essay_500_chars` | Return an approximately 500-character Chinese essay five times | Merged `content` fragments from all five runs | | 01_02 | `01_02_structured_json_1k` | Return at least 1KB of structured JSON five times | Merged `content` fragments from all five runs | | 01_03 | `01_03_tool_content_string_500_chars` | Save an approximately 500-character essay through `save_content(content: string)` | `arguments` fragments | diff --git a/m3_format_check/m3_stream_tests.py b/m3_format_check/m3_stream_tests.py index ac7043d..1f68b15 100644 --- a/m3_format_check/m3_stream_tests.py +++ b/m3_format_check/m3_stream_tests.py @@ -854,6 +854,70 @@ def test_02_07_content_and_reasoning_content_not_coexist_in_chunk(self): f"content 与 reasoning_content,示例: {first_failure['samples']}" ) + def test_02_08_stream_no_utf8_replacement_char_in_emoji(self): + """流式 delta.content 不得出现 U+FFFD(�)替换字符。 + + 背景(0823 innomatrix emoji 字符截断问题):部分 provider 的输出链路 + 存在 UTF-8 分帧(framing)问题,会把一个四字节 emoji 从字节中间截断。 + 随后服务端把非法字节转成 EF BF BD,即 Unicode 替换字符 U+FFFD(�), + 并丢弃剩余字节。因为返回的 JSON 里已经落成了 �,客户端无论如何缓存、 + 拼接都无法恢复原始 emoji,直接导致输出损坏。 + + 用大量四字节 emoji + 编号的 prompt 提高覆盖:emoji 越密集,分帧边界 + 恰好落在多字节字符中间的概率越高,越容易暴露实现里 flush 的字节截断问题。 + 连续跑 20 次,任意一次在聚合 content 或任一 delta 分片里检测到 U+FFFD + 即判失败。 + """ + prompt = ( + "请用一行输出 50 个各不相同的 emoji,格式为 '文本 N.'," + "以空格分隔,N 从 1 到 50。必须正好 50 个,不重复,不要任何解释说明。" + "尽量多使用四字节(surrogate pair)的彩色 emoji,如 😁😂🤣😊🙃🥰🤩🤔🥱🤪🧐🥺。" + ) + run_count = 20 + first_failure = None + for run_idx in range(1, run_count + 1): + result = oai_chat({ + "messages": oai_simple_messages(prompt), + "max_tokens": 4096, + }, stream=True) + assert_oai_stream_success(result) + + offenders = [] + for idx, chunk in enumerate(result.get("chunks") or []): + if not isinstance(chunk, dict): + continue + for choice_idx, choice in enumerate(chunk.get("choices") or []): + if not isinstance(choice, dict): + continue + delta = choice.get("delta") or {} + if not isinstance(delta, dict): + continue + content = delta.get("content") + if isinstance(content, str) and "\ufffd" in content: + offenders.append({ + "chunk_index": idx, + "choice_index": choice_idx, + "fffd_count": content.count("\ufffd"), + "content_preview": content[:80], + }) + + if offenders: + first_failure = { + "run": run_idx, + "offender_count": len(offenders), + "total_fffd": sum(o["fffd_count"] for o in offenders), + "samples": offenders[:3], + } + break + + assert first_failure is None, ( + f"02_08 连续 {run_count} 次中第 {first_failure['run']} 次在流式 " + f"delta.content 检测到 U+FFFD(�)替换字符,涉及 " + f"{first_failure['offender_count']} 个 chunk、共 " + f"{first_failure['total_fffd']} 个 �,疑似 UTF-8 分帧把四字节 emoji " + f"从中间截断;示例: {first_failure['samples']}" + ) + class TestContentStreamPacketLengthDistribution: """Run pure-content scenarios repeatedly, then aggregate packet statistics.""" From 789f57e73de1a5ae94d86b7c1b05b5e6a8bbc15b Mon Sep 17 00:00:00 2001 From: huahua Date: Wed, 26 Aug 2026 11:12:00 +0800 Subject: [PATCH 2/3] test(m3): drop provider-specific reference from utf8 case docs --- m3_format_check/docs/m3_stream_cases.md | 2 +- m3_format_check/docs/m3_stream_cases_en.md | 2 +- m3_format_check/m3_stream_tests.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/m3_format_check/docs/m3_stream_cases.md b/m3_format_check/docs/m3_stream_cases.md index 2171f05..d6f866e 100644 --- a/m3_format_check/docs/m3_stream_cases.md +++ b/m3_format_check/docs/m3_stream_cases.md @@ -49,7 +49,7 @@ |:---:|:---|:---|:---| | 02_06 | `test_02_06_stream_usage_only_in_last_chunk` | `stream_options.include_usage=true` 文本场景 | 独立 SSE 协议校验，不属于分包质量规则 | | 02_07 | `test_02_07_content_and_reasoning_content_not_coexist_in_chunk` | 使用长推理和长回答请求连续执行 20 次，检查每个 `delta` 中非空 `content` 与 `reasoning_content` 不会同时出现 | 独立 SSE 协议校验，不属于分包质量规则 | -| 02_08 | `test_02_08_stream_no_utf8_replacement_char_in_emoji` | 使用高密度 emoji 请求连续执行 20 次，检查任意 `delta.content` 中都不出现 `U+FFFD`（`�`）替换字符 | 防止 provider 把四字节 emoji 从 UTF-8 分帧中间截断（0823 innomatrix emoji 截断问题） | +| 02_08 | `test_02_08_stream_no_utf8_replacement_char_in_emoji` | 使用高密度 emoji 请求连续执行 20 次，检查任意 `delta.content` 中都不出现 `U+FFFD`（`�`）替换字符 | 防止 provider 把四字节 emoji 从 UTF-8 分帧中间截断 | | 01_01 | `01_01_essay_500_chars` | 直接返回约 500 字中文作文，默认执行 5 次 | 合并 5 次 `content` 分包后判定 | | 01_02 | `01_02_structured_json_1k` | 返回不少于 1KB 的结构化 JSON，默认执行 5 次 | 合并 5 次 `content` 分包后判定 | | 01_03 | `01_03_tool_content_string_500_chars` | 调用仅有 `content: string` 参数的 `save_content`，保存约 500 字作文 | `arguments` 分包 | diff --git a/m3_format_check/docs/m3_stream_cases_en.md b/m3_format_check/docs/m3_stream_cases_en.md index 6aeb476..a2a8f13 100644 --- a/m3_format_check/docs/m3_stream_cases_en.md +++ b/m3_format_check/docs/m3_stream_cases_en.md @@ -49,7 +49,7 @@ A target field with no non-empty fragments is an immediate `FAIL`. The decision |:---:|:---|:---|:---| | 02_06 | `test_02_06_stream_usage_only_in_last_chunk` | Text request with `stream_options.include_usage=true` | Independent SSE protocol check; not part of packet-quality rules | | 02_07 | `test_02_07_content_and_reasoning_content_not_coexist_in_chunk` | Run a long-reasoning, long-answer request 20 times and verify that non-empty `content` and `reasoning_content` never coexist in one `delta` | Independent SSE protocol check; not part of packet-quality rules | -| 02_08 | `test_02_08_stream_no_utf8_replacement_char_in_emoji` | Run an emoji-dense request 20 times and verify no `U+FFFD` (`�`) replacement char appears in any `delta.content` | Guards against providers truncating a 4-byte emoji mid-UTF-8-frame (0823 innomatrix emoji truncation issue) | +| 02_08 | `test_02_08_stream_no_utf8_replacement_char_in_emoji` | Run an emoji-dense request 20 times and verify no `U+FFFD` (`�`) replacement char appears in any `delta.content` | Guards against providers truncating a 4-byte emoji mid-UTF-8-frame | | 01_01 | `01_01_essay_500_chars` | Return an approximately 500-character Chinese essay five times | Merged `content` fragments from all five runs | | 01_02 | `01_02_structured_json_1k` | Return at least 1KB of structured JSON five times | Merged `content` fragments from all five runs | | 01_03 | `01_03_tool_content_string_500_chars` | Save an approximately 500-character essay through `save_content(content: string)` | `arguments` fragments | diff --git a/m3_format_check/m3_stream_tests.py b/m3_format_check/m3_stream_tests.py index 1f68b15..b0a0bca 100644 --- a/m3_format_check/m3_stream_tests.py +++ b/m3_format_check/m3_stream_tests.py @@ -857,8 +857,8 @@ def test_02_07_content_and_reasoning_content_not_coexist_in_chunk(self): def test_02_08_stream_no_utf8_replacement_char_in_emoji(self): """流式 delta.content 不得出现 U+FFFD(�)替换字符。 - 背景(0823 innomatrix emoji 字符截断问题):部分 provider 的输出链路 - 存在 UTF-8 分帧(framing)问题,会把一个四字节 emoji 从字节中间截断。 + 背景:部分 provider 的输出链路存在 UTF-8 分帧(framing)问题, + 会把一个四字节 emoji 从字节中间截断。 随后服务端把非法字节转成 EF BF BD,即 Unicode 替换字符 U+FFFD(�), 并丢弃剩余字节。因为返回的 JSON 里已经落成了 �,客户端无论如何缓存、 拼接都无法恢复原始 emoji,直接导致输出损坏。 From c1dd4e6cb83b7b9942991474ebe21ba2044697fb Mon Sep 17 00:00:00 2001 From: huahua Date: Wed, 26 Aug 2026 11:40:57 +0800 Subject: [PATCH 3/3] test(m3): move U+FFFD emoji case into text cases as test_02_07 Relocate the streaming U+FFFD emoji-truncation check from m3_stream_tests.py to m3_text_tests.py TestSSEStream (module 02 sse_stream) as test_02_07_stream_no_utf8_replacement_char_in_emoji, and move its documentation into the text-case reference docs (bumping the sse_stream count and total item count accordingly). --- m3_format_check/docs/m3_stream_cases.md | 3 +- m3_format_check/docs/m3_stream_cases_en.md | 3 +- m3_format_check/docs/m3_text_cases.md | 7 ++- m3_format_check/docs/m3_text_cases_en.md | 7 ++- m3_format_check/m3_stream_tests.py | 64 ---------------------- m3_format_check/m3_text_tests.py | 64 ++++++++++++++++++++++ 6 files changed, 74 insertions(+), 74 deletions(-) diff --git a/m3_format_check/docs/m3_stream_cases.md b/m3_format_check/docs/m3_stream_cases.md index d6f866e..d0cab18 100644 --- a/m3_format_check/docs/m3_stream_cases.md +++ b/m3_format_check/docs/m3_stream_cases.md @@ -4,7 +4,7 @@ ## 测试范围 -- `TestSSEStream`：校验 `stream_options.include_usage=true` 时 usage 仅出现在最后一个 chunk，连续 20 次请求中任一 `delta` 不会同时包含非空 `content` 和 `reasoning_content`，以及高密度 emoji 请求连续 20 次中 `delta.content` 都不出现 `U+FFFD`（`�`）替换字符。 +- `TestSSEStream`：校验 `stream_options.include_usage=true` 时 usage 仅出现在最后一个 chunk，以及连续 20 次请求中任一 `delta` 不会同时包含非空 `content` 和 `reasoning_content`。 - `TestContentStreamPacketLengthDistribution`：纯 content 场景默认执行 5 次，将 5 次的所有非空 `delta.content` 分包合并后统一判定。可通过 `M3_STREAM_CONTENT_RUNS` 调整次数。 - `TestToolCallStreamPacketLengthDistribution`：每个 tool call 场景执行 1 次，独立使用自己的阈值判定非空 `delta.tool_calls[].function.arguments` 分包。并行工具场景同时校验整体分布和每个已观察到的 `tool_call.index` 分布。 - 分包质量场景只关注分包质量，不校验正文长度、JSON 合法性、工具名称、参数内容、工具数量等功能正确性。 @@ -49,7 +49,6 @@ |:---:|:---|:---|:---| | 02_06 | `test_02_06_stream_usage_only_in_last_chunk` | `stream_options.include_usage=true` 文本场景 | 独立 SSE 协议校验，不属于分包质量规则 | | 02_07 | `test_02_07_content_and_reasoning_content_not_coexist_in_chunk` | 使用长推理和长回答请求连续执行 20 次，检查每个 `delta` 中非空 `content` 与 `reasoning_content` 不会同时出现 | 独立 SSE 协议校验，不属于分包质量规则 | -| 02_08 | `test_02_08_stream_no_utf8_replacement_char_in_emoji` | 使用高密度 emoji 请求连续执行 20 次，检查任意 `delta.content` 中都不出现 `U+FFFD`（`�`）替换字符 | 防止 provider 把四字节 emoji 从 UTF-8 分帧中间截断 | | 01_01 | `01_01_essay_500_chars` | 直接返回约 500 字中文作文，默认执行 5 次 | 合并 5 次 `content` 分包后判定 | | 01_02 | `01_02_structured_json_1k` | 返回不少于 1KB 的结构化 JSON，默认执行 5 次 | 合并 5 次 `content` 分包后判定 | | 01_03 | `01_03_tool_content_string_500_chars` | 调用仅有 `content: string` 参数的 `save_content`，保存约 500 字作文 | `arguments` 分包 | diff --git a/m3_format_check/docs/m3_stream_cases_en.md b/m3_format_check/docs/m3_stream_cases_en.md index a2a8f13..cc0bb06 100644 --- a/m3_format_check/docs/m3_stream_cases_en.md +++ b/m3_format_check/docs/m3_stream_cases_en.md @@ -4,7 +4,7 @@ ## Scope -- `TestSSEStream`: verifies that usage appears only in the final chunk when `stream_options.include_usage=true`, that non-empty `content` and `reasoning_content` never coexist in a `delta` across 20 requests, and that no `U+FFFD` (`�`) replacement char appears in `delta.content` for an emoji-dense request across 20 requests. +- `TestSSEStream`: verifies that usage appears only in the final chunk when `stream_options.include_usage=true`, and that non-empty `content` and `reasoning_content` never coexist in a `delta` across 20 requests. - `TestContentStreamPacketLengthDistribution`: each pure-content scenario runs five times by default. All non-empty `delta.content` fragments from the five runs are merged and evaluated once. Override the run count with `M3_STREAM_CONTENT_RUNS`. - `TestToolCallStreamPacketLengthDistribution`: each tool-call scenario runs once and uses its own thresholds for non-empty `delta.tool_calls[].function.arguments` fragments. The parallel-tool scenario evaluates both the combined distribution and every observed `tool_call.index` distribution. - The packet-quality scenarios evaluate packet-size quality only. They do not validate output length, JSON validity, tool names, argument contents, or tool-call counts. @@ -49,7 +49,6 @@ A target field with no non-empty fragments is an immediate `FAIL`. The decision |:---:|:---|:---|:---| | 02_06 | `test_02_06_stream_usage_only_in_last_chunk` | Text request with `stream_options.include_usage=true` | Independent SSE protocol check; not part of packet-quality rules | | 02_07 | `test_02_07_content_and_reasoning_content_not_coexist_in_chunk` | Run a long-reasoning, long-answer request 20 times and verify that non-empty `content` and `reasoning_content` never coexist in one `delta` | Independent SSE protocol check; not part of packet-quality rules | -| 02_08 | `test_02_08_stream_no_utf8_replacement_char_in_emoji` | Run an emoji-dense request 20 times and verify no `U+FFFD` (`�`) replacement char appears in any `delta.content` | Guards against providers truncating a 4-byte emoji mid-UTF-8-frame | | 01_01 | `01_01_essay_500_chars` | Return an approximately 500-character Chinese essay five times | Merged `content` fragments from all five runs | | 01_02 | `01_02_structured_json_1k` | Return at least 1KB of structured JSON five times | Merged `content` fragments from all five runs | | 01_03 | `01_03_tool_content_string_500_chars` | Save an approximately 500-character essay through `save_content(content: string)` | `arguments` fragments | diff --git a/m3_format_check/docs/m3_text_cases.md b/m3_format_check/docs/m3_text_cases.md index 0d92925..018c004 100644 --- a/m3_format_check/docs/m3_text_cases.md +++ b/m3_format_check/docs/m3_text_cases.md @@ -9,7 +9,7 @@ | 模块编号 | 模块名 | 主题 | 函数数 | |:---:|:---|:---|:---:| | 01 | basic_text | 基础文本对话(非流式) | 3 | -| 02 | sse_stream | SSE 流式协议字段 | 6 | +| 02 | sse_stream | SSE 流式协议字段 | 7 | | 03 | multiturn | 多轮对话 | 2 | | 04 | thinking | thinking 思考开关 | 4 | | 05 | sampling | 采样参数(temperature / top_p / seed) | 3 | @@ -49,6 +49,7 @@ | 02_04 | `test_02_04_stream_chunk_fields` | 流式 chunk 必带字段 | id / choices / object 全部存在 | | 02_05 | `test_02_05_text_include_usage` | `stream_options.include_usage=true` 文本场景 | 流应正常返回 usage chunk | | 02_06 | `test_02_06_stream_usage_only_in_last_chunk` | `stream_options.include_usage=true` 文本场景 | usage 非空且三字段 > 0,且只出现在流式最后一个 data chunk | +| 02_07 | `test_02_07_stream_no_utf8_replacement_char_in_emoji` | 高密度 emoji 流式请求连续执行 20 次 | 任意 `delta.content` 中都不出现 `U+FFFD`（`�`）替换字符;防止四字节 emoji 被 UTF-8 分帧从中间截断 | ## 03 multiturn — 多轮对话 @@ -253,7 +254,7 @@ --- -## 附录:parametrize 展开后的 155 个 items +## 附录:parametrize 展开后的 156 个 items 凡函数签名带 `@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])` 的会展开为 2 个 items;`max_tokens` 的两个 case 各展开为 2 个 items。 @@ -264,4 +265,4 @@ | `mt ∈ {512000, 524288}` | 06_09 | | `mt ∈ {524289, 1000000}` | 06_10 | -总 items = 116 函数 - 34 (`stream` 双值函数) - 1 (`ctx_tokens × stream` 函数) - 2 (`mt` 双值函数) + 34×2 + 1×4 + 2×2 = **155**。 +总 items = 117 函数 - 34 (`stream` 双值函数) - 1 (`ctx_tokens × stream` 函数) - 2 (`mt` 双值函数) + 34×2 + 1×4 + 2×2 = **156**。 diff --git a/m3_format_check/docs/m3_text_cases_en.md b/m3_format_check/docs/m3_text_cases_en.md index 6be6e41..4bd490c 100644 --- a/m3_format_check/docs/m3_text_cases_en.md +++ b/m3_format_check/docs/m3_text_cases_en.md @@ -9,7 +9,7 @@ | Module ID | Module Name | Theme | Functions | |:---:|:---|:---|:---:| | 01 | basic_text | Basic text conversation (non-stream) | 3 | -| 02 | sse_stream | SSE streaming protocol fields | 6 | +| 02 | sse_stream | SSE streaming protocol fields | 7 | | 03 | multiturn | Multi-turn conversation | 2 | | 04 | thinking | thinking toggle | 4 | | 05 | sampling | Sampling params (temperature / top_p / seed) | 3 | @@ -49,6 +49,7 @@ | 02_04 | `test_02_04_stream_chunk_fields` | Stream chunk required fields | id / choices / object all present | | 02_05 | `test_02_05_text_include_usage` | `stream_options.include_usage=true` (text) | Stream should return usage chunk | | 02_06 | `test_02_06_stream_usage_only_in_last_chunk` | `stream_options.include_usage=true` (text) | usage non-empty with three positive token fields, present only in the final data chunk | +| 02_07 | `test_02_07_stream_no_utf8_replacement_char_in_emoji` | Emoji-dense streaming request run 20 times | No `U+FFFD` (`�`) replacement char in any `delta.content`; guards against a 4-byte emoji being truncated mid-UTF-8-frame | ## 03 multiturn — Multi-turn conversation @@ -254,7 +255,7 @@ --- -## Appendix: 155 items after parametrize expansion +## Appendix: 156 items after parametrize expansion Functions decorated with `@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"])` expand to 2 items each; the two `max_tokens` parametrized cases each expand to 2 items. @@ -265,4 +266,4 @@ Functions decorated with `@pytest.mark.parametrize("stream", [False, True], ids= | `mt ∈ {512000, 524288}` | 06_09 | | `mt ∈ {524289, 1000000}` | 06_10 | -Total items = 116 functions - 34 two-value `stream` functions - 1 `ctx_tokens × stream` function - 2 two-value `mt` functions + 34×2 + 1×4 + 2×2 = **155**. +Total items = 117 functions - 34 two-value `stream` functions - 1 `ctx_tokens × stream` function - 2 two-value `mt` functions + 34×2 + 1×4 + 2×2 = **156**. diff --git a/m3_format_check/m3_stream_tests.py b/m3_format_check/m3_stream_tests.py index b0a0bca..ac7043d 100644 --- a/m3_format_check/m3_stream_tests.py +++ b/m3_format_check/m3_stream_tests.py @@ -854,70 +854,6 @@ def test_02_07_content_and_reasoning_content_not_coexist_in_chunk(self): f"content 与 reasoning_content,示例: {first_failure['samples']}" ) - def test_02_08_stream_no_utf8_replacement_char_in_emoji(self): - """流式 delta.content 不得出现 U+FFFD(�)替换字符。 - - 背景:部分 provider 的输出链路存在 UTF-8 分帧(framing)问题, - 会把一个四字节 emoji 从字节中间截断。 - 随后服务端把非法字节转成 EF BF BD,即 Unicode 替换字符 U+FFFD(�), - 并丢弃剩余字节。因为返回的 JSON 里已经落成了 �,客户端无论如何缓存、 - 拼接都无法恢复原始 emoji,直接导致输出损坏。 - - 用大量四字节 emoji + 编号的 prompt 提高覆盖:emoji 越密集,分帧边界 - 恰好落在多字节字符中间的概率越高,越容易暴露实现里 flush 的字节截断问题。 - 连续跑 20 次,任意一次在聚合 content 或任一 delta 分片里检测到 U+FFFD - 即判失败。 - """ - prompt = ( - "请用一行输出 50 个各不相同的 emoji,格式为 '文本 N.'," - "以空格分隔,N 从 1 到 50。必须正好 50 个,不重复,不要任何解释说明。" - "尽量多使用四字节(surrogate pair)的彩色 emoji,如 😁😂🤣😊🙃🥰🤩🤔🥱🤪🧐🥺。" - ) - run_count = 20 - first_failure = None - for run_idx in range(1, run_count + 1): - result = oai_chat({ - "messages": oai_simple_messages(prompt), - "max_tokens": 4096, - }, stream=True) - assert_oai_stream_success(result) - - offenders = [] - for idx, chunk in enumerate(result.get("chunks") or []): - if not isinstance(chunk, dict): - continue - for choice_idx, choice in enumerate(chunk.get("choices") or []): - if not isinstance(choice, dict): - continue - delta = choice.get("delta") or {} - if not isinstance(delta, dict): - continue - content = delta.get("content") - if isinstance(content, str) and "\ufffd" in content: - offenders.append({ - "chunk_index": idx, - "choice_index": choice_idx, - "fffd_count": content.count("\ufffd"), - "content_preview": content[:80], - }) - - if offenders: - first_failure = { - "run": run_idx, - "offender_count": len(offenders), - "total_fffd": sum(o["fffd_count"] for o in offenders), - "samples": offenders[:3], - } - break - - assert first_failure is None, ( - f"02_08 连续 {run_count} 次中第 {first_failure['run']} 次在流式 " - f"delta.content 检测到 U+FFFD(�)替换字符,涉及 " - f"{first_failure['offender_count']} 个 chunk、共 " - f"{first_failure['total_fffd']} 个 �,疑似 UTF-8 分帧把四字节 emoji " - f"从中间截断;示例: {first_failure['samples']}" - ) - class TestContentStreamPacketLengthDistribution: """Run pure-content scenarios repeatedly, then aggregate packet statistics.""" diff --git a/m3_format_check/m3_text_tests.py b/m3_format_check/m3_text_tests.py index 3aec3ae..4ae6523 100644 --- a/m3_format_check/m3_text_tests.py +++ b/m3_format_check/m3_text_tests.py @@ -150,6 +150,70 @@ def test_02_06_stream_usage_only_in_last_chunk(self): assert_oai_stream_success(r) assert_stream_usage_only_in_last_chunk(r, msg="02_06 text include_usage") + def test_02_07_stream_no_utf8_replacement_char_in_emoji(self): + """流式 delta.content 不得出现 U+FFFD(�)替换字符。 + + 背景:部分 provider 的输出链路存在 UTF-8 分帧(framing)问题, + 会把一个四字节 emoji 从字节中间截断。 + 随后服务端把非法字节转成 EF BF BD,即 Unicode 替换字符 U+FFFD(�), + 并丢弃剩余字节。因为返回的 JSON 里已经落成了 �,客户端无论如何缓存、 + 拼接都无法恢复原始 emoji,直接导致输出损坏。 + + 用大量四字节 emoji + 编号的 prompt 提高覆盖:emoji 越密集,分帧边界 + 恰好落在多字节字符中间的概率越高,越容易暴露实现里 flush 的字节截断问题。 + 连续跑 20 次,任意一次在聚合 content 或任一 delta 分片里检测到 U+FFFD + 即判失败。 + """ + prompt = ( + "请用一行输出 50 个各不相同的 emoji,格式为 '文本 N.'," + "以空格分隔,N 从 1 到 50。必须正好 50 个,不重复,不要任何解释说明。" + "尽量多使用四字节(surrogate pair)的彩色 emoji,如 😁😂🤣😊🙃🥰🤩🤔🥱🤪🧐🥺。" + ) + run_count = 20 + first_failure = None + for run_idx in range(1, run_count + 1): + result = oai_chat({ + "messages": oai_simple_messages(prompt), + "max_tokens": 4096, + }, stream=True) + assert_oai_stream_success(result) + + offenders = [] + for idx, chunk in enumerate(result.get("chunks") or []): + if not isinstance(chunk, dict): + continue + for choice_idx, choice in enumerate(chunk.get("choices") or []): + if not isinstance(choice, dict): + continue + delta = choice.get("delta") or {} + if not isinstance(delta, dict): + continue + content = delta.get("content") + if isinstance(content, str) and "\ufffd" in content: + offenders.append({ + "chunk_index": idx, + "choice_index": choice_idx, + "fffd_count": content.count("\ufffd"), + "content_preview": content[:80], + }) + + if offenders: + first_failure = { + "run": run_idx, + "offender_count": len(offenders), + "total_fffd": sum(o["fffd_count"] for o in offenders), + "samples": offenders[:3], + } + break + + assert first_failure is None, ( + f"02_07 连续 {run_count} 次中第 {first_failure['run']} 次在流式 " + f"delta.content 检测到 U+FFFD(�)替换字符,涉及 " + f"{first_failure['offender_count']} 个 chunk、共 " + f"{first_failure['total_fffd']} 个 �,疑似 UTF-8 分帧把四字节 emoji " + f"从中间截断;示例: {first_failure['samples']}" + ) + # ============================================================ # 03 multiturn — multi-turn conversation