From ede52a20b80126e78d61fd0679a47d7f39a981e6 Mon Sep 17 00:00:00 2001
From: Zedong <xihaolong@gmail.com>
Date: Sun, 1 Nov 2026 23:47:13 +0800
Subject: [PATCH] fix(llm): use UTF-8 for model CLI text pipes

---
 openevolve/llm/claude_code.py      |  1 +
 openevolve/llm/codex_app_server.py |  1 +
 openevolve/llm/copilot_cli.py      |  1 +
 tests/test_cli_utf8.py             | 81 ++++++++++++++++++++++++++++++
 4 files changed, 84 insertions(+)
 create mode 100644 tests/test_cli_utf8.py

diff --git a/openevolve/llm/claude_code.py b/openevolve/llm/claude_code.py
index 5bd3c9fb3..0842e39fa 100644
--- a/openevolve/llm/claude_code.py
+++ b/openevolve/llm/claude_code.py
@@ -120,6 +120,7 @@ def _run_cli(self, cmd: list, timeout: int, prompt: Optional[str] = None) -> str
                 input=prompt,
                 capture_output=True,
                 text=True,
+                encoding="utf-8",
                 timeout=timeout,
                 cwd=self.cwd,
             )
diff --git a/openevolve/llm/codex_app_server.py b/openevolve/llm/codex_app_server.py
index b0800bdbc..5b71d2e40 100644
--- a/openevolve/llm/codex_app_server.py
+++ b/openevolve/llm/codex_app_server.py
@@ -355,6 +355,7 @@ def _start_session(self) -> _AppServerSession:
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             text=True,
+            encoding="utf-8",
             cwd=self.cwd,
         )
         return _AppServerSession(proc)
diff --git a/openevolve/llm/copilot_cli.py b/openevolve/llm/copilot_cli.py
index 1b70ecc7f..53ebd806a 100644
--- a/openevolve/llm/copilot_cli.py
+++ b/openevolve/llm/copilot_cli.py
@@ -152,6 +152,7 @@ def _run_cli(self, cmd: List[str], timeout: int) -> str:
                 cmd,
                 capture_output=True,
                 text=True,
+                encoding="utf-8",
                 timeout=timeout,
                 cwd=self.cwd,
             )
diff --git a/tests/test_cli_utf8.py b/tests/test_cli_utf8.py
new file mode 100644
index 000000000..a70747ee4
--- /dev/null
+++ b/tests/test_cli_utf8.py
@@ -0,0 +1,81 @@
+"""Exercise CLI text pipes with real processes under a non-UTF-8 locale."""
+
+import os
+import subprocess
+import sys
+import unittest
+
+CHECK_CLI = r"""
+import json
+import locale
+import sys
+import time
+from unittest.mock import patch
+
+if locale.getpreferredencoding(False).lower().replace('-', '') == 'utf8':
+    sys.exit(77)
+
+text = chr(0x4E2D) + chr(0x6587) + chr(0x1F9EA)
+write_text = 'import sys; sys.stdout.buffer.write(' + repr(text.encode('utf-8')) + ')'
+command = [sys.executable, '-c', write_text]
+case = sys.argv[1]
+
+if case in ('claude_output', 'claude_input'):
+    from openevolve.llm.claude_code import ClaudeCodeLLM
+    model = ClaudeCodeLLM()
+    if case == 'claude_input':
+        command = [sys.executable, '-c', 'import sys; sys.stdout.buffer.write(sys.stdin.buffer.read())']
+        result = model._run_cli(command, timeout=5, prompt=text)
+    else:
+        result = model._run_cli(command, timeout=5)
+elif case == 'copilot_output':
+    from openevolve.llm.copilot_cli import CopilotCLILLM
+    result = CopilotCLILLM()._run_cli(command, timeout=5)
+else:
+    from openevolve.llm.codex_app_server import CodexAppServerLLM
+    payload = {'method': 'example', 'params': {'text': text}}
+    wire = (json.dumps(payload, ensure_ascii=False) + chr(10)).encode('utf-8')
+    command = [sys.executable, '-c', 'import sys; sys.stdout.buffer.write(' + repr(wire) + ')']
+    model = CodexAppServerLLM()
+    with patch.object(model, '_build_command', return_value=command):
+        session = model._start_session()
+    try:
+        message = session.next_message(time.monotonic() + 5)
+        assert message['method'] == 'example'
+        result = message['params']['text']
+    finally:
+        session.close()
+
+assert result == text, (ascii(result), ascii(text))
+"""
+
+
+class TestCliUtf8(unittest.TestCase):
+    def _check_cli(self, case):
+        env = dict(os.environ, LC_ALL="C", PYTHONUTF8="0", PYTHONCOERCECLOCALE="0")
+        result = subprocess.run(
+            [sys.executable, "-c", CHECK_CLI, case],
+            capture_output=True,
+            encoding="utf-8",
+            timeout=15,
+            env=env,
+        )
+        if result.returncode == 77:
+            self.skipTest("This platform retains a UTF-8 default locale")
+        self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_claude_reads_utf8_output(self):
+        self._check_cli("claude_output")
+
+    def test_claude_sends_utf8_stdin(self):
+        self._check_cli("claude_input")
+
+    def test_copilot_reads_utf8_output(self):
+        self._check_cli("copilot_output")
+
+    def test_codex_reads_utf8_jsonl(self):
+        self._check_cli("codex_output")
+
+
+if __name__ == "__main__":
+    unittest.main()