From 28d373480396959399aae1e7f91074ae4f241de0 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Wed, 7 Oct 2026 17:47:35 +0700 Subject: [PATCH] fix: count literal special tokens like <|endoftext|> as plain text Repos that vendor tokenizer code (e.g. CLIP) contain the literal string '<|endoftext|>'. tiktoken rejects it by default, so count_tokens raised and module clustering aborted before any LLM call. --- codewiki/src/be/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/codewiki/src/be/utils.py b/codewiki/src/be/utils.py index e4cc795d..bf0571de 100644 --- a/codewiki/src/be/utils.py +++ b/codewiki/src/be/utils.py @@ -56,7 +56,9 @@ def count_tokens(text: str) -> int: """ Count the number of tokens in a text. """ - length = len(enc.encode(text)) + # source code may contain literal special tokens such as "<|endoftext|>" + # (e.g. CLIP tokenizers); count them as plain text instead of raising + length = len(enc.encode(text, disallowed_special=())) # logger.debug(f"Number of tokens: {length}") return length