diff --git a/codewiki/src/be/utils.py b/codewiki/src/be/utils.py
index e4cc795d..bf0571de 100644
--- a/codewiki/src/be/utils.py
+++ b/codewiki/src/be/utils.py
@@ -56,7 +56,9 @@ def count_tokens(text: str) -> int:
     """
     Count the number of tokens in a text.
     """
-    length = len(enc.encode(text))
+    # source code may contain literal special tokens such as "<|endoftext|>"
+    # (e.g. CLIP tokenizers); count them as plain text instead of raising
+    length = len(enc.encode(text, disallowed_special=()))
     # logger.debug(f"Number of tokens: {length}")
     return length