From 08bf9593b2bcad5b32759bae230913dd26b3e36e Mon Sep 17 00:00:00 2001 From: mrT23 Date: Mon, 14 Apr 2025 21:15:19 +0300 Subject: [PATCH] Fix tokenizer fallback to use o200k_base instead of cl100k_base --- pr_agent/algo/token_handler.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pr_agent/algo/token_handler.py b/pr_agent/algo/token_handler.py index f1393e38..60cf2c84 100644 --- a/pr_agent/algo/token_handler.py +++ b/pr_agent/algo/token_handler.py @@ -19,8 +19,11 @@ class TokenEncoder: with cls._lock: # Lock acquisition to ensure thread safety if cls._encoder_instance is None or model != cls._model: cls._model = model - cls._encoder_instance = encoding_for_model(cls._model) if "gpt" in cls._model else get_encoding( - "cl100k_base") + try: + cls._encoder_instance = encoding_for_model(cls._model) if "gpt" in cls._model else get_encoding( + "o200k_base") + except: + cls._encoder_instance = get_encoding("o200k_base") return cls._encoder_instance