From 259fa84eeb19a651189e0707b69c16219a9da856 Mon Sep 17 00:00:00 2001
From: "Hussam.lawen" <hussam789@gmail.com>
Date: Thu, 6 Jul 2023 13:22:12 +0300
Subject: [PATCH] disabling encoding error on special_tokens

---
 pr_agent/algo/token_handler.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pr_agent/algo/token_handler.py b/pr_agent/algo/token_handler.py
index 52ceb49f..53fb3ac9 100644
--- a/pr_agent/algo/token_handler.py
+++ b/pr_agent/algo/token_handler.py
@@ -21,4 +21,4 @@ class TokenHandler:
         return system_prompt_tokens + user_prompt_tokens
 
     def count_tokens(self, patch: str) -> int:
-        return len(self.encoder.encode(patch))
\ No newline at end of file
+        return len(self.encoder.encode(patch, disallowed_special=()))
\ No newline at end of file