diff --git a/pr_agent/algo/token_handler.py b/pr_agent/algo/token_handler.py index 52ceb49f..53fb3ac9 100644 --- a/pr_agent/algo/token_handler.py +++ b/pr_agent/algo/token_handler.py @@ -21,4 +21,4 @@ class TokenHandler: return system_prompt_tokens + user_prompt_tokens def count_tokens(self, patch: str) -> int: - return len(self.encoder.encode(patch)) \ No newline at end of file + return len(self.encoder.encode(patch, disallowed_special=())) \ No newline at end of file