xiaxinhang/pr-agent
1
0
Fork 0
mirror of https://github.com/qodo-ai/pr-agent.git synced 2025-07-04 12:50:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

disabling encoding error on special_tokens

Browse Source
This commit is contained in:
Hussam.lawen
2023-07-06 13:22:12 +03:00
parent 74dfae8dbe
commit 259fa84eeb
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pr_agent/algo/token_handler.py
View File

@ -21,4 +21,4 @@ class TokenHandler:
return system_prompt_tokens + user_prompt_tokens return system_prompt_tokens + user_prompt_tokens
def count_tokens(self, patch: str) -> int: def count_tokens(self, patch: str) -> int:
return len(self.encoder.encode(patch)) return len(self.encoder.encode(patch, disallowed_special=()))