xiaxinhang/pr-agent
1
0
Fork 0
mirror of https://github.com/qodo-ai/pr-agent.git synced 2025-07-05 13:20:39 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #7 from Codium-ai/algo/fix_speacial_tokens

Browse Source 
Fix encoding error on special_tokens
This commit is contained in:
Ori Kotek
2023-07-06 14:14:52 +03:00
committed by GitHub
parent d0c544e650 259fa84eeb
commit 1eea60c6a5
1 changed files with 1 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pr_agent/algo/token_handler.py
View File

@ -21,4 +21,4 @@ class TokenHandler:
return system_prompt_tokens + user_prompt_tokens return system_prompt_tokens + user_prompt_tokens
def count_tokens(self, patch: str) -> int: def count_tokens(self, patch: str) -> int:
return len(self.encoder.encode(patch)) return len(self.encoder.encode(patch, disallowed_special=()))