xiaxinhang/pr-agent
1
0
Fork 0
mirror of https://github.com/qodo-ai/pr-agent.git synced 2025-07-03 04:10:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

TokenEncoder

Browse Source
This commit is contained in:
mrT23
2024-04-03 08:42:50 +03:00
parent a13c6e964b
commit 9c3673209d
2 changed files with 20 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pr_agent/algo/utils.py
View File

@ -12,7 +12,7 @@ import yaml
from starlette_context import context
from pr_agent.algo import MAX_TOKENS
from pr_agent.algo.token_handler import get_token_encoder
from pr_agent.algo.token_handler import TokenEncoder
from pr_agent.config_loader import get_settings, global_settings
from pr_agent.algo.types import FilePatchInfo
from pr_agent.log import get_logger
@ -566,7 +566,7 @@ def clip_tokens(text: str, max_tokens: int, add_three_dots=True) -> str:
return text
try:
encoder = get_token_encoder()
encoder = TokenEncoder.get_token_encoder()
num_input_tokens = len(encoder.encode(text))
if num_input_tokens <= max_tokens:
return text