xiaxinhang/pr-agent
1
0
Fork 0
mirror of https://github.com/qodo-ai/pr-agent.git synced 2025-07-03 04:10:49 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feature/gha-outputs-1

Browse Source
This commit is contained in:
idubnori
2024-04-10 23:27:44 +09:00
committed by GitHub
parent aef1c6ecde f0c5aec0e4
commit 9e4ffd824c
48 changed files with 441 additions and 429 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
pr_agent/algo/utils.py
View File

@ -13,7 +13,7 @@ import yaml
from starlette_context import context
from pr_agent.algo import MAX_TOKENS
from pr_agent.algo.token_handler import get_token_encoder
from pr_agent.algo.token_handler import TokenEncoder
from pr_agent.config_loader import get_settings, global_settings
from pr_agent.algo.types import FilePatchInfo
from pr_agent.log import get_logger
@ -567,7 +567,7 @@ def clip_tokens(text: str, max_tokens: int, add_three_dots=True) -> str:
return text
try:
encoder = get_token_encoder()
encoder = TokenEncoder.get_token_encoder()
num_input_tokens = len(encoder.encode(text))
if num_input_tokens <= max_tokens:
return text
@ -576,7 +576,7 @@ def clip_tokens(text: str, max_tokens: int, add_three_dots=True) -> str:
num_output_chars = int(chars_per_token * max_tokens)
clipped_text = text[:num_output_chars]
if add_three_dots:
clipped_text += "...(truncated)"
clipped_text += "\n...(truncated)"
return clipped_text
except Exception as e:
get_logger().warning(f"Failed to clip tokens: {e}")