xiaxinhang/pr-agent
1
0
Fork 0
mirror of https://github.com/qodo-ai/pr-agent.git synced 2025-07-03 20:30:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Protect for empty description

Browse Source
This commit is contained in:
Ori Kotek
2023-08-10 02:08:36 +03:00
parent e00500b90c
commit b206b1c5ff
1 changed files with 13 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
pr_agent/algo/pr_processing.py
View File

@ -298,12 +298,16 @@ def clip_tokens(text: str, max_tokens: int) -> str:
str: The clipped string. str: The clipped string.
""" """
# We'll estimate the number of tokens by hueristically assuming 2.5 tokens per word # We'll estimate the number of tokens by hueristically assuming 2.5 tokens per word
encoder = get_token_encoder() try:
num_input_tokens = len(encoder.encode(text)) encoder = get_token_encoder()
if num_input_tokens <= max_tokens: num_input_tokens = len(encoder.encode(text))
return text if num_input_tokens <= max_tokens:
num_chars = len(text) return text
chars_per_token = num_chars / num_input_tokens num_chars = len(text)
num_output_chars = int(chars_per_token * max_tokens) chars_per_token = num_chars / num_input_tokens
clipped_text = text[:num_output_chars] num_output_chars = int(chars_per_token * max_tokens)
return clipped_text clipped_text = text[:num_output_chars]
return clipped_text
except Exception as e:
logging.warning(f"Failed to clip tokens: {e}")
return text