From b206b1c5ff42ed77943b72d0758f89d050348027 Mon Sep 17 00:00:00 2001 From: Ori Kotek Date: Thu, 10 Aug 2023 02:08:36 +0300 Subject: [PATCH] Protect for empty description --- pr_agent/algo/pr_processing.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/pr_agent/algo/pr_processing.py b/pr_agent/algo/pr_processing.py index b195f9f4..3a08a86d 100644 --- a/pr_agent/algo/pr_processing.py +++ b/pr_agent/algo/pr_processing.py @@ -298,12 +298,16 @@ def clip_tokens(text: str, max_tokens: int) -> str: str: The clipped string. """ # We'll estimate the number of tokens by hueristically assuming 2.5 tokens per word - encoder = get_token_encoder() - num_input_tokens = len(encoder.encode(text)) - if num_input_tokens <= max_tokens: - return text - num_chars = len(text) - chars_per_token = num_chars / num_input_tokens - num_output_chars = int(chars_per_token * max_tokens) - clipped_text = text[:num_output_chars] - return clipped_text + try: + encoder = get_token_encoder() + num_input_tokens = len(encoder.encode(text)) + if num_input_tokens <= max_tokens: + return text + num_chars = len(text) + chars_per_token = num_chars / num_input_tokens + num_output_chars = int(chars_per_token * max_tokens) + clipped_text = text[:num_output_chars] + return clipped_text + except Exception as e: + logging.warning(f"Failed to clip tokens: {e}") + return text \ No newline at end of file