refactor: Move clip_tokens function from pr_processing to utils module, and add tests

This commit is contained in:
mrT23
2023-11-26 08:29:47 +02:00
parent d7df4287f8
commit 9465b7b577
7 changed files with 58 additions and 37 deletions

View File

@ -0,0 +1,19 @@
# Generated by CodiumAI
import pytest
from pr_agent.algo.utils import clip_tokens
class TestClipTokens:
def test_clip(self):
text = "line1\nline2\nline3\nline4\nline5\nline6"
max_tokens = 25
result = clip_tokens(text, max_tokens)
assert result == text
max_tokens = 10
result = clip_tokens(text, max_tokens)
expected_results = 'line1\nline2\nline3\nli...(truncated)'
assert result == expected_results