Add large_patch_policy configuration and implement patch clipping logic

This commit is contained in:
mrT23
2024-05-29 14:00:04 +03:00
parent 911c1268fc
commit 6e6f54933e

View File

@ -15,5 +15,5 @@ class TestClipTokens:
max_tokens = 10
result = clip_tokens(text, max_tokens)
expected_results = 'line1\nline2\nline3\nli\n...(truncated)'
expected_results = 'line1\nline2\nline3\n\n...(truncated)'
assert result == expected_results