From 4edb8b89d14e65d2080a0aac73c2513d6b916f16 Mon Sep 17 00:00:00 2001 From: mrT23 Date: Tue, 18 Feb 2025 11:46:22 +0200 Subject: [PATCH 1/3] feat: add support for custom reasoning models --- docs/docs/usage-guide/changing_a_model.md | 3 +++ pr_agent/algo/ai_handlers/litellm_ai_handler.py | 4 ++-- pr_agent/settings/configuration.toml | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/docs/usage-guide/changing_a_model.md b/docs/docs/usage-guide/changing_a_model.md index 420f3736..d221b953 100644 --- a/docs/docs/usage-guide/changing_a_model.md +++ b/docs/docs/usage-guide/changing_a_model.md @@ -201,3 +201,6 @@ fallback_models=["custom_model_name"] custom_model_max_tokens= ... ``` (3) Go to [litellm documentation](https://litellm.vercel.app/docs/proxy/quick_start#supported-llms), find the model you want to use, and set the relevant environment variables. + +(4) Most reasoning models do not support chat-style inputs (`system` and `user` messages) or temperature settings. +To bypass chat templates and temperature controls, set `config.custom_reasoning_model = true` in your configuration file. diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index 0ef29b51..22a0c096 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -205,7 +205,7 @@ class LiteLLMAIHandler(BaseAiHandler): {"type": "image_url", "image_url": {"url": img_path}}] # Currently, some models do not support a separate system and user prompts - if model in self.user_message_only_models: + if model in self.user_message_only_models or get_settings().config.custom_reasoning_model: user = f"{system}\n\n\n{user}" system = "" get_logger().info(f"Using model {model}, combining system and user prompts") @@ -227,7 +227,7 @@ class LiteLLMAIHandler(BaseAiHandler): } # Add temperature only if model supports it - if model not in self.no_support_temperature_models: + if model not in self.no_support_temperature_models or get_settings().config.custom_reasoning_model: kwargs["temperature"] = temperature if get_settings().litellm.get("enable_callbacks", False): diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index 01c09207..fc4c6548 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -17,6 +17,7 @@ use_global_settings_file=true disable_auto_feedback = false ai_timeout=120 # 2minutes skip_keys = [] +custom_reasoning_model = false # when true, no system message and not temperature will be used # token limits max_description_tokens = 500 max_commits_tokens = 500 From 0317951e32d6012cf21eb8c3a0a9c068543c6a4d Mon Sep 17 00:00:00 2001 From: Tal Date: Tue, 18 Feb 2025 11:48:25 +0200 Subject: [PATCH 2/3] Update pr_agent/settings/configuration.toml Co-authored-by: qodo-merge-pro-for-open-source[bot] <189517486+qodo-merge-pro-for-open-source[bot]@users.noreply.github.com> --- pr_agent/settings/configuration.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index fc4c6548..ad09243a 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -17,7 +17,7 @@ use_global_settings_file=true disable_auto_feedback = false ai_timeout=120 # 2minutes skip_keys = [] -custom_reasoning_model = false # when true, no system message and not temperature will be used +custom_reasoning_model = false # when true, disables system messages and temperature controls for models that don't support chat-style inputs # token limits max_description_tokens = 500 max_commits_tokens = 500 From 35059cadf7fadc67aedb78f6ad57982c42783629 Mon Sep 17 00:00:00 2001 From: Tal Date: Tue, 18 Feb 2025 11:50:48 +0200 Subject: [PATCH 3/3] Update pr_agent/algo/ai_handlers/litellm_ai_handler.py Co-authored-by: qodo-merge-pro-for-open-source[bot] <189517486+qodo-merge-pro-for-open-source[bot]@users.noreply.github.com> --- pr_agent/algo/ai_handlers/litellm_ai_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index 22a0c096..b22b834e 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -227,7 +227,7 @@ class LiteLLMAIHandler(BaseAiHandler): } # Add temperature only if model supports it - if model not in self.no_support_temperature_models or get_settings().config.custom_reasoning_model: + if model not in self.no_support_temperature_models and not get_settings().config.custom_reasoning_model: kwargs["temperature"] = temperature if get_settings().litellm.get("enable_callbacks", False):