From 5447dd2ac60cfd604c1b704b1c47f2f6d131dd70 Mon Sep 17 00:00:00 2001 From: Kenny Dizi Date: Mon, 3 Mar 2025 18:21:57 +0700 Subject: [PATCH 01/11] Add support claude extended thinking configurations --- pr_agent/settings/configuration.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index 4e004722..b04cb424 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -58,6 +58,10 @@ reasoning_effort = "medium" # "low", "medium", "high" enable_auto_approval=false # Set to true to enable auto-approval of PRs under certain conditions auto_approve_for_low_review_effort=-1 # -1 to disable, [1-5] to set the threshold for auto-approval auto_approve_for_no_suggestions=false # If true, the PR will be auto-approved if there are no suggestions +# extended thinking +enable_claude_extended_thinking = false # Set to true to enable extended thinking feature +extended_thinking_budget_tokens = 32000 +extended_thinking_max_output_tokens = 64000 [pr_reviewer] # /review # From 5e30e190b891c413eb7825ae695bd2b914a46663 Mon Sep 17 00:00:00 2001 From: Kenny Dizi Date: Mon, 3 Mar 2025 18:22:31 +0700 Subject: [PATCH 02/11] Define models that support extended thinking feature --- pr_agent/algo/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pr_agent/algo/__init__.py b/pr_agent/algo/__init__.py index 34eac5e1..72804dfc 100644 --- a/pr_agent/algo/__init__.py +++ b/pr_agent/algo/__init__.py @@ -60,6 +60,7 @@ MAX_TOKENS = { 'anthropic/claude-3-5-sonnet-20240620': 100000, 'anthropic/claude-3-5-sonnet-20241022': 100000, 'anthropic/claude-3-7-sonnet-20250219': 200000, + 'claude-3-7-sonnet-20250219': 200000, 'anthropic/claude-3-5-haiku-20241022': 100000, 'bedrock/anthropic.claude-instant-v1': 100000, 'bedrock/anthropic.claude-v2': 100000, @@ -110,3 +111,8 @@ SUPPORT_REASONING_EFFORT_MODELS = [ "o3-mini", "o3-mini-2025-01-31" ] + +CLAUDE_EXTENDED_THINKING_MODELS = [ + "anthropic/claude-3-7-sonnet-20250219", + "claude-3-7-sonnet-20250219" +] From 7623e1a4194ee749f99f8689c84bb8f5349c59c7 Mon Sep 17 00:00:00 2001 From: Kenny Dizi Date: Mon, 3 Mar 2025 18:23:45 +0700 Subject: [PATCH 03/11] Removed trailing spaces --- pr_agent/algo/ai_handlers/litellm_ai_handler.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index e2894f1b..d70f00a6 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -6,7 +6,7 @@ import requests from litellm import acompletion from tenacity import retry, retry_if_exception_type, stop_after_attempt -from pr_agent.algo import NO_SUPPORT_TEMPERATURE_MODELS, SUPPORT_REASONING_EFFORT_MODELS, USER_MESSAGE_ONLY_MODELS +from pr_agent.algo import CLAUDE_EXTENDED_THINKING_MODELS, NO_SUPPORT_TEMPERATURE_MODELS, SUPPORT_REASONING_EFFORT_MODELS, USER_MESSAGE_ONLY_MODELS from pr_agent.algo.ai_handlers.base_ai_handler import BaseAiHandler from pr_agent.algo.utils import ReasoningEffort, get_version from pr_agent.config_loader import get_settings @@ -105,6 +105,9 @@ class LiteLLMAIHandler(BaseAiHandler): # Models that support reasoning effort self.support_reasoning_models = SUPPORT_REASONING_EFFORT_MODELS + # Models that support extended thinking + self.claude_extended_thinking_models = CLAUDE_EXTENDED_THINKING_MODELS + def prepare_logs(self, response, system, user, resp, finish_reason): response_log = response.dict().copy() response_log['system'] = system @@ -264,13 +267,13 @@ class LiteLLMAIHandler(BaseAiHandler): except json.JSONDecodeError as e: raise ValueError(f"LITELLM.EXTRA_HEADERS contains invalid JSON: {str(e)}") kwargs["extra_headers"] = litellm_extra_headers - + get_logger().debug("Prompts", artifact={"system": system, "user": user}) - + if get_settings().config.verbosity_level >= 2: get_logger().info(f"\nSystem prompt:\n{system}") get_logger().info(f"\nUser prompt:\n{user}") - + response = await acompletion(**kwargs) except (openai.APIError, openai.APITimeoutError) as e: get_logger().warning(f"Error during LLM inference: {e}") From 215c10cc8c779c9fa5db318270eee1b98b273608 Mon Sep 17 00:00:00 2001 From: Kenny Dizi Date: Mon, 3 Mar 2025 18:29:33 +0700 Subject: [PATCH 04/11] Add thinking block to request parameters --- pr_agent/algo/ai_handlers/litellm_ai_handler.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index d70f00a6..1dc3f3a5 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -245,6 +245,17 @@ class LiteLLMAIHandler(BaseAiHandler): get_logger().info(f"Adding reasoning_effort with value {reasoning_effort} to model {model}.") kwargs["reasoning_effort"] = reasoning_effort + # https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking + if (model in self.claude_extended_thinking_models) and get_settings().config.get("enable_claude_extended_thinking", False): + extended_thinking_budget_tokens = get_settings().config.get("extended_thinking_budget_tokens", 32000) + extended_thinking_max_output_tokens = get_settings().config.get("extended_thinking_max_output_tokens", 64000) + kwargs["thinking"] = { + "type": "enabled", + "budget_tokens": extended_thinking_budget_tokens + } + get_logger().info(f"Adding max output tokens {extended_thinking_max_output_tokens} to model {model}, extended thinking budget tokens: {extended_thinking_budget_tokens}") + kwargs["max_tokens"] = extended_thinking_max_output_tokens + if get_settings().litellm.get("enable_callbacks", False): kwargs = self.add_litellm_callbacks(kwargs) From 440d2368a4ef00f87e3f1533caa365088be51056 Mon Sep 17 00:00:00 2001 From: Kenny Dizi Date: Mon, 3 Mar 2025 18:30:52 +0700 Subject: [PATCH 05/11] Set temperature to 1 when using extended thinking --- pr_agent/algo/ai_handlers/litellm_ai_handler.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index 1dc3f3a5..55e4e572 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -256,6 +256,11 @@ class LiteLLMAIHandler(BaseAiHandler): get_logger().info(f"Adding max output tokens {extended_thinking_max_output_tokens} to model {model}, extended thinking budget tokens: {extended_thinking_budget_tokens}") kwargs["max_tokens"] = extended_thinking_max_output_tokens + # temperature may only be set to 1 when thinking is enabled + if get_settings().config.verbosity_level >= 2: + get_logger().info("Temperature may only be set to 1 when thinking is enabled with claude models.") + kwargs["temperature"] = 1 + if get_settings().litellm.get("enable_callbacks", False): kwargs = self.add_litellm_callbacks(kwargs) From b5ce49cbc0ef8e28dd4a516641502dad1206fa0a Mon Sep 17 00:00:00 2001 From: Kenny Dizi Date: Mon, 3 Mar 2025 18:31:17 +0700 Subject: [PATCH 06/11] Update document for dedicated claude models configuration --- docs/docs/usage-guide/changing_a_model.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docs/docs/usage-guide/changing_a_model.md b/docs/docs/usage-guide/changing_a_model.md index fd491012..c6a4701a 100644 --- a/docs/docs/usage-guide/changing_a_model.md +++ b/docs/docs/usage-guide/changing_a_model.md @@ -221,3 +221,10 @@ To bypass chat templates and temperature controls, set `config.custom_reasoning_ reasoning_efffort= = "medium" # "low", "medium", "high" With the OpenAI models that support reasoning effort (eg: o3-mini), you can specify its reasoning effort via `config` section. The default value is `medium`. You can change it to `high` or `low` based on your usage. + +### Anthropic models + +[config] +enable_claude_extended_thinking = false # Set to true to enable extended thinking feature +extended_thinking_budget_tokens = 32000 +extended_thinking_max_output_tokens = 64000 From 30bf7572b0806cbf0f73b8735ca4500873f83269 Mon Sep 17 00:00:00 2001 From: Kenny Dizi Date: Mon, 3 Mar 2025 18:44:26 +0700 Subject: [PATCH 07/11] Validate extended thinking parameters --- pr_agent/algo/ai_handlers/litellm_ai_handler.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index 55e4e572..640f9a27 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -249,6 +249,15 @@ class LiteLLMAIHandler(BaseAiHandler): if (model in self.claude_extended_thinking_models) and get_settings().config.get("enable_claude_extended_thinking", False): extended_thinking_budget_tokens = get_settings().config.get("extended_thinking_budget_tokens", 32000) extended_thinking_max_output_tokens = get_settings().config.get("extended_thinking_max_output_tokens", 64000) + + # Validate extended thinking parameters + if not isinstance(extended_thinking_budget_tokens, int) or extended_thinking_budget_tokens <= 0: + raise ValueError(f"extended_thinking_budget_tokens must be a positive integer, got {extended_thinking_budget_tokens}") + if not isinstance(extended_thinking_max_output_tokens, int) or extended_thinking_max_output_tokens <= 0: + raise ValueError(f"extended_thinking_max_output_tokens must be a positive integer, got {extended_thinking_max_output_tokens}") + if extended_thinking_max_output_tokens < extended_thinking_budget_tokens: + raise ValueError(f"extended_thinking_max_output_tokens ({extended_thinking_max_output_tokens}) must be greater than or equal to extended_thinking_budget_tokens ({extended_thinking_budget_tokens})") + kwargs["thinking"] = { "type": "enabled", "budget_tokens": extended_thinking_budget_tokens From a8935dece3c845da79d90a94e6527f2e44fd3728 Mon Sep 17 00:00:00 2001 From: Kenny Dizi Date: Fri, 7 Mar 2025 17:27:56 +0700 Subject: [PATCH 08/11] Using 2048 for extended_thinking_budget_tokens as well as extended_thinking_max_output_tokens --- pr_agent/algo/ai_handlers/litellm_ai_handler.py | 4 ++-- pr_agent/settings/configuration.toml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index 640f9a27..fb94a8b8 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -247,8 +247,8 @@ class LiteLLMAIHandler(BaseAiHandler): # https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking if (model in self.claude_extended_thinking_models) and get_settings().config.get("enable_claude_extended_thinking", False): - extended_thinking_budget_tokens = get_settings().config.get("extended_thinking_budget_tokens", 32000) - extended_thinking_max_output_tokens = get_settings().config.get("extended_thinking_max_output_tokens", 64000) + extended_thinking_budget_tokens = get_settings().config.get("extended_thinking_budget_tokens", 2048) + extended_thinking_max_output_tokens = get_settings().config.get("extended_thinking_max_output_tokens", 2048) # Validate extended thinking parameters if not isinstance(extended_thinking_budget_tokens, int) or extended_thinking_budget_tokens <= 0: diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index b04cb424..ffd924c5 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -58,10 +58,10 @@ reasoning_effort = "medium" # "low", "medium", "high" enable_auto_approval=false # Set to true to enable auto-approval of PRs under certain conditions auto_approve_for_low_review_effort=-1 # -1 to disable, [1-5] to set the threshold for auto-approval auto_approve_for_no_suggestions=false # If true, the PR will be auto-approved if there are no suggestions -# extended thinking +# extended thinking for Claude reasoning models enable_claude_extended_thinking = false # Set to true to enable extended thinking feature -extended_thinking_budget_tokens = 32000 -extended_thinking_max_output_tokens = 64000 +extended_thinking_budget_tokens = 2048 +extended_thinking_max_output_tokens = 2048 [pr_reviewer] # /review # From 121d90a9dab6e07da87165807bde8c1f2f67f10c Mon Sep 17 00:00:00 2001 From: Kenny Dizi Date: Fri, 7 Mar 2025 17:32:03 +0700 Subject: [PATCH 09/11] Update document for extended thinking tokens --- docs/docs/usage-guide/changing_a_model.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/docs/usage-guide/changing_a_model.md b/docs/docs/usage-guide/changing_a_model.md index c6a4701a..134a22ab 100644 --- a/docs/docs/usage-guide/changing_a_model.md +++ b/docs/docs/usage-guide/changing_a_model.md @@ -226,5 +226,5 @@ With the OpenAI models that support reasoning effort (eg: o3-mini), you can spec [config] enable_claude_extended_thinking = false # Set to true to enable extended thinking feature -extended_thinking_budget_tokens = 32000 -extended_thinking_max_output_tokens = 64000 +extended_thinking_budget_tokens = 2048 +extended_thinking_max_output_tokens = 2048 From f9d5e72058056c1cbe62fd7292a1265ff6d0c6ca Mon Sep 17 00:00:00 2001 From: Kenny Dizi Date: Sat, 8 Mar 2025 08:35:34 +0700 Subject: [PATCH 10/11] Move logic to _configure_claude_extended_thinking --- .../algo/ai_handlers/litellm_ai_handler.py | 59 ++++++++++++------- 1 file changed, 37 insertions(+), 22 deletions(-) diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index fb94a8b8..cb064096 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -120,6 +120,42 @@ class LiteLLMAIHandler(BaseAiHandler): response_log['main_pr_language'] = 'unknown' return response_log + def _configure_claude_extended_thinking(self, model: str, kwargs: dict) -> dict: + """ + Configure Claude extended thinking parameters if applicable. + + Args: + model (str): The AI model being used + kwargs (dict): The keyword arguments for the model call + + Returns: + dict: Updated kwargs with extended thinking configuration + """ + extended_thinking_budget_tokens = get_settings().config.get("extended_thinking_budget_tokens", 2048) + extended_thinking_max_output_tokens = get_settings().config.get("extended_thinking_max_output_tokens", 2048) + + # Validate extended thinking parameters + if not isinstance(extended_thinking_budget_tokens, int) or extended_thinking_budget_tokens <= 0: + raise ValueError(f"extended_thinking_budget_tokens must be a positive integer, got {extended_thinking_budget_tokens}") + if not isinstance(extended_thinking_max_output_tokens, int) or extended_thinking_max_output_tokens <= 0: + raise ValueError(f"extended_thinking_max_output_tokens must be a positive integer, got {extended_thinking_max_output_tokens}") + if extended_thinking_max_output_tokens < extended_thinking_budget_tokens: + raise ValueError(f"extended_thinking_max_output_tokens ({extended_thinking_max_output_tokens}) must be greater than or equal to extended_thinking_budget_tokens ({extended_thinking_budget_tokens})") + + kwargs["thinking"] = { + "type": "enabled", + "budget_tokens": extended_thinking_budget_tokens + } + get_logger().info(f"Adding max output tokens {extended_thinking_max_output_tokens} to model {model}, extended thinking budget tokens: {extended_thinking_budget_tokens}") + kwargs["max_tokens"] = extended_thinking_max_output_tokens + + # temperature may only be set to 1 when thinking is enabled + if get_settings().config.verbosity_level >= 2: + get_logger().info("Temperature may only be set to 1 when thinking is enabled with claude models.") + kwargs["temperature"] = 1 + + return kwargs + def add_litellm_callbacks(selfs, kwargs) -> dict: captured_extra = [] @@ -247,28 +283,7 @@ class LiteLLMAIHandler(BaseAiHandler): # https://docs.anthropic.com/en/docs/build-with-claude/extended-thinking if (model in self.claude_extended_thinking_models) and get_settings().config.get("enable_claude_extended_thinking", False): - extended_thinking_budget_tokens = get_settings().config.get("extended_thinking_budget_tokens", 2048) - extended_thinking_max_output_tokens = get_settings().config.get("extended_thinking_max_output_tokens", 2048) - - # Validate extended thinking parameters - if not isinstance(extended_thinking_budget_tokens, int) or extended_thinking_budget_tokens <= 0: - raise ValueError(f"extended_thinking_budget_tokens must be a positive integer, got {extended_thinking_budget_tokens}") - if not isinstance(extended_thinking_max_output_tokens, int) or extended_thinking_max_output_tokens <= 0: - raise ValueError(f"extended_thinking_max_output_tokens must be a positive integer, got {extended_thinking_max_output_tokens}") - if extended_thinking_max_output_tokens < extended_thinking_budget_tokens: - raise ValueError(f"extended_thinking_max_output_tokens ({extended_thinking_max_output_tokens}) must be greater than or equal to extended_thinking_budget_tokens ({extended_thinking_budget_tokens})") - - kwargs["thinking"] = { - "type": "enabled", - "budget_tokens": extended_thinking_budget_tokens - } - get_logger().info(f"Adding max output tokens {extended_thinking_max_output_tokens} to model {model}, extended thinking budget tokens: {extended_thinking_budget_tokens}") - kwargs["max_tokens"] = extended_thinking_max_output_tokens - - # temperature may only be set to 1 when thinking is enabled - if get_settings().config.verbosity_level >= 2: - get_logger().info("Temperature may only be set to 1 when thinking is enabled with claude models.") - kwargs["temperature"] = 1 + kwargs = self._configure_claude_extended_thinking(model, kwargs) if get_settings().litellm.get("enable_callbacks", False): kwargs = self.add_litellm_callbacks(kwargs) From 222155e4f2cda90e7781bb52e96d9e663b8aca49 Mon Sep 17 00:00:00 2001 From: Kenny Dizi Date: Sat, 8 Mar 2025 08:53:29 +0700 Subject: [PATCH 11/11] Optimize logging --- pr_agent/algo/ai_handlers/litellm_ai_handler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index cb064096..cae65500 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -146,7 +146,8 @@ class LiteLLMAIHandler(BaseAiHandler): "type": "enabled", "budget_tokens": extended_thinking_budget_tokens } - get_logger().info(f"Adding max output tokens {extended_thinking_max_output_tokens} to model {model}, extended thinking budget tokens: {extended_thinking_budget_tokens}") + if get_settings().config.verbosity_level >= 2: + get_logger().info(f"Adding max output tokens {extended_thinking_max_output_tokens} to model {model}, extended thinking budget tokens: {extended_thinking_budget_tokens}") kwargs["max_tokens"] = extended_thinking_max_output_tokens # temperature may only be set to 1 when thinking is enabled