diff --git a/README.md b/README.md index 3dca2fa2..03a076fa 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ CodiumAI PR-Agent aims to help efficiently review and handle pull requests, by p ### May 12, 2024 Inspired by [AlphaCodium](https://github.com/Codium-ai/AlphaCodium) flow engineering scheme, PR-Agent now performs **self-reflection** on the code suggestions it provides, enabling to remove invalid suggestions, and score the valid ones. The suggestions will be presented sorted by their score, enabling to focus on the most important ones first. +You can also choose to remove suggestions below a certain importance score threshold, by setting the `pr_code_suggestions.suggestions_score_threshold` [configuration](https://pr-agent-docs.codium.ai/tools/improve/#configuration-options). diff --git a/docs/docs/tools/improve.md b/docs/docs/tools/improve.md index e8b4dd6f..7906f426 100644 --- a/docs/docs/tools/improve.md +++ b/docs/docs/tools/improve.md @@ -90,6 +90,10 @@ Hence, the total number of suggestions is proportional to the number of chunks, self_reflect_on_suggestions If set to true, the improve tool will calculate an importance score for each suggestion [1-10], and sort the suggestion labels group based on this score. Default is true. + + suggestions_score_threshold + Any suggestion with importance score less than this threshold will be removed. Default is 0. Highly recommend not to set this value above 7-8, since above it may clip relevant suggestions that can be useful. + enable_help_text If set to true, the tool will display a help text in the comment. Default is true. diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index c1d0dded..b84bce61 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -85,7 +85,9 @@ extra_instructions = "" rank_suggestions = false enable_help_text=false persistent_comment=false +# suggestions scoring self_reflect_on_suggestions=true +suggestions_score_threshold=0 # [0-10]. highly recommend not to set this value above 8, since above it may clip highly relevant suggestions # params for '/improve --extended' mode auto_extended_mode=true num_code_suggestions_per_chunk=4 diff --git a/pr_agent/tools/pr_code_suggestions.py b/pr_agent/tools/pr_code_suggestions.py index a192e80d..8ea9b595 100644 --- a/pr_agent/tools/pr_code_suggestions.py +++ b/pr_agent/tools/pr_code_suggestions.py @@ -344,14 +344,15 @@ class PRCodeSuggestions: data = {"code_suggestions": []} for i, predictions in enumerate(prediction_list): if "code_suggestions" in predictions: + score_threshold = max(1,get_settings().pr_code_suggestions.suggestions_score_threshold) for prediction in predictions["code_suggestions"]: try: if get_settings().pr_code_suggestions.self_reflect_on_suggestions: score = int(prediction["score"]) - if score > 0: + if score >= score_threshold: data["code_suggestions"].append(prediction) else: - get_logger().info(f"Removing suggestions {i}, because score is {score}", + get_logger().info(f"Removing suggestions {i}, because score is {score}, and score_threshold is {score_threshold}", artifact=prediction) else: get_logger().error(f"Error getting PR diff, no code suggestions found in call {i + 1}") @@ -517,11 +518,12 @@ class PRCodeSuggestions: pr_body += f"Why: {suggestion['score_why']}\n\n" pr_body += f"" + pr_body += f"" + # # add another column for 'score' if get_settings().pr_code_suggestions.self_reflect_on_suggestions: pr_body += f"{suggestion['score']}\n\n" - pr_body += f"" pr_body += f""