diff --git a/docs/docs/tools/improve.md b/docs/docs/tools/improve.md index 9e58219e..ba9a0695 100644 --- a/docs/docs/tools/improve.md +++ b/docs/docs/tools/improve.md @@ -342,6 +342,10 @@ Note: Chunking is primarily relevant for large PRs. For most PRs (up to 500 line wiki_page_accepted_suggestions If set to true, the tool will automatically track accepted suggestions in a dedicated wiki page called `.pr_agent_accepted_suggestions`. Default is true. + + allow_thumbs_up_down + If set to true, all code suggestions will have thumbs up and thumbs down buttons, to encourage users to provide feedback on the suggestions. Default is false. + ??? example "Params for number of suggestions and AI calls" @@ -359,10 +363,6 @@ Note: Chunking is primarily relevant for large PRs. For most PRs (up to 500 line max_number_of_calls Maximum number of chunks. Default is 3. - - rank_extended_suggestions - If set to true, the tool will rank the suggestions, based on importance. Default is true. - ## A note on code suggestions quality diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index 32c6d71c..f6e38255 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -115,7 +115,6 @@ dual_publishing_score_threshold=-1 # -1 to disable, [0-10] to set the threshold focus_only_on_problems=true # extra_instructions = "" -rank_suggestions = false enable_help_text=false enable_chat_text=false enable_intro_text=true @@ -130,7 +129,7 @@ auto_extended_mode=true num_code_suggestions_per_chunk=4 max_number_of_calls = 3 parallel_calls = true -rank_extended_suggestions = false + final_clip_factor = 0.8 # self-review checkbox demand_code_suggestions_self_review=false # add a checkbox for the author to self-review the code suggestions @@ -140,6 +139,7 @@ fold_suggestions_on_self_review=true # Pro feature. if true, the code suggestion # Suggestion impact 💎 publish_post_process_suggestion_impact=true wiki_page_accepted_suggestions=true +allow_thumbs_up_down=false [pr_custom_prompt] # /custom_prompt # prompt = """\ diff --git a/pr_agent/tools/pr_code_suggestions.py b/pr_agent/tools/pr_code_suggestions.py index 9f34718a..447a15cf 100644 --- a/pr_agent/tools/pr_code_suggestions.py +++ b/pr_agent/tools/pr_code_suggestions.py @@ -21,7 +21,7 @@ from pr_agent.config_loader import get_settings from pr_agent.git_providers import (AzureDevopsProvider, GithubProvider, GitLabProvider, get_git_provider, get_git_provider_with_context) -from pr_agent.git_providers.git_provider import get_main_pr_language +from pr_agent.git_providers.git_provider import get_main_pr_language, GitProvider from pr_agent.log import get_logger from pr_agent.servers.help import HelpMessage from pr_agent.tools.pr_description import insert_br_after_x_chars @@ -103,6 +103,8 @@ class PRCodeSuggestions: relevant_configs = {'pr_code_suggestions': dict(get_settings().pr_code_suggestions), 'config': dict(get_settings().config)} get_logger().debug("Relevant configs", artifacts=relevant_configs) + + # publish "Preparing suggestions..." comments if (get_settings().config.publish_output and get_settings().config.publish_output_progress and not get_settings().config.get('is_auto_command', False)): if self.git_provider.is_supported("gfm_markdown"): @@ -110,33 +112,26 @@ class PRCodeSuggestions: else: self.git_provider.publish_comment("Preparing suggestions...", is_temporary=True) + # call the model to get the suggestions, and self-reflect on them if not self.is_extended: data = await retry_with_fallback_models(self._prepare_prediction) else: data = await retry_with_fallback_models(self._prepare_prediction_extended) if not data: data = {"code_suggestions": []} + self.data = data + # Handle the case where the PR has no suggestions if (data is None or 'code_suggestions' not in data or not data['code_suggestions']): - pr_body = "## PR Code Suggestions ✨\n\nNo code suggestions found for the PR." - get_logger().warning('No code suggestions found for the PR.') - if get_settings().config.publish_output and get_settings().config.publish_output_no_suggestions: - get_logger().debug(f"PR output", artifact=pr_body) - if self.progress_response: - self.git_provider.edit_comment(self.progress_response, body=pr_body) - else: - self.git_provider.publish_comment(pr_body) - else: - get_settings().data = {"artifact": ""} + await self.publish_no_suggestions() return - if (not self.is_extended and get_settings().pr_code_suggestions.rank_suggestions) or \ - (self.is_extended and get_settings().pr_code_suggestions.rank_extended_suggestions): - get_logger().info('Ranking Suggestions...') - data['code_suggestions'] = await self.rank_suggestions(data['code_suggestions']) - + # publish the suggestions if get_settings().config.publish_output: + # If a temporary comment was published, remove it self.git_provider.remove_initial_comment() + + # Publish table summarized suggestions if ((not get_settings().pr_code_suggestions.commitable_code_suggestions) and self.git_provider.is_supported("gfm_markdown")): @@ -146,10 +141,7 @@ class PRCodeSuggestions: # require self-review if get_settings().pr_code_suggestions.demand_code_suggestions_self_review: - text = get_settings().pr_code_suggestions.code_suggestions_self_review_text - pr_body += f"\n\n- [ ] {text}" - if get_settings().pr_code_suggestions.approve_pr_on_self_review: - pr_body += ' ' + pr_body = await self.add_self_review_text(pr_body) # add usage guide if (get_settings().pr_code_suggestions.enable_chat_text and get_settings().config.is_auto_command @@ -165,13 +157,13 @@ class PRCodeSuggestions: pr_body += show_relevant_configurations(relevant_section='pr_code_suggestions') # publish the PR comment - if get_settings().pr_code_suggestions.persistent_comment: - final_update_message = False - self.publish_persistent_comment_with_history(pr_body, + if get_settings().pr_code_suggestions.persistent_comment: # true by default + self.publish_persistent_comment_with_history(self.git_provider, + pr_body, initial_header="## PR Code Suggestions ✨", update_header=True, name="suggestions", - final_update_message=final_update_message, + final_update_message=False, max_previous_comments=get_settings().pr_code_suggestions.max_history_len, progress_response=self.progress_response) else: @@ -182,24 +174,9 @@ class PRCodeSuggestions: # dual publishing mode if int(get_settings().pr_code_suggestions.dual_publishing_score_threshold) > 0: - data_above_threshold = {'code_suggestions': []} - try: - for suggestion in data['code_suggestions']: - if int(suggestion.get('score', 0)) >= int(get_settings().pr_code_suggestions.dual_publishing_score_threshold) \ - and suggestion.get('improved_code'): - data_above_threshold['code_suggestions'].append(suggestion) - if not data_above_threshold['code_suggestions'][-1]['existing_code']: - get_logger().info(f'Identical existing and improved code for dual publishing found') - data_above_threshold['code_suggestions'][-1]['existing_code'] = suggestion[ - 'improved_code'] - if data_above_threshold['code_suggestions']: - get_logger().info( - f"Publishing {len(data_above_threshold['code_suggestions'])} suggestions in dual publishing mode") - self.push_inline_code_suggestions(data_above_threshold) - except Exception as e: - get_logger().error(f"Failed to publish dual publishing suggestions, error: {e}") + await self.dual_publishing(data) else: - self.push_inline_code_suggestions(data) + await self.push_inline_code_suggestions(data) if self.progress_response: self.git_provider.remove_comment(self.progress_response) else: @@ -218,47 +195,108 @@ class PRCodeSuggestions: self.git_provider.remove_initial_comment() self.git_provider.publish_comment(f"Failed to generate code suggestions for PR") except Exception as e: - pass + get_logger().exception(f"Failed to update persistent review, error: {e}") - def publish_persistent_comment_with_history(self, pr_comment: str, + async def add_self_review_text(self, pr_body): + text = get_settings().pr_code_suggestions.code_suggestions_self_review_text + pr_body += f"\n\n- [ ] {text}" + approve_pr_on_self_review = get_settings().pr_code_suggestions.approve_pr_on_self_review + fold_suggestions_on_self_review = get_settings().pr_code_suggestions.fold_suggestions_on_self_review + if approve_pr_on_self_review and not fold_suggestions_on_self_review: + pr_body += ' ' + elif fold_suggestions_on_self_review and not approve_pr_on_self_review: + pr_body += ' ' + else: + pr_body += ' ' + return pr_body + + async def publish_no_suggestions(self): + pr_body = "## PR Code Suggestions ✨\n\nNo code suggestions found for the PR." + if get_settings().config.publish_output: + get_logger().warning('No code suggestions found for the PR.') + get_logger().debug(f"PR output", artifact=pr_body) + if self.progress_response: + self.git_provider.edit_comment(self.progress_response, body=pr_body) + else: + self.git_provider.publish_comment(pr_body) + else: + get_settings().data = {"artifact": ""} + + async def dual_publishing(self, data): + data_above_threshold = {'code_suggestions': []} + try: + for suggestion in data['code_suggestions']: + if int(suggestion.get('score', 0)) >= int( + get_settings().pr_code_suggestions.dual_publishing_score_threshold) \ + and suggestion.get('improved_code'): + data_above_threshold['code_suggestions'].append(suggestion) + if not data_above_threshold['code_suggestions'][-1]['existing_code']: + get_logger().info(f'Identical existing and improved code for dual publishing found') + data_above_threshold['code_suggestions'][-1]['existing_code'] = suggestion[ + 'improved_code'] + if data_above_threshold['code_suggestions']: + get_logger().info( + f"Publishing {len(data_above_threshold['code_suggestions'])} suggestions in dual publishing mode") + await self.push_inline_code_suggestions(data_above_threshold) + except Exception as e: + get_logger().error(f"Failed to publish dual publishing suggestions, error: {e}") + + @staticmethod + def publish_persistent_comment_with_history(git_provider: GitProvider, + pr_comment: str, initial_header: str, update_header: bool = True, name='review', final_update_message=True, max_previous_comments=4, - progress_response=None): + progress_response=None, + only_fold=False): - if isinstance(self.git_provider, AzureDevopsProvider): # get_latest_commit_url is not supported yet + def _extract_link(comment_text: str): + r = re.compile(r"") + match = r.search(comment_text) + + up_to_commit_txt = "" + if match: + up_to_commit_txt = f" up to commit {match.group(0)[4:-3].strip()}" + return up_to_commit_txt + + if isinstance(git_provider, AzureDevopsProvider): # get_latest_commit_url is not supported yet if progress_response: - self.git_provider.edit_comment(progress_response, pr_comment) + git_provider.edit_comment(progress_response, pr_comment) + new_comment = progress_response else: - self.git_provider.publish_comment(pr_comment) - return + new_comment = git_provider.publish_comment(pr_comment) + return new_comment history_header = f"#### Previous suggestions\n" - last_commit_num = self.git_provider.get_latest_commit_url().split('/')[-1][:7] - latest_suggestion_header = f"Latest suggestions up to {last_commit_num}" + last_commit_num = git_provider.get_latest_commit_url().split('/')[-1][:7] + if only_fold: # A user clicked on the 'self-review' checkbox + text = get_settings().pr_code_suggestions.code_suggestions_self_review_text + latest_suggestion_header = f"\n\n- [x] {text}" + else: + latest_suggestion_header = f"Latest suggestions up to {last_commit_num}" latest_commit_html_comment = f"" found_comment = None if max_previous_comments > 0: try: - prev_comments = list(self.git_provider.get_issue_comments()) + prev_comments = list(git_provider.get_issue_comments()) for comment in prev_comments: if comment.body.startswith(initial_header): prev_suggestions = comment.body found_comment = comment - comment_url = self.git_provider.get_comment_url(comment) + comment_url = git_provider.get_comment_url(comment) if history_header.strip() not in comment.body: # no history section # extract everything between and
in comment.body including and
table_index = comment.body.find("") if table_index == -1: - self.git_provider.edit_comment(comment, pr_comment) + git_provider.edit_comment(comment, pr_comment) continue # find http link from comment.body[:table_index] - up_to_commit_txt = self.extract_link(comment.body[:table_index]) + up_to_commit_txt = _extract_link(comment.body[:table_index]) prev_suggestion_table = comment.body[ table_index:comment.body.rfind("
") + len("")] @@ -279,7 +317,7 @@ class PRCodeSuggestions: # get text after the latest_suggestion_header in comment.body table_ind = latest_table.find("") - up_to_commit_txt = self.extract_link(latest_table[:table_ind]) + up_to_commit_txt = _extract_link(latest_table[:table_ind]) latest_table = latest_table[table_ind:latest_table.rfind("
") + len("")] # enforce max_previous_comments @@ -306,11 +344,12 @@ class PRCodeSuggestions: get_logger().info(f"Persistent mode - updating comment {comment_url} to latest {name} message") if progress_response: # publish to 'progress_response' comment, because it refreshes immediately - self.git_provider.edit_comment(progress_response, pr_comment_updated) - self.git_provider.remove_comment(comment) + git_provider.edit_comment(progress_response, pr_comment_updated) + git_provider.remove_comment(comment) + comment = progress_response else: - self.git_provider.edit_comment(comment, pr_comment_updated) - return + git_provider.edit_comment(comment, pr_comment_updated) + return comment except Exception as e: get_logger().exception(f"Failed to update persistent review, error: {e}") pass @@ -319,9 +358,12 @@ class PRCodeSuggestions: body = pr_comment.replace(initial_header, "").strip() pr_comment = f"{initial_header}\n\n{latest_commit_html_comment}\n\n{body}\n\n" if progress_response: - self.git_provider.edit_comment(progress_response, pr_comment) + git_provider.edit_comment(progress_response, pr_comment) + new_comment = progress_response else: - self.git_provider.publish_comment(pr_comment) + new_comment = git_provider.publish_comment(pr_comment) + return new_comment + def extract_link(self, s): r = re.compile(r"") @@ -372,50 +414,7 @@ class PRCodeSuggestions: response_reflect = await self.self_reflect_on_suggestions(data["code_suggestions"], patches_diff, model=model_reflection) if response_reflect: - response_reflect_yaml = load_yaml(response_reflect) - code_suggestions_feedback = response_reflect_yaml["code_suggestions"] - if len(code_suggestions_feedback) == len(data["code_suggestions"]): - for i, suggestion in enumerate(data["code_suggestions"]): - try: - suggestion["score"] = code_suggestions_feedback[i]["suggestion_score"] - suggestion["score_why"] = code_suggestions_feedback[i]["why"] - - if 'relevant_lines_start' not in suggestion: - relevant_lines_start = code_suggestions_feedback[i].get('relevant_lines_start', -1) - relevant_lines_end = code_suggestions_feedback[i].get('relevant_lines_end', -1) - suggestion['relevant_lines_start'] = relevant_lines_start - suggestion['relevant_lines_end'] = relevant_lines_end - if relevant_lines_start < 0 or relevant_lines_end < 0: - suggestion["score"] = 0 - - try: - if get_settings().config.publish_output: - suggestion_statistics_dict = {'score': int(suggestion["score"]), - 'label': suggestion["label"].lower().strip()} - get_logger().info(f"PR-Agent suggestions statistics", - statistics=suggestion_statistics_dict, analytics=True) - except Exception as e: - get_logger().error(f"Failed to log suggestion statistics, error: {e}") - pass - - except Exception as e: # - get_logger().error(f"Error processing suggestion score {i}", - artifact={"suggestion": suggestion, - "code_suggestions_feedback": code_suggestions_feedback[i]}) - suggestion["score"] = 7 - suggestion["score_why"] = "" - - # if the before and after code is the same, clear one of them - try: - if suggestion['existing_code'] == suggestion['improved_code']: - get_logger().debug( - f"edited improved suggestion {i + 1}, because equal to existing code: {suggestion['existing_code']}") - if get_settings().pr_code_suggestions.commitable_code_suggestions: - suggestion['improved_code'] = "" # we need 'existing_code' to locate the code in the PR - else: - suggestion['existing_code'] = "" - except Exception as e: - get_logger().error(f"Error processing suggestion {i + 1}, error: {e}") + await self.analyze_self_reflection_response(data, response_reflect) else: # get_logger().error(f"Could not self-reflect on suggestions. using default score 7") for i, suggestion in enumerate(data["code_suggestions"]): @@ -424,6 +423,58 @@ class PRCodeSuggestions: return data + async def analyze_self_reflection_response(self, data, response_reflect): + response_reflect_yaml = load_yaml(response_reflect) + code_suggestions_feedback = response_reflect_yaml.get("code_suggestions", []) + if code_suggestions_feedback and len(code_suggestions_feedback) == len(data["code_suggestions"]): + for i, suggestion in enumerate(data["code_suggestions"]): + try: + suggestion["score"] = code_suggestions_feedback[i]["suggestion_score"] + suggestion["score_why"] = code_suggestions_feedback[i]["why"] + + if 'relevant_lines_start' not in suggestion: + relevant_lines_start = code_suggestions_feedback[i].get('relevant_lines_start', -1) + relevant_lines_end = code_suggestions_feedback[i].get('relevant_lines_end', -1) + suggestion['relevant_lines_start'] = relevant_lines_start + suggestion['relevant_lines_end'] = relevant_lines_end + if relevant_lines_start < 0 or relevant_lines_end < 0: + suggestion["score"] = 0 + + try: + if get_settings().config.publish_output: + if not suggestion["score"]: + score = -1 + else: + score = int(suggestion["score"]) + label = suggestion["label"].lower().strip() + label = label.replace('
', ' ') + suggestion_statistics_dict = {'score': score, + 'label': label} + get_logger().info(f"PR-Agent suggestions statistics", + statistics=suggestion_statistics_dict, analytics=True) + except Exception as e: + get_logger().error(f"Failed to log suggestion statistics, error: {e}") + pass + + except Exception as e: # + get_logger().error(f"Error processing suggestion score {i}", + artifact={"suggestion": suggestion, + "code_suggestions_feedback": code_suggestions_feedback[i]}) + suggestion["score"] = 7 + suggestion["score_why"] = "" + + # if the before and after code is the same, clear one of them + try: + if suggestion['existing_code'] == suggestion['improved_code']: + get_logger().debug( + f"edited improved suggestion {i + 1}, because equal to existing code: {suggestion['existing_code']}") + if get_settings().pr_code_suggestions.commitable_code_suggestions: + suggestion['improved_code'] = "" # we need 'existing_code' to locate the code in the PR + else: + suggestion['existing_code'] = "" + except Exception as e: + get_logger().error(f"Error processing suggestion {i + 1}, error: {e}") + @staticmethod def _truncate_if_needed(suggestion): max_code_suggestion_length = get_settings().get("PR_CODE_SUGGESTIONS.MAX_CODE_SUGGESTION_LENGTH", 0) @@ -487,7 +538,7 @@ class PRCodeSuggestions: return data - def push_inline_code_suggestions(self, data): + async def push_inline_code_suggestions(self, data): code_suggestions = [] if not data['code_suggestions']: @@ -585,7 +636,9 @@ class PRCodeSuggestions: patches_diff_lines = patches_diff.splitlines() for i, line in enumerate(patches_diff_lines): if line.strip(): - if line[0].isdigit(): + if line.isnumeric(): + patches_diff_lines[i] = '' + elif line[0].isdigit(): # find the first letter in the line that starts with a valid letter for j, char in enumerate(line): if not char.isdigit(): @@ -643,62 +696,6 @@ class PRCodeSuggestions: self.data = data = None return data - async def rank_suggestions(self, data: List) -> List: - """ - Call a model to rank (sort) code suggestions based on their importance order. - - Args: - data (List): A list of code suggestions to be ranked. - - Returns: - List: The ranked list of code suggestions. - """ - - suggestion_list = [] - if not data: - return suggestion_list - for suggestion in data: - suggestion_list.append(suggestion) - data_sorted = [[]] * len(suggestion_list) - - if len(suggestion_list) == 1: - return suggestion_list - - try: - suggestion_str = "" - for i, suggestion in enumerate(suggestion_list): - suggestion_str += f"suggestion {i + 1}: " + str(suggestion) + '\n\n' - - variables = {'suggestion_list': suggestion_list, 'suggestion_str': suggestion_str} - model = get_settings().config.model - environment = Environment(undefined=StrictUndefined) - system_prompt = environment.from_string(get_settings().pr_sort_code_suggestions_prompt.system).render( - variables) - user_prompt = environment.from_string(get_settings().pr_sort_code_suggestions_prompt.user).render(variables) - response, finish_reason = await self.ai_handler.chat_completion(model=model, system=system_prompt, - user=user_prompt) - - sort_order = load_yaml(response) - for s in sort_order['Sort Order']: - suggestion_number = s['suggestion number'] - importance_order = s['importance order'] - data_sorted[importance_order - 1] = suggestion_list[suggestion_number - 1] - - if get_settings().pr_code_suggestions.final_clip_factor != 1: - max_len = max( - len(data_sorted), - int(get_settings().pr_code_suggestions.num_code_suggestions_per_chunk), - ) - new_len = int(0.5 + max_len * get_settings().pr_code_suggestions.final_clip_factor) - if new_len < len(data_sorted): - data_sorted = data_sorted[:new_len] - except Exception as e: - if get_settings().config.verbosity_level >= 1: - get_logger().info(f"Could not sort suggestions, error: {e}") - data_sorted = suggestion_list - - return data_sorted - def generate_summarized_suggestions(self, data: Dict) -> str: try: pr_body = "## PR Code Suggestions ✨\n\n" @@ -814,7 +811,12 @@ class PRCodeSuggestions: get_logger().info(f"Failed to publish summarized code suggestions, error: {e}") return "" - async def self_reflect_on_suggestions(self, suggestion_list: List, patches_diff: str, model: str) -> str: + async def self_reflect_on_suggestions(self, + suggestion_list: List, + patches_diff: str, + model: str, + prev_suggestions_str: str = "", + dedicated_prompt: str = "") -> str: if not suggestion_list: return "" @@ -827,13 +829,21 @@ class PRCodeSuggestions: 'suggestion_str': suggestion_str, "diff": patches_diff, 'num_code_suggestions': len(suggestion_list), + 'prev_suggestions_str': prev_suggestions_str, "is_ai_metadata": get_settings().get("config.enable_ai_metadata", False)} environment = Environment(undefined=StrictUndefined) - system_prompt_reflect = environment.from_string( - get_settings().pr_code_suggestions_reflect_prompt.system).render( - variables) - user_prompt_reflect = environment.from_string( - get_settings().pr_code_suggestions_reflect_prompt.user).render(variables) + + if dedicated_prompt: + system_prompt_reflect = environment.from_string( + get_settings().get(dedicated_prompt).system).render(variables) + user_prompt_reflect = environment.from_string( + get_settings().get(dedicated_prompt).user).render(variables) + else: + system_prompt_reflect = environment.from_string( + get_settings().pr_code_suggestions_reflect_prompt.system).render(variables) + user_prompt_reflect = environment.from_string( + get_settings().pr_code_suggestions_reflect_prompt.user).render(variables) + with get_logger().contextualize(command="self_reflect_on_suggestions"): response_reflect, finish_reason_reflect = await self.ai_handler.chat_completion(model=model, system=system_prompt_reflect, @@ -841,4 +851,4 @@ class PRCodeSuggestions: except Exception as e: get_logger().info(f"Could not reflect on suggestions, error: {e}") return "" - return response_reflect + return response_reflect \ No newline at end of file