Merge pull request #512 from Codium-ai/trlabeling_files_extended

Refactoring and Enhancement of PR Description Formatting in 'pr_description.py'

INSTALL.md

@@ -206,6 +206,7 @@ Allowing you to automate the review process on your private or public repositori
    - Set the following events:
      - Issue comment
      - Pull request
+     - Push (if you need to enable triggering on PR update)
 
 2. Generate a random secret for your app, and save it for later. For example, you can use:
 

RELEASE_NOTES.md

@@ -1,3 +1,21 @@
+## [Version 0.11] - 2023-12-07
+- codiumai/pr-agent:0.11
+- codiumai/pr-agent:0.11-github_app
+- codiumai/pr-agent:0.11-bitbucket-app
+- codiumai/pr-agent:0.11-gitlab_webhook
+- codiumai/pr-agent:0.11-github_polling
+- codiumai/pr-agent:0.11-github_action
+
+### Added::Algo
+- New section in `/describe` tool - [PR changes walkthrough](https://github.com/Codium-ai/pr-agent/pull/509)
+- Improving PR Agent [prompts](https://github.com/Codium-ai/pr-agent/pull/501)
+- Persistent tools (`/review`, `/describe`) now send an [update message](https://github.com/Codium-ai/pr-agent/pull/499) after finishing
+- Add Amazon Bedrock [support](https://github.com/Codium-ai/pr-agent/pull/483)
+
+### Fixed
+- Update [dependencies](https://github.com/Codium-ai/pr-agent/pull/503) in requirements.txt for Python 3.12
+
+
 ## [Version 0.10] - 2023-11-15
 - codiumai/pr-agent:0.10
 - codiumai/pr-agent:0.10-github_app

docs/DESCRIBE.md

@@ -1,6 +1,6 @@
 # Describe Tool
 
-The `describe` tool scans the PR code changes, and automatically generates PR description - title, type, summary, code walkthrough and labels.
+The `describe` tool scans the PR code changes, and automatically generates PR description - title, type, summary, walkthrough and labels.
 It can be invoked manually by commenting on any PR:
 ```
 /describe
@@ -26,10 +26,15 @@ Under the section 'pr_description', the [configuration file](./../pr_agent/setti
 - `keep_original_user_title`: if set to true, the tool will keep the original PR title, and won't change it. Default is false.
 
 - `extra_instructions`: Optional extra instructions to the tool. For example: "focus on the changes in the file X. Ignore change in ...".
+
 - To enable `custom labels`, apply the configuration changes described [here](./GENERATE_CUSTOM_LABELS.md#configuration-changes)
+
 - `enable_pr_type`: if set to false, it will not show the `PR type` as a text value in the description content. Default is true.
+
 - `final_update_message`: if set to true, it will add a comment message [`PR Description updated to latest commit...`](https://github.com/Codium-ai/pr-agent/pull/499#issuecomment-1837412176) after finishing calling `/describe`. Default is true.
 
+- `enable_semantic_files_types`: if set to true, "PR changes walkthrough" section will be generated. Default is true.
+  
 ### Markers template
 
 markers enable to easily integrate user's content and auto-generated content, with a template-like mechanism.

pr_agent/git_providers/git_provider.py

@@ -26,6 +26,8 @@ class FilePatchInfo:
     tokens: int = -1
     edit_type: EDIT_TYPE = EDIT_TYPE.UNKNOWN
     old_filename: str = None
+    num_plus_lines: int = -1
+    num_minus_lines: int = -1
 
 
 class GitProvider(ABC):

pr_agent/git_providers/github_provider.py

@@ -143,8 +143,15 @@ class GithubProvider(GitProvider):
                 else:
                     get_logger().error(f"Unknown edit type: {file.status}")
                     edit_type = EDIT_TYPE.UNKNOWN
+
+                # count number of lines added and removed
+                patch_lines = patch.splitlines(keepends=True)
+                num_plus_lines = len([line for line in patch_lines if line.startswith('+')])
+                num_minus_lines = len([line for line in patch_lines if line.startswith('-')])
                 file_patch_canonical_structure = FilePatchInfo(original_file_content_str, new_file_content_str, patch,
-                                                               file.filename, edit_type=edit_type)
+                                                               file.filename, edit_type=edit_type,
+                                                               num_plus_lines=num_plus_lines,
+                                                               num_minus_lines=num_minus_lines,)
                 diff_files.append(file_patch_canonical_structure)
 
             self.diff_files = diff_files

pr_agent/git_providers/gitlab_provider.py

@@ -115,12 +115,20 @@ class GitLabProvider(GitProvider):
                 if not patch:
                     patch = load_large_diff(filename, new_file_content_str, original_file_content_str)
 
+
+                # count number of lines added and removed
+                patch_lines = patch.splitlines(keepends=True)
+                num_plus_lines = len([line for line in patch_lines if line.startswith('+')])
+                num_minus_lines = len([line for line in patch_lines if line.startswith('-')])
                 diff_files.append(
                     FilePatchInfo(original_file_content_str, new_file_content_str,
                                   patch=patch,
                                   filename=filename,
                                   edit_type=edit_type,
-                                  old_filename=None if diff['old_path'] == diff['new_path'] else diff['old_path']))
+                                  old_filename=None if diff['old_path'] == diff['new_path'] else diff['old_path'],
+                                  num_plus_lines=num_plus_lines,
+                                  num_minus_lines=num_minus_lines, ))
+
         self.diff_files = diff_files
         return diff_files
 

pr_agent/settings/pr_description_prompts.toml

@@ -33,13 +33,14 @@ class PRType(str, Enum):
 {%- if enable_file_walkthrough %}
 class FileWalkthrough(BaseModel):
     filename: str = Field(description="the relevant file full path")
-    changes_in_file: str = Field(description="minimal and concise description of the changes in the relevant file")
+    changes_in_file: str = Field(description="minimal and concise summary of the changes in the relevant file")
 {%- endif %}
 
 {%- if enable_semantic_files_types %}
-class SemanticFileLabels(BaseModel):
+Class FileDescription(BaseModel):
-    label: str = Field(description="a semantic label that represents a type of code changes that occurred in the PR. Possible values (partial list): 'bug fix', 'tests', 'enhancement', 'documentation', 'error handling', 'configuration changes', 'logging', 'dependencies', 'new feature', 'other', ...")
+    filename: str = Field(description="the relevant file full path")
-    files: List[str] = Field(description="a list of file names related to the chosen semantic label. Present the file full path, and nothing else. A file should appear only in a single label, that best describes the type of changes in the file. If you are not sure, prefer a more general label.")
+    changes_summary: str = Field(description="minimal and concise summary of the changes in the relevant file")
+    label: str = Field(description="a single semantic label that represents a type of code changes that occurred in the File. Possible values (partial list): 'bug fix', 'tests', 'enhancement', 'documentation', 'error handling', 'configuration changes', 'dependencies', 'formatting', 'miscellaneous', ...")
 {%- endif %}
 
 Class PRDescription(BaseModel):
@@ -53,7 +54,7 @@ Class PRDescription(BaseModel):
     main_files_walkthrough: List[FileWalkthrough] = Field(max_items=10)
 {%- endif %}
 {%- if enable_semantic_files_types %}
-    pr_files_labels[List[SemanticFileLabels]] = Field(min_items=2, description="A list of semantic labels that describe the type of changes in the PR files.")
+    pr_files[List[FileDescription]] = Field(max_items=15")
 {%- endif %}
 =====
 
@@ -79,12 +80,12 @@ main_files_walkthrough:
 - ...
 {%- endif %}
 {%- if enable_semantic_files_types %}
-pr_files_labels:
+pr_files:
-- label: ...
+- filename: |
-  files:
-  - |
     ...
-  - |
+  changes_summary: |
+    ...
+  label: |
     ...
 ...
 {%- endif %}

pr_agent/tools/pr_description.py

@@ -30,6 +30,11 @@ class PRDescription:
         )
         self.pr_id = self.git_provider.get_pr_id()
 
+        if get_settings().pr_description.enable_semantic_files_types and not self.git_provider.is_supported(
+                "gfm_markdown"):
+            get_logger().debug(f"Disabling semantic files types for {self.pr_id}")
+            get_settings().pr_description.enable_semantic_files_types = False
+
         # Initialize the AI handler
         self.ai_handler = AiHandler()
     
@@ -81,6 +86,9 @@ class PRDescription:
             else:
                 return None
 
+            if get_settings().pr_description.enable_semantic_files_types:
+                self._prepare_file_labels()
+
             pr_labels = []
             if get_settings().pr_description.publish_labels:
                 pr_labels = self._prepare_labels()
@@ -259,12 +267,13 @@ class PRDescription:
         # except for the items containing the word 'walkthrough'
         pr_body = ""
         for idx, (key, value) in enumerate(self.data.items()):
+            if key == 'pr_files':
+                value = self.file_label_dict
+                key_publish = "PR changes walkthrough"
+            else:
                 key_publish = key.rstrip(':').replace("_", " ").capitalize()
-            if key == 'pr_files_labels':
-                key_publish = 'PR Changes Analysis'
             pr_body += f"## {key_publish}\n"
             if 'walkthrough' in key.lower():
-                # for filename, description in value.items():
                 if self.git_provider.is_supported("gfm_markdown"):
                     pr_body += "<details> <summary>files:</summary>\n\n"
                 for file in value:
@@ -273,38 +282,8 @@ class PRDescription:
                     pr_body += f'- `{filename}`: {description}\n'
                 if self.git_provider.is_supported("gfm_markdown"):
                     pr_body += "</details>\n"
-            elif 'pr_files_labels' in key.lower():
+            elif 'pr_files' in key.lower():
-                pr_body += """\n| | Relevant Files """
+                pr_body = self.process_pr_files_prediction(pr_body, value)
-                pr_body += "&nbsp; " * 60
-                pr_body += """|\n|-----------|-------------|\n"""
-                for semantic_label in value:
-                    # for filename, description in value.items():
-                    s_label = semantic_label['label'].strip("'").strip('"')
-                    if self.git_provider.is_supported("gfm_markdown"):
-                        # pr_body += f"<details> <summary>{semantic_label['label']}</summary>\n\n"
-                        pr_body += f"| **{s_label}** | <details><summary>files:</summary><ul>"
-                    else:
-                        pr_body += f"| **{s_label}** | "
-
-                    for file in semantic_label['files']:
-                        filename = file.replace("'", "`")
-                        # description = file['changes_in_file']
-                        # pr_body += f'- `{filename}`\n'
-
-                        # try to add line numbers link to code suggestions
-                        if hasattr(self.git_provider, 'get_line_link'):
-                            filename = filename.strip()
-                            link = self.git_provider.get_line_link(filename, relevant_line_start=-1)
-                            if link:
-                                filename = f"[{filename}]({link})"
-                        if self.git_provider.is_supported("gfm_markdown"):
-                            pr_body += f"<li>{filename}</li>"
-                        else:
-                            pr_body += f"{filename}  &emsp; &emsp;"
-                    if self.git_provider.is_supported("gfm_markdown"):
-                        pr_body += "</ul></details>|\n"
-                    else:
-                        pr_body += "|"
             else:
                 # if the value is a list, join its items by comma
                 if isinstance(value, list):
@@ -317,3 +296,102 @@ class PRDescription:
             get_logger().info(f"title:\n{title}\n{pr_body}")
 
         return title, pr_body
+
+    def _prepare_file_labels(self):
+        self.file_label_dict = {}
+        for file in self.data['pr_files']:
+            try:
+                filename = file['filename'].replace("'", "`").replace('"', '`')
+                changes_summary = file['changes_summary']
+                label = file['label']
+                if label not in self.file_label_dict:
+                    self.file_label_dict[label] = []
+                self.file_label_dict[label].append((filename, changes_summary))
+            except Exception as e:
+                get_logger().error(f"Error preparing file label dict {self.pr_id}: {e}")
+                pass
+
+    def process_pr_files_prediction(self, pr_body, value):
+        if not self.git_provider.is_supported("gfm_markdown"):
+            get_logger().info(f"Disabling semantic files types for {self.pr_id} since gfm_markdown is not supported")
+            return pr_body
+
+        try:
+            pr_body += "<table>"
+            header = f"Relevant files"
+            delta = 65
+            header += "&nbsp; " * delta
+            pr_body += f"""<thead><tr><th></th><th>{header}</th></tr></thead>"""
+            pr_body += """<tbody>"""
+            for semantic_label in value.keys():
+                s_label = semantic_label.strip("'").strip('"')
+                pr_body += f"""<tr><td><strong>{s_label.capitalize()}</strong></td>"""
+                list_tuples = value[semantic_label]
+                pr_body += f"""<td><details><summary>{len(list_tuples)} files</summary><table>"""
+                for filename, file_change_description in list_tuples:
+                    filename = filename.replace("'", "`")
+                    filename_publish = filename.split("/")[-1]
+                    filename_publish = f"{filename_publish}"
+                    if len(filename_publish) < (delta - 5):
+                        filename_publish += "&nbsp; " * ((delta - 5) - len(filename_publish))
+                    diff_plus_minus = ""
+                    diff_files = self.git_provider.diff_files
+                    for f in diff_files:
+                        if f.filename.lower() == filename.lower():
+                            num_plus_lines = f.num_plus_lines
+                            num_minus_lines = f.num_minus_lines
+                            diff_plus_minus += f"+{num_plus_lines}/-{num_minus_lines}"
+                            break
+
+                    # try to add line numbers link to code suggestions
+                    link = ""
+                    if hasattr(self.git_provider, 'get_line_link'):
+                        filename = filename.strip()
+                        link = self.git_provider.get_line_link(filename, relevant_line_start=-1)
+
+                    file_change_description = self._insert_br_after_x_chars(file_change_description, x=(delta - 5))
+                    pr_body += f"""
+<tr>
+  <td>
+    <details>
+      <summary><strong>{filename_publish}</strong></summary>
+      <ul>
+        {filename}<br><br>
+        <strong>{file_change_description}</strong>
+      </ul>
+    </details>
+  </td>
+  <td><a href="{link}"> {diff_plus_minus}</a></td>
+
+</tr>                    
+"""
+                pr_body += """</table></details></td></tr>"""
+            pr_body += """</tr></tbody></table>"""
+
+        except Exception as e:
+            get_logger().error(f"Error processing pr files to markdown {self.pr_id}: {e}")
+            pass
+        return pr_body
+
+    def _insert_br_after_x_chars(self, text, x=70):
+        """
+        Insert <br> into a string after a word that increases its length above x characters.
+        """
+        if len(text) < x:
+            return text
+
+        words = text.split(' ')
+        new_text = ""
+        current_length = 0
+
+        for word in words:
+            # Check if adding this word exceeds x characters
+            if current_length + len(word) > x:
+                new_text += "<br>"  # Insert line break
+                current_length = 0  # Reset counter
+
+            # Add the word to the new text
+            new_text += word + " "
+            current_length += len(word) + 1  # Add 1 for the space
+
+        return new_text.strip()  # Remove trailing space