From 8f0f08006f405bfab3cf8f8f353e4a216ffc5426 Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 14 Apr 2024 12:00:19 +0300 Subject: [PATCH 1/7] s --- pr_agent/algo/ai_handlers/base_ai_handler.py | 2 +- .../algo/ai_handlers/litellm_ai_handler.py | 12 +++++++- pr_agent/servers/github_app.py | 9 ++++-- pr_agent/settings/configuration.toml | 6 ++-- pr_agent/tools/pr_questions.py | 29 +++++++++++++++++-- 5 files changed, 49 insertions(+), 9 deletions(-) diff --git a/pr_agent/algo/ai_handlers/base_ai_handler.py b/pr_agent/algo/ai_handlers/base_ai_handler.py index c8473fb3..b5166b8e 100644 --- a/pr_agent/algo/ai_handlers/base_ai_handler.py +++ b/pr_agent/algo/ai_handlers/base_ai_handler.py @@ -15,7 +15,7 @@ class BaseAiHandler(ABC): pass @abstractmethod - async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2): + async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2, img_path: str = None): """ This method should be implemented to return a chat completion from the AI model. Args: diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index d07542f6..536faf41 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -102,13 +102,23 @@ class LiteLLMAIHandler(BaseAiHandler): retry=retry_if_exception_type((openai.APIError, openai.APIConnectionError, openai.Timeout)), # No retry on RateLimitError stop=stop_after_attempt(OPENAI_RETRIES) ) - async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2): + async def chat_completion(self, model: str, system: str, user: str, temperature: float = 0.2, img_path: str = None): try: resp, finish_reason = None, None deployment_id = self.deployment_id if self.azure: model = 'azure/' + model messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] + if img_path: + import requests + r = requests.get(img_path, allow_redirects=True) + if r.status_code == 404: + error_msg = "The image link is not alive. Please repost the image, get a new address, and send the question again." + get_logger().error(error_msg) + return f"{error_msg}", "error" + messages[1]["content"] = [{"type": "text", "text": messages[1]["content"]}, + {"type": "image_url", "image_url": {"url": img_path}}] + kwargs = { "model": model, "deployment_id": deployment_id, diff --git a/pr_agent/servers/github_app.py b/pr_agent/servers/github_app.py index bc7042b1..6d942289 100644 --- a/pr_agent/servers/github_app.py +++ b/pr_agent/servers/github_app.py @@ -86,8 +86,13 @@ async def handle_comments_on_pr(body: Dict[str, Any], return {} comment_body = body.get("comment", {}).get("body") if comment_body and isinstance(comment_body, str) and not comment_body.lstrip().startswith("/"): - get_logger().info("Ignoring comment not starting with /") - return {} + if '/ask' in comment_body and comment_body.strip().startswith('> ![image]'): + comment_body_split = comment_body.split('/ask') + comment_body = '/ask' + comment_body_split[1] +'/n' +comment_body_split[0].strip() + get_logger().info(f"Reformatting comment_body so command is at the beginning: {comment_body}") + else: + get_logger().info("Ignoring comment not starting with /") + return {} disable_eyes = False if "issue" in body and "pull_request" in body["issue"] and "url" in body["issue"]["pull_request"]: api_url = body["issue"]["pull_request"]["url"] diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index 39282706..d49b345c 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -1,7 +1,7 @@ [config] -model="gpt-4" # "gpt-4-0125-preview" -model_turbo="gpt-4-0125-preview" -fallback_models=["gpt-3.5-turbo-16k"] +model="gpt-4-turbo" # "gpt-4-0125-preview" +model_turbo="gpt-4-turbo" +fallback_models=["gpt-4-0125-preview"] git_provider="github" publish_output=true publish_output_progress=true diff --git a/pr_agent/tools/pr_questions.py b/pr_agent/tools/pr_questions.py index 4e1d3c1e..1e2a360e 100644 --- a/pr_agent/tools/pr_questions.py +++ b/pr_agent/tools/pr_questions.py @@ -56,6 +56,12 @@ class PRQuestions: get_logger().debug("Relevant configs", artifacts=relevant_configs) if get_settings().config.publish_output: self.git_provider.publish_comment("Preparing answer...", is_temporary=True) + + # identify image + img_path = self.idenfity_image_in_comment() + if img_path: + get_logger().debug(f"Image path identified", artifact=img_path) + await retry_with_fallback_models(self._prepare_prediction) pr_comment = self._prepare_pr_answer() @@ -71,6 +77,19 @@ class PRQuestions: self.git_provider.remove_initial_comment() return "" + def idenfity_image_in_comment(self): + img_path = '' + if '![image]' in self.question_str: + # assuming structure: + # /ask question ... > ![image](img_path) + img_path = self.question_str.split('![image]')[1].strip().strip('()') + self.vars['img_path'] = img_path + elif 'https://' in self.question_str and '.png' in self.question_str: # direct image link + # include https:// in the image path + img_path = 'https://' + self.question_str.split('https://')[1] + self.vars['img_path'] = img_path + return img_path + async def _prepare_prediction(self, model: str): self.patches_diff = get_pr_diff(self.git_provider, self.token_handler, model) if self.patches_diff: @@ -86,8 +105,14 @@ class PRQuestions: environment = Environment(undefined=StrictUndefined) system_prompt = environment.from_string(get_settings().pr_questions_prompt.system).render(variables) user_prompt = environment.from_string(get_settings().pr_questions_prompt.user).render(variables) - response, finish_reason = await self.ai_handler.chat_completion(model=model, temperature=0.2, - system=system_prompt, user=user_prompt) + if 'img_path' in variables: + img_path = self.vars['img_path'] + response, finish_reason = await self.ai_handler.chat_completion(model=model, temperature=0.2, + system=system_prompt, user=user_prompt, + img_path=img_path) + else: + response, finish_reason = await self.ai_handler.chat_completion(model=model, temperature=0.2, + system=system_prompt, user=user_prompt) return response def _prepare_pr_answer(self) -> str: From 4683a29819be315268e922dd3652fca53a50cec8 Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 14 Apr 2024 12:34:14 +0300 Subject: [PATCH 2/7] s --- docs/docs/tools/ask.md | 35 +++++++++++++++++-- .../algo/ai_handlers/litellm_ai_handler.py | 2 +- pr_agent/servers/github_app.py | 2 +- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/docs/docs/tools/ask.md b/docs/docs/tools/ask.md index 453807fd..79c4f00f 100644 --- a/docs/docs/tools/ask.md +++ b/docs/docs/tools/ask.md @@ -7,9 +7,9 @@ It can be invoked manually by commenting on any PR: ``` For example: -![Ask Comment](https://codium.ai/images/pr_agent/ask_comment.png){width=768} +![Ask Comment](https://codium.ai/images/pr_agent/ask_comment.png){width=512} -![Ask](https://codium.ai/images/pr_agent/ask.png){width=768} +![Ask](https://codium.ai/images/pr_agent/ask.png){width=512} ## Ask lines @@ -18,6 +18,35 @@ You can run `/ask` on specific lines of code in the PR from the PR's diff view. - To select multiple lines, click on the '+' sign of the first line and then hold and drag to select the rest of the lines. - write `/ask "..."` in the comment box and press `Add single comment` button. -![Ask Line](https://codium.ai/images/pr_agent/Ask_line.png){width=768} +![Ask Line](https://codium.ai/images/pr_agent/Ask_line.png){width=512} Note that the tool does not have "memory" of previous questions, and answers each question independently. + +## Ask on images (using the PR code as context) + +You can also ask questions about images that appear in the comment, where the entire PR is considered as the context. The tool will answer questions based on the images in the PR. +The basic syntax is: +``` +/ask "..." + +[Image](https://real_link_to_image) +``` + +Note that GitHub has a mecahnism of pasting images in comments. However, pasted image does not provide a direct link. +To get a direct link to the image, we recommend using the following steps: +1) send a comment that contains only the image: + +![Ask image1](https://codium.ai/images/pr_agent/ask_images1.png){width=512} + +2) quote reply to that comment: + +![Ask image2](https://codium.ai/images/pr_agent/ask_images2.png){width=512} + +3) type the question below the image: + +![Ask image3](https://codium.ai/images/pr_agent/ask_images3.png){width=512} +![Ask image4](https://codium.ai/images/pr_agent/ask_images3.png){width=512} + +4) post the comment, and receive the answer: + +![Ask image5](https://codium.ai/images/pr_agent/ask_images5.png){width=512} \ No newline at end of file diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index 536faf41..a547d956 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -113,7 +113,7 @@ class LiteLLMAIHandler(BaseAiHandler): import requests r = requests.get(img_path, allow_redirects=True) if r.status_code == 404: - error_msg = "The image link is not alive. Please repost the image, get a new address, and send the question again." + error_msg = f"The image link is not [alive](img_path).\nPlease repost the original image as a comment, and send the question again with 'quote reply' (see [instructions](https://pr-agent-docs.codium.ai/tools/ask/#ask-on-images-using-the-pr-code-as-context))." get_logger().error(error_msg) return f"{error_msg}", "error" messages[1]["content"] = [{"type": "text", "text": messages[1]["content"]}, diff --git a/pr_agent/servers/github_app.py b/pr_agent/servers/github_app.py index 6d942289..f0d1340d 100644 --- a/pr_agent/servers/github_app.py +++ b/pr_agent/servers/github_app.py @@ -88,7 +88,7 @@ async def handle_comments_on_pr(body: Dict[str, Any], if comment_body and isinstance(comment_body, str) and not comment_body.lstrip().startswith("/"): if '/ask' in comment_body and comment_body.strip().startswith('> ![image]'): comment_body_split = comment_body.split('/ask') - comment_body = '/ask' + comment_body_split[1] +'/n' +comment_body_split[0].strip() + comment_body = '/ask' + comment_body_split[1] +' \n' +comment_body_split[0].strip().lstrip('>') get_logger().info(f"Reformatting comment_body so command is at the beginning: {comment_body}") else: get_logger().info("Ignoring comment not starting with /") From f0230fce79dbb2fcba44fde112e042b282f034e4 Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 14 Apr 2024 12:37:54 +0300 Subject: [PATCH 3/7] gpt-4-turbo-2024-04-09 --- pr_agent/settings/configuration.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index d49b345c..4ab511c1 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -1,6 +1,6 @@ [config] -model="gpt-4-turbo" # "gpt-4-0125-preview" -model_turbo="gpt-4-turbo" +model="gpt-4-turbo-2024-04-09" +model_turbo="gpt-4-turbo-2024-04-09" fallback_models=["gpt-4-0125-preview"] git_provider="github" publish_output=true From 86e64501dfb69ff87aa2b165035902ccae6d2b2f Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 14 Apr 2024 12:43:26 +0300 Subject: [PATCH 4/7] ask --- pr_agent/servers/help.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pr_agent/servers/help.py b/pr_agent/servers/help.py index 9e85887d..d76f70a4 100644 --- a/pr_agent/servers/help.py +++ b/pr_agent/servers/help.py @@ -160,16 +160,17 @@ It can be invoked manually by commenting on any PR: /ask "..." ``` -Note that the tool does not have "memory" of previous questions, and answers each question independently. +Note that the tool does not have "memory" of previous questions, and answers each question independently. +You can ask questions about the entire PR, about specific code lines, or about an image related to the PR code changes. """ - output += "\n\n" - - # general - output += "\n\n\n\n" - - output += "
More PR-Agent commands
\n\n" - output += HelpMessage.get_general_bot_help_text() - output += "\n\n
" + # output += "\n\n" + # + # # # general + # # output += "\n\n\n\n" + # + # output += "
More PR-Agent commands
\n\n" + # # output += HelpMessage.get_general_bot_help_text() + # # output += "\n\n
" output += f"\n\nSee the [ask usage](https://pr-agent-docs.codium.ai/tools/ask/) page for a comprehensive guide on using this tool.\n\n" From 92ef2b4464ce6fed2ab68d901428cc7cc03eb752 Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 14 Apr 2024 14:09:58 +0300 Subject: [PATCH 5/7] ask --- README.md | 5 +++++ .../algo/ai_handlers/litellm_ai_handler.py | 18 +++++++++++------- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 1ac8b7da..91f7a5ef 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,11 @@ CodiumAI PR-Agent aims to help efficiently review and handle pull requests, by p ## News and Updates +### April 14, 2024 +You can now ask questions about images that appear in the comment, where the entire PR is considered as the context. +see [here](https://pr-agent-docs.codium.ai/tools/ask/#ask-on-images-using-the-pr-code-as-context). +![Ask image5](https://codium.ai/images/pr_agent/ask_images5.png){width=512} + ### March 24, 2024 PR-Agent is now available for easy installation via [pip](https://pr-agent-docs.codium.ai/installation/locally/#using-pip-package). diff --git a/pr_agent/algo/ai_handlers/litellm_ai_handler.py b/pr_agent/algo/ai_handlers/litellm_ai_handler.py index a547d956..4acd55ea 100644 --- a/pr_agent/algo/ai_handlers/litellm_ai_handler.py +++ b/pr_agent/algo/ai_handlers/litellm_ai_handler.py @@ -1,5 +1,5 @@ import os - +import requests import boto3 import litellm import openai @@ -110,12 +110,16 @@ class LiteLLMAIHandler(BaseAiHandler): model = 'azure/' + model messages = [{"role": "system", "content": system}, {"role": "user", "content": user}] if img_path: - import requests - r = requests.get(img_path, allow_redirects=True) - if r.status_code == 404: - error_msg = f"The image link is not [alive](img_path).\nPlease repost the original image as a comment, and send the question again with 'quote reply' (see [instructions](https://pr-agent-docs.codium.ai/tools/ask/#ask-on-images-using-the-pr-code-as-context))." - get_logger().error(error_msg) - return f"{error_msg}", "error" + try: + # check if the image link is alive + r = requests.head(img_path, allow_redirects=True) + if r.status_code == 404: + error_msg = f"The image link is not [alive](img_path).\nPlease repost the original image as a comment, and send the question again with 'quote reply' (see [instructions](https://pr-agent-docs.codium.ai/tools/ask/#ask-on-images-using-the-pr-code-as-context))." + get_logger().error(error_msg) + return f"{error_msg}", "error" + except Exception as e: + get_logger().error(f"Error fetching image: {img_path}", e) + return f"Error fetching image: {img_path}", "error" messages[1]["content"] = [{"type": "text", "text": messages[1]["content"]}, {"type": "image_url", "image_url": {"url": img_path}}] From 506e3007c4549262b82c2478ca0c83afccfca569 Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 14 Apr 2024 14:11:04 +0300 Subject: [PATCH 6/7] ask --- README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 91f7a5ef..40453d37 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,9 @@ CodiumAI PR-Agent aims to help efficiently review and handle pull requests, by p ### April 14, 2024 You can now ask questions about images that appear in the comment, where the entire PR is considered as the context. -see [here](https://pr-agent-docs.codium.ai/tools/ask/#ask-on-images-using-the-pr-code-as-context). -![Ask image5](https://codium.ai/images/pr_agent/ask_images5.png){width=512} +see [here](https://pr-agent-docs.codium.ai/tools/ask/#ask-on-images-using-the-pr-code-as-context) for more details. + + ### March 24, 2024 PR-Agent is now available for easy installation via [pip](https://pr-agent-docs.codium.ai/installation/locally/#using-pip-package). From 44eb0b4f237d68df03955082c8338464bdf478ee Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 14 Apr 2024 14:12:48 +0300 Subject: [PATCH 7/7] ask --- pr_agent/tools/pr_questions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr_agent/tools/pr_questions.py b/pr_agent/tools/pr_questions.py index 1e2a360e..d78d0880 100644 --- a/pr_agent/tools/pr_questions.py +++ b/pr_agent/tools/pr_questions.py @@ -84,7 +84,7 @@ class PRQuestions: # /ask question ... > ![image](img_path) img_path = self.question_str.split('![image]')[1].strip().strip('()') self.vars['img_path'] = img_path - elif 'https://' in self.question_str and '.png' in self.question_str: # direct image link + elif 'https://' in self.question_str and ('.png' in self.question_str or 'jpg' in self.question_str): # direct image link # include https:// in the image path img_path = 'https://' + self.question_str.split('https://')[1] self.vars['img_path'] = img_path