From 43e91b0df72244972ac71662c358438bb031638f Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 8 Dec 2024 11:27:43 +0200 Subject: [PATCH 1/3] feat: add health test for PR agent commands and improve output handling --- pr_agent/tools/pr_code_suggestions.py | 3 +- pr_agent/tools/pr_description.py | 4 ++ pr_agent/tools/pr_reviewer.py | 4 ++ tests/health_test/main.py | 70 +++++++++++++++++++++++++++ 4 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 tests/health_test/main.py diff --git a/pr_agent/tools/pr_code_suggestions.py b/pr_agent/tools/pr_code_suggestions.py index 31ad2bd9..9f34718a 100644 --- a/pr_agent/tools/pr_code_suggestions.py +++ b/pr_agent/tools/pr_code_suggestions.py @@ -204,7 +204,8 @@ class PRCodeSuggestions: self.git_provider.remove_comment(self.progress_response) else: get_logger().info('Code suggestions generated for PR, but not published since publish_output is False.') - get_settings().data = {"artifact": data} + pr_body = self.generate_summarized_suggestions(data) + get_settings().data = {"artifact": pr_body} return except Exception as e: get_logger().error(f"Failed to generate code suggestions for PR, error: {e}", diff --git a/pr_agent/tools/pr_description.py b/pr_agent/tools/pr_description.py index e5cfea2f..0b4e19cd 100644 --- a/pr_agent/tools/pr_description.py +++ b/pr_agent/tools/pr_description.py @@ -171,6 +171,10 @@ class PRDescription: update_comment = f"**[PR Description]({pr_url})** updated to latest commit ({latest_commit_url})" self.git_provider.publish_comment(update_comment) self.git_provider.remove_initial_comment() + else: + get_logger().info('PR description, but not published since publish_output is False.') + get_settings().data = {"artifact": pr_body} + return except Exception as e: get_logger().error(f"Error generating PR description {self.pr_id}: {e}") diff --git a/pr_agent/tools/pr_reviewer.py b/pr_agent/tools/pr_reviewer.py index e008703e..c49155e4 100644 --- a/pr_agent/tools/pr_reviewer.py +++ b/pr_agent/tools/pr_reviewer.py @@ -170,6 +170,10 @@ class PRReviewer: self.git_provider.remove_initial_comment() if get_settings().pr_reviewer.inline_code_comments: self._publish_inline_code_comments() + else: + get_logger().info("Review output is not published") + get_settings().data = {"artifact": pr_review} + return except Exception as e: get_logger().error(f"Failed to review PR: {e}") diff --git a/tests/health_test/main.py b/tests/health_test/main.py new file mode 100644 index 00000000..15030565 --- /dev/null +++ b/tests/health_test/main.py @@ -0,0 +1,70 @@ +import argparse +import asyncio +import copy +import os +from pathlib import Path + +from starlette_context import request_cycle_context, context + +from pr_agent.cli import run_command +from pr_agent.config_loader import get_settings, global_settings + +from pr_agent.agent.pr_agent import PRAgent, commands +from pr_agent.log import get_logger, setup_logger +from tests.e2e_tests import e2e_utils + +log_level = os.environ.get("LOG_LEVEL", "INFO") +setup_logger(log_level) + + +async def run_async(): + pr_url = "https://github.com/Codium-ai/pr-agent/pull/1385" # A small PR to test the agent + + get_settings().set("config.git_provider", "github") + get_settings().set("config.publish_output", False) + get_settings().set("config.fallback_models", []) + + agent = PRAgent() + try: + # Run the 'describe' command + get_logger().info(f"\nSanity check for the 'describe' command...") + original_settings = copy.deepcopy(get_settings()) + await agent.handle_request(pr_url, ['describe']) + pr_header_body = dict(get_settings().data)['artifact'] + assert pr_header_body.startswith('###') and 'PR Type' in pr_header_body and 'Description' in pr_header_body + context['settings'] = copy.deepcopy(original_settings) # Restore settings state after each test to prevent test interference + get_logger().info("PR description generated successfully\n") + + # Run the 'review' command + get_logger().info(f"\nSanity check for the 'review' command...") + original_settings = copy.deepcopy(get_settings()) + await agent.handle_request(pr_url, ['review']) + pr_review_body = dict(get_settings().data)['artifact'] + assert pr_review_body.startswith('##') and 'PR Reviewer Guide' in pr_review_body + context['settings'] = copy.deepcopy(original_settings) # Restore settings state after each test to prevent test interference + get_logger().info("PR review generated successfully\n") + + # Run the 'improve' command + get_logger().info(f"\nSanity check for the 'improve' command...") + original_settings = copy.deepcopy(get_settings()) + await agent.handle_request(pr_url, ['improve']) + pr_improve_body = dict(get_settings().data)['artifact'] + assert pr_improve_body.startswith('##') and 'PR Code Suggestions' in pr_improve_body + context['settings'] = copy.deepcopy(original_settings) # Restore settings state after each test to prevent test interference + get_logger().info("PR improvements generated successfully\n") + + get_logger().info(f"\n\n========\nHealth test passed successfully\n========") + + except Exception as e: + get_logger().exception(f"\n\n========\nHealth test failed\n========") + raise e + + +def run(): + with request_cycle_context({}): + context['settings'] = copy.deepcopy(global_settings) + asyncio.run(run_async()) + + +if __name__ == '__main__': + run() From e8fc351ce9e2944058b70290ba669255676ce1f4 Mon Sep 17 00:00:00 2001 From: mrT23 Date: Sun, 8 Dec 2024 11:39:15 +0200 Subject: [PATCH 2/3] docs: add CLI health check section and reorganize automations documentation --- .../docs/usage-guide/automations_and_usage.md | 48 ++++++++++++++----- 1 file changed, 37 insertions(+), 11 deletions(-) diff --git a/docs/docs/usage-guide/automations_and_usage.md b/docs/docs/usage-guide/automations_and_usage.md index 454f83d0..d0222f15 100644 --- a/docs/docs/usage-guide/automations_and_usage.md +++ b/docs/docs/usage-guide/automations_and_usage.md @@ -1,4 +1,5 @@ ## Local repo (CLI) + When running from your locally cloned Qodo Merge repo (CLI), your local configuration file will be used. Examples of invoking the different tools via the CLI: @@ -35,9 +36,28 @@ This is useful for debugging or experimenting with different tools. Default is "github". +### CLI Health Check +To verify that Qodo Merge has been configured correctly, you can run this health check command from the repository root: +```bash +python -m tests.health_test.main +``` -### Online usage +If the health check passes, you will see the following output: + +``` +======== +Health test passed successfully +======== +``` + +At the end of the run. + +Before running the health check, ensure you have: +- Configured your [LLM provider](https://qodo-merge-docs.qodo.ai/usage-guide/changing_a_model/) +- Added a valid GitHub token to your configuration file + +## Online usage Online usage means invoking Qodo Merge tools by [comments](https://github.com/Codium-ai/pr-agent/pull/229#issuecomment-1695021901) on a PR. Commands for invoking the different tools via comments: @@ -57,7 +77,11 @@ For example, if you want to edit the `review` tool configurations, you can run: ``` Any configuration value in [configuration file](https://github.com/Codium-ai/pr-agent/blob/main/pr_agent/settings/configuration.toml) file can be similarly edited. Comment `/config` to see the list of available configurations. -## Disabling automatic feedback + +## Qodo Merge Automatic Feedback + + +### Disabling all automatic feedback To easily disable all automatic feedback from Qodo Merge (GitHub App, GitLab Webhook, BitBucket App, Azure DevOps Webhook), set in a configuration file: @@ -66,13 +90,15 @@ To easily disable all automatic feedback from Qodo Merge (GitHub App, GitLab Web disable_auto_feedback = true ``` -## GitHub App +When this parameter is set to `true`, Qodo Merge will not run any automatic tools (like `describe`, `review`, `improve`) when a new PR is opened, or when new code is pushed to an open PR. + +### GitHub App !!! note "Configurations for Qodo Merge Pro" Qodo Merge Pro for GitHub is an App, hosted by CodiumAI. So all the instructions below are relevant also for Qodo Merge Pro users. Same goes for [GitLab webhook](#gitlab-webhook) and [BitBucket App](#bitbucket-app) sections. -### GitHub app automatic tools when a new PR is opened +#### GitHub app automatic tools when a new PR is opened The [github_app](https://github.com/Codium-ai/pr-agent/blob/main/pr_agent/settings/configuration.toml#L108) section defines GitHub app specific configurations. @@ -105,7 +131,7 @@ pr_commands = ["describe", "review"] In this case, only the `describe` and `review` tools will run automatically when a new PR is opened. -### GitHub app automatic tools for push actions (commits to an open PR) +#### GitHub app automatic tools for push actions (commits to an open PR) In addition to running automatic tools when a PR is opened, the GitHub app can also respond to new code that is pushed to an open PR. @@ -121,7 +147,7 @@ push_commands = [ ``` This means that when new code is pushed to the PR, the Qodo Merge will run the `describe` and `review` tools, with the specified parameters. -## GitHub Action +### GitHub Action `GitHub Action` is a different way to trigger Qodo Merge tools, and uses a different configuration mechanism than `GitHub App`.
You can configure settings for `GitHub Action` by adding environment variables under the env section in `.github/workflows/pr_agent.yml` file. Specifically, start by setting the following environment variables: @@ -155,7 +181,7 @@ publish_labels = false to prevent Qodo Merge from publishing labels when running the `describe` tool. -## GitLab Webhook +### GitLab Webhook After setting up a GitLab webhook, to control which commands will run automatically when a new MR is opened, you can set the `pr_commands` parameter in the configuration file, similar to the GitHub App: ```toml @@ -181,7 +207,7 @@ push_commands = [ Note that to use the 'handle_push_trigger' feature, you need to give the gitlab webhook also the "Push events" scope. -## BitBucket App +### BitBucket App Similar to GitHub app, when running Qodo Merge from BitBucket App, the default [configuration file](https://github.com/Codium-ai/pr-agent/blob/main/pr_agent/settings/configuration.toml) from a pre-built docker will be initially loaded. By uploading a local `.pr_agent.toml` file to the root of the repo's main branch, you can edit and customize any configuration parameter. Note that you need to upload `.pr_agent.toml` prior to creating a PR, in order for the configuration to take effect. @@ -200,7 +226,7 @@ If you experience a lack of responses from Qodo Merge, you might want to set: `b This will prevent Qodo Merge from acquiring the full file content, and will only use the diff content. This will reduce the number of requests made to BitBucket, at the cost of small decrease in accuracy, as dynamic context will not be applicable. -### BitBucket Self-Hosted App automatic tools +#### BitBucket Self-Hosted App automatic tools To control which commands will run automatically when a new PR is opened, you can set the `pr_commands` parameter in the configuration file: Specifically, set the following values: @@ -225,7 +251,7 @@ push_commands = [ ] ``` -## Azure DevOps provider +### Azure DevOps provider To use Azure DevOps provider use the following settings in configuration.toml: ```toml @@ -247,7 +273,7 @@ org = "https://dev.azure.com/YOUR_ORGANIZATION/" # pat = "YOUR_PAT_TOKEN" needed only if using PAT for authentication ``` -### Azure DevOps Webhook +#### Azure DevOps Webhook To control which commands will run automatically when a new PR is opened, you can set the `pr_commands` parameter in the configuration file, similar to the GitHub App: ```toml From 7d5e59cd4056101df19957ad37bbbcbb51db735d Mon Sep 17 00:00:00 2001 From: Tal Date: Sun, 8 Dec 2024 11:41:05 +0200 Subject: [PATCH 3/3] Update tests/health_test/main.py Co-authored-by: qodo-merge-pro-for-open-source[bot] <189517486+qodo-merge-pro-for-open-source[bot]@users.noreply.github.com> --- tests/health_test/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/health_test/main.py b/tests/health_test/main.py index 15030565..da9142e4 100644 --- a/tests/health_test/main.py +++ b/tests/health_test/main.py @@ -18,7 +18,7 @@ setup_logger(log_level) async def run_async(): - pr_url = "https://github.com/Codium-ai/pr-agent/pull/1385" # A small PR to test the agent + pr_url = os.getenv('TEST_PR_URL', 'https://github.com/Codium-ai/pr-agent/pull/1385') get_settings().set("config.git_provider", "github") get_settings().set("config.publish_output", False)