From 0396e107069803893aaeb2f7dc44fb5cdc416277 Mon Sep 17 00:00:00 2001 From: zmeir Date: Tue, 18 Jul 2023 16:27:42 +0300 Subject: [PATCH 1/5] Add configuration to request a score for the PR This can help teams compare the review of the PR agent with that of a human reviewer, and fine-tune a score threshold for automatic approval where they decide the agent's review is satisfactory. --- CONFIGURATION.md | 1 + pr_agent/algo/utils.py | 1 + pr_agent/settings/configuration.toml | 1 + pr_agent/settings/pr_reviewer_prompts.toml | 6 ++++++ pr_agent/tools/pr_reviewer.py | 1 + 5 files changed, 10 insertions(+) diff --git a/CONFIGURATION.md b/CONFIGURATION.md index 3d03d7d5..6b47dbe7 100644 --- a/CONFIGURATION.md +++ b/CONFIGURATION.md @@ -9,6 +9,7 @@ You can select your git_provider with the flag `git_provider` in the `config` se You can enable/disable the different PR Reviewer abilities with the following flags (`pr_reviewer` section): ``` require_focused_review=true +require_score_review=true require_tests_review=true require_security_review=true ``` diff --git a/pr_agent/algo/utils.py b/pr_agent/algo/utils.py index f813b8cd..2fcd56c1 100644 --- a/pr_agent/algo/utils.py +++ b/pr_agent/algo/utils.py @@ -12,6 +12,7 @@ def convert_to_markdown(output_data: dict) -> str: emojis = { "Main theme": "๐ŸŽฏ", "Type of PR": "๐Ÿ“Œ", + "Score": "๐Ÿ…", "Relevant tests added": "๐Ÿงช", "Unrelated changes": "โš ๏ธ", "Focused PR": "โœจ", diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index fc2fa2b8..3ccbd59c 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -6,6 +6,7 @@ verbosity_level=0 # 0,1,2 [pr_reviewer] require_focused_review=true +require_score_review=true require_tests_review=true require_security_review=true num_code_suggestions=3 diff --git a/pr_agent/settings/pr_reviewer_prompts.toml b/pr_agent/settings/pr_reviewer_prompts.toml index 51a873c4..e8732b53 100644 --- a/pr_agent/settings/pr_reviewer_prompts.toml +++ b/pr_agent/settings/pr_reviewer_prompts.toml @@ -20,6 +20,12 @@ You must use the following JSON schema to format your answer: "type": "string", "enum": ["Bug fix", "Tests", "Bug fix with tests", "Refactoring", "Enhancement", "Documentation", "Other"] }, +{%- if require_score %} + "Score": { + "type": "float", + "description": "Rate this PR on a scale of 0-100 (inclusive), where 0 means the worst possible code, and 100 means code of the highest quality without any bugs or performace issues that is ready to be merged immediately and run in production at scale." + }, +{%- endif %} {%- if require_tests %} "Relevant tests added": { "type": "string", diff --git a/pr_agent/tools/pr_reviewer.py b/pr_agent/tools/pr_reviewer.py index 4f6fde74..52e022c0 100644 --- a/pr_agent/tools/pr_reviewer.py +++ b/pr_agent/tools/pr_reviewer.py @@ -35,6 +35,7 @@ class PRReviewer: "description": self.git_provider.get_pr_description(), "language": self.main_language, "diff": "", # empty diff for initial calculation + "require_score": settings.pr_reviewer.require_score_review, "require_tests": settings.pr_reviewer.require_tests_review, "require_security": settings.pr_reviewer.require_security_review, "require_focused": settings.pr_reviewer.require_focused_review, From f8aea909b4455f99b621bb1f63a1d5e7dced2dd1 Mon Sep 17 00:00:00 2001 From: zmeir Date: Wed, 19 Jul 2023 10:57:35 +0300 Subject: [PATCH 2/5] Add example output --- pr_agent/settings/pr_reviewer_prompts.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pr_agent/settings/pr_reviewer_prompts.toml b/pr_agent/settings/pr_reviewer_prompts.toml index e8732b53..3128c0e2 100644 --- a/pr_agent/settings/pr_reviewer_prompts.toml +++ b/pr_agent/settings/pr_reviewer_prompts.toml @@ -89,6 +89,9 @@ Example output: { "Main theme": "xxx", "Type of PR": "Bug fix", +{%- if require_score %} + "Score": 85.2, +{%- endif %} {%- if require_tests %} "Relevant tests added": "No", {%- endif %} From fc8494d69663a9db1395239df20dccb05408b747 Mon Sep 17 00:00:00 2001 From: zmeir Date: Wed, 19 Jul 2023 10:59:52 +0300 Subject: [PATCH 3/5] Rephrase score description in promt --- pr_agent/settings/pr_reviewer_prompts.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr_agent/settings/pr_reviewer_prompts.toml b/pr_agent/settings/pr_reviewer_prompts.toml index 3128c0e2..9a55dcf4 100644 --- a/pr_agent/settings/pr_reviewer_prompts.toml +++ b/pr_agent/settings/pr_reviewer_prompts.toml @@ -23,7 +23,7 @@ You must use the following JSON schema to format your answer: {%- if require_score %} "Score": { "type": "float", - "description": "Rate this PR on a scale of 0-100 (inclusive), where 0 means the worst possible code, and 100 means code of the highest quality without any bugs or performace issues that is ready to be merged immediately and run in production at scale." + "description": "Rate this PR on a scale of 0-100 (inclusive), where 0 means the worst possible PR code, and 100 means PR code of the highest quality, without any bugs or performance issues, that is ready to be merged immediately and run in production at scale." }, {%- endif %} {%- if require_tests %} From e17dd66dce07c458f10383a26a990f76b075a119 Mon Sep 17 00:00:00 2001 From: zmeir Date: Wed, 19 Jul 2023 11:00:28 +0300 Subject: [PATCH 4/5] Disable score review by default --- pr_agent/settings/configuration.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr_agent/settings/configuration.toml b/pr_agent/settings/configuration.toml index 3ccbd59c..485f2a53 100644 --- a/pr_agent/settings/configuration.toml +++ b/pr_agent/settings/configuration.toml @@ -6,7 +6,7 @@ verbosity_level=0 # 0,1,2 [pr_reviewer] require_focused_review=true -require_score_review=true +require_score_review=false require_tests_review=true require_security_review=true num_code_suggestions=3 From 7e2449b228231a08d61603ab8599728b9167f632 Mon Sep 17 00:00:00 2001 From: zmeir Date: Wed, 19 Jul 2023 13:37:35 +0300 Subject: [PATCH 5/5] Changed score type to int --- pr_agent/settings/pr_reviewer_prompts.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pr_agent/settings/pr_reviewer_prompts.toml b/pr_agent/settings/pr_reviewer_prompts.toml index 9a55dcf4..100e6234 100644 --- a/pr_agent/settings/pr_reviewer_prompts.toml +++ b/pr_agent/settings/pr_reviewer_prompts.toml @@ -22,7 +22,7 @@ You must use the following JSON schema to format your answer: }, {%- if require_score %} "Score": { - "type": "float", + "type": "int", "description": "Rate this PR on a scale of 0-100 (inclusive), where 0 means the worst possible PR code, and 100 means PR code of the highest quality, without any bugs or performance issues, that is ready to be merged immediately and run in production at scale." }, {%- endif %} @@ -90,7 +90,7 @@ Example output: "Main theme": "xxx", "Type of PR": "Bug fix", {%- if require_score %} - "Score": 85.2, + "Score": 89, {%- endif %} {%- if require_tests %} "Relevant tests added": "No",