From 6c4a5bae52f969ee1fedb4bd7e855d7c333b8ee0 Mon Sep 17 00:00:00 2001 From: zmeir Date: Mon, 7 Aug 2023 16:17:06 +0300 Subject: [PATCH 1/9] Support fallback deployments to accompany fallback models This is useful for example in Azure OpenAI deployments where you have a different deployment per model, so the current fallback implementation doesn't work (still uses the same deployment for each fallback attempt) --- pr_agent/algo/ai_handler.py | 8 +++++++- pr_agent/algo/pr_processing.py | 17 +++++++++++++++-- pr_agent/settings/.secrets_template.toml | 1 + 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/pr_agent/algo/ai_handler.py b/pr_agent/algo/ai_handler.py index 57221518..cfca63f6 100644 --- a/pr_agent/algo/ai_handler.py +++ b/pr_agent/algo/ai_handler.py @@ -27,7 +27,6 @@ class AiHandler: self.azure = False if get_settings().get("OPENAI.ORG", None): litellm.organization = get_settings().openai.org - self.deployment_id = get_settings().get("OPENAI.DEPLOYMENT_ID", None) if get_settings().get("OPENAI.API_TYPE", None): if get_settings().openai.api_type == "azure": self.azure = True @@ -45,6 +44,13 @@ class AiHandler: except AttributeError as e: raise ValueError("OpenAI key is required") from e + @property + def deployment_id(self): + """ + Returns the deployment ID for the OpenAI API. + """ + return get_settings().get("OPENAI.DEPLOYMENT_ID", None) + @retry(exceptions=(APIError, Timeout, TryAgain, AttributeError, RateLimitError), tries=OPENAI_RETRIES, delay=2, backoff=2, jitter=(1, 3)) async def chat_completion(self, model: str, temperature: float, system: str, user: str): diff --git a/pr_agent/algo/pr_processing.py b/pr_agent/algo/pr_processing.py index 8b319446..fae2535a 100644 --- a/pr_agent/algo/pr_processing.py +++ b/pr_agent/algo/pr_processing.py @@ -208,13 +208,26 @@ def pr_generate_compressed_diff(top_langs: list, token_handler: TokenHandler, mo async def retry_with_fallback_models(f: Callable): + # getting all models model = get_settings().config.model fallback_models = get_settings().config.fallback_models if not isinstance(fallback_models, list): - fallback_models = [fallback_models] + fallback_models = [m.strip() for m in fallback_models.split(",")] all_models = [model] + fallback_models - for i, model in enumerate(all_models): + + # getting all deployments + deployment_id = get_settings().get("openai.deployment_id", None) + fallback_deployments = get_settings().get("openai.fallback_deployments", []) + if not isinstance(fallback_deployments, list) and fallback_deployments: + fallback_deployments = [d.strip() for d in fallback_deployments.split(",")] + if fallback_deployments: + all_deployments = [deployment_id] + fallback_deployments + else: + all_deployments = [deployment_id] * len(all_models) + # try each (model, deployment_id) pair until one is successful, otherwise raise exception + for i, (model, deployment_id) in enumerate(zip(all_models, all_deployments)): try: + get_settings().set("openai.deployment_id", deployment_id) return await f(model) except Exception as e: logging.warning(f"Failed to generate prediction with {model}: {traceback.format_exc()}") diff --git a/pr_agent/settings/.secrets_template.toml b/pr_agent/settings/.secrets_template.toml index 36b529a6..25a6562f 100644 --- a/pr_agent/settings/.secrets_template.toml +++ b/pr_agent/settings/.secrets_template.toml @@ -14,6 +14,7 @@ key = "" # Acquire through https://platform.openai.com #api_version = '2023-05-15' # Check Azure documentation for the current API version #api_base = "" # The base URL for your Azure OpenAI resource. e.g. "https://.openai.azure.com" #deployment_id = "" # The deployment name you chose when you deployed the engine +#fallback_deployments = [] # Match your fallback models from configuration.toml with the appropriate deployment_id [anthropic] key = "" # Optional, uncomment if you want to use Anthropic. Acquire through https://www.anthropic.com/ From 7f1849a86712e117465faf573485169d175872c1 Mon Sep 17 00:00:00 2001 From: zmeir Date: Mon, 7 Aug 2023 22:42:53 +0300 Subject: [PATCH 2/9] Logging --- pr_agent/algo/ai_handler.py | 8 +++++++- pr_agent/algo/pr_processing.py | 7 +++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pr_agent/algo/ai_handler.py b/pr_agent/algo/ai_handler.py index cfca63f6..5fcb13c7 100644 --- a/pr_agent/algo/ai_handler.py +++ b/pr_agent/algo/ai_handler.py @@ -74,9 +74,15 @@ class AiHandler: TryAgain: If there is an attribute error during OpenAI inference. """ try: + deployment_id = self.deployment_id + if get_settings().config.verbosity_level >= 2: + logging.debug( + f"Generating completion with {model}" + f"{(' from deployment ' + deployment_id) if deployment_id else ''}" + ) response = await acompletion( model=model, - deployment_id=self.deployment_id, + deployment_id=deployment_id, messages=[ {"role": "system", "content": system}, {"role": "user", "content": user} diff --git a/pr_agent/algo/pr_processing.py b/pr_agent/algo/pr_processing.py index fae2535a..be4f4d5f 100644 --- a/pr_agent/algo/pr_processing.py +++ b/pr_agent/algo/pr_processing.py @@ -214,7 +214,6 @@ async def retry_with_fallback_models(f: Callable): if not isinstance(fallback_models, list): fallback_models = [m.strip() for m in fallback_models.split(",")] all_models = [model] + fallback_models - # getting all deployments deployment_id = get_settings().get("openai.deployment_id", None) fallback_deployments = get_settings().get("openai.fallback_deployments", []) @@ -230,7 +229,11 @@ async def retry_with_fallback_models(f: Callable): get_settings().set("openai.deployment_id", deployment_id) return await f(model) except Exception as e: - logging.warning(f"Failed to generate prediction with {model}: {traceback.format_exc()}") + logging.warning( + f"Failed to generate prediction with {model}" + f"{(' from deployment ' + deployment_id) if deployment_id else ''}: " + f"{traceback.format_exc()}" + ) if i == len(all_models) - 1: # If it's the last iteration raise # Re-raise the last exception From 7762a672508f3aef10e034bcbf83126f102b5908 Mon Sep 17 00:00:00 2001 From: zmeir Date: Sun, 13 Aug 2023 10:55:44 +0300 Subject: [PATCH 3/9] Fail if not enough fallback deployments --- pr_agent/algo/pr_processing.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pr_agent/algo/pr_processing.py b/pr_agent/algo/pr_processing.py index 140046a3..db311dac 100644 --- a/pr_agent/algo/pr_processing.py +++ b/pr_agent/algo/pr_processing.py @@ -221,6 +221,9 @@ async def retry_with_fallback_models(f: Callable): fallback_deployments = [d.strip() for d in fallback_deployments.split(",")] if fallback_deployments: all_deployments = [deployment_id] + fallback_deployments + if len(fallback_deployments) < len(fallback_models): + raise ValueError(f"The number of fallback deployments ({len(all_deployments)}) " + f"is less than the number of fallback models ({len(all_models)})") else: all_deployments = [deployment_id] * len(all_models) # try each (model, deployment_id) pair until one is successful, otherwise raise exception From edcf89a45664fa4e407f529d4f9aa58972fc7e91 Mon Sep 17 00:00:00 2001 From: zmeir Date: Sun, 13 Aug 2023 10:56:16 +0300 Subject: [PATCH 4/9] Improve comment --- pr_agent/settings/.secrets_template.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr_agent/settings/.secrets_template.toml b/pr_agent/settings/.secrets_template.toml index 25a6562f..0ac75519 100644 --- a/pr_agent/settings/.secrets_template.toml +++ b/pr_agent/settings/.secrets_template.toml @@ -14,7 +14,7 @@ key = "" # Acquire through https://platform.openai.com #api_version = '2023-05-15' # Check Azure documentation for the current API version #api_base = "" # The base URL for your Azure OpenAI resource. e.g. "https://.openai.azure.com" #deployment_id = "" # The deployment name you chose when you deployed the engine -#fallback_deployments = [] # Match your fallback models from configuration.toml with the appropriate deployment_id +#fallback_deployments = [] # For each fallback model specified in configuration.toml in the [config] section, specify the appropriate deployment_id [anthropic] key = "" # Optional, uncomment if you want to use Anthropic. Acquire through https://www.anthropic.com/ From 6ca0655517dbb72e02d97494d912a220362f1210 Mon Sep 17 00:00:00 2001 From: zmeir Date: Sun, 13 Aug 2023 11:03:10 +0300 Subject: [PATCH 5/9] Extracted to helper functions --- pr_agent/algo/pr_processing.py | 44 ++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/pr_agent/algo/pr_processing.py b/pr_agent/algo/pr_processing.py index db311dac..adab9506 100644 --- a/pr_agent/algo/pr_processing.py +++ b/pr_agent/algo/pr_processing.py @@ -208,24 +208,8 @@ def pr_generate_compressed_diff(top_langs: list, token_handler: TokenHandler, mo async def retry_with_fallback_models(f: Callable): - # getting all models - model = get_settings().config.model - fallback_models = get_settings().config.fallback_models - if not isinstance(fallback_models, list): - fallback_models = [m.strip() for m in fallback_models.split(",")] - all_models = [model] + fallback_models - # getting all deployments - deployment_id = get_settings().get("openai.deployment_id", None) - fallback_deployments = get_settings().get("openai.fallback_deployments", []) - if not isinstance(fallback_deployments, list) and fallback_deployments: - fallback_deployments = [d.strip() for d in fallback_deployments.split(",")] - if fallback_deployments: - all_deployments = [deployment_id] + fallback_deployments - if len(fallback_deployments) < len(fallback_models): - raise ValueError(f"The number of fallback deployments ({len(all_deployments)}) " - f"is less than the number of fallback models ({len(all_models)})") - else: - all_deployments = [deployment_id] * len(all_models) + all_models = _get_all_models() + all_deployments = _get_all_deployments(all_models) # try each (model, deployment_id) pair until one is successful, otherwise raise exception for i, (model, deployment_id) in enumerate(zip(all_models, all_deployments)): try: @@ -241,6 +225,30 @@ async def retry_with_fallback_models(f: Callable): raise # Re-raise the last exception +def _get_all_models() -> List[str]: + model = get_settings().config.model + fallback_models = get_settings().config.fallback_models + if not isinstance(fallback_models, list): + fallback_models = [m.strip() for m in fallback_models.split(",")] + all_models = [model] + fallback_models + return all_models + + +def _get_all_deployments(all_models: List[str]) -> List[str]: + deployment_id = get_settings().get("openai.deployment_id", None) + fallback_deployments = get_settings().get("openai.fallback_deployments", []) + if not isinstance(fallback_deployments, list) and fallback_deployments: + fallback_deployments = [d.strip() for d in fallback_deployments.split(",")] + if fallback_deployments: + all_deployments = [deployment_id] + fallback_deployments + if len(all_deployments) < len(all_models): + raise ValueError(f"The number of deployments ({len(all_deployments)}) " + f"is less than the number of models ({len(all_models)})") + else: + all_deployments = [deployment_id] * len(all_models) + return all_deployments + + def find_line_number_of_relevant_line_in_file(diff_files: List[FilePatchInfo], relevant_file: str, relevant_line_in_file: str) -> Tuple[int, int]: From b7b650eb05ba8a4e984eb3caf9e2899099d94d41 Mon Sep 17 00:00:00 2001 From: Zohar Meir <33152084+zmeir@users.noreply.github.com> Date: Sun, 13 Aug 2023 11:32:02 +0300 Subject: [PATCH 6/9] Remove extra blank line in help message --- pr_agent/servers/help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pr_agent/servers/help.py b/pr_agent/servers/help.py index 1c8e1c3f..1ee7fc4d 100644 --- a/pr_agent/servers/help.py +++ b/pr_agent/servers/help.py @@ -2,7 +2,7 @@ commands_text = "> **/review [-i]**: Request a review of your Pull Request. For "considers changes since the last review, include the '-i' option.\n" \ "> **/describe**: Modify the PR title and description based on the contents of the PR.\n" \ "> **/improve**: Suggest improvements to the code in the PR. \n" \ - "> **/ask \\**: Pose a question about the PR.\n\n" \ + "> **/ask \\**: Pose a question about the PR.\n" \ "> **/update_changelog**: Update the changelog based on the PR's contents.\n\n" \ ">To edit any configuration parameter from **configuration.toml**, add --config_path=new_value\n" \ ">For example: /review --pr_reviewer.extra_instructions=\"focus on the file: ...\" \n" \ From ce29d9eb49ee7b99cebc143f1a572d918a9a934d Mon Sep 17 00:00:00 2001 From: Tim Perkins Date: Sun, 13 Aug 2023 07:28:05 -0400 Subject: [PATCH 7/9] Contents read-only permission needed by /review -i --- INSTALL.md | 1 + 1 file changed, 1 insertion(+) diff --git a/INSTALL.md b/INSTALL.md index 76a00671..4589e30f 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -128,6 +128,7 @@ Allowing you to automate the review process on your private or public repositori - Pull requests: Read & write - Issue comment: Read & write - Metadata: Read-only + - Contents: Read-only - Set the following events: - Issue comment - Pull request From cca951d787917c1c8d41433ff7ba6c26be5ecdac Mon Sep 17 00:00:00 2001 From: sarbjitgrewal Date: Mon, 14 Aug 2023 14:30:30 +0530 Subject: [PATCH 8/9] fix bitbucket ask issue --- pr_agent/git_providers/bitbucket_provider.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pr_agent/git_providers/bitbucket_provider.py b/pr_agent/git_providers/bitbucket_provider.py index 07b92295..52429c48 100644 --- a/pr_agent/git_providers/bitbucket_provider.py +++ b/pr_agent/git_providers/bitbucket_provider.py @@ -26,6 +26,13 @@ class BitbucketProvider: if pr_url: self.set_pr(pr_url) + def get_repo_settings(self): + try: + contents = self.repo_obj.get_contents(".pr_agent.toml", ref=self.pr.head.sha).decoded_content + return contents + except Exception: + return "" + def is_supported(self, capability: str) -> bool: if capability in ['get_issue_comments', 'create_inline_comment', 'publish_inline_comments', 'get_labels']: return False From 1db53ae1adcfb4a4e483974d15d6a34803fcdea6 Mon Sep 17 00:00:00 2001 From: sarbjitgrewal Date: Mon, 14 Aug 2023 14:45:25 +0530 Subject: [PATCH 9/9] update readme file --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ec00ec2..d2c4e171 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ CodiumAI `PR-Agent` is an open-source tool aiming to help developers review pull |-------|---------------------------------------------|:------:|:------:|:---------:| | TOOLS | Review | :white_check_mark: | :white_check_mark: | :white_check_mark: | | | ⮑ Inline review | :white_check_mark: | :white_check_mark: | | -| | Ask | :white_check_mark: | :white_check_mark: | | +| | Ask | :white_check_mark: | :white_check_mark: | :white_check_mark: | | Auto-Description | :white_check_mark: | :white_check_mark: | | | | Improve Code | :white_check_mark: | :white_check_mark: | | | | Reflect and Review | :white_check_mark: | | |