Merge remote-tracking branch 'origin/main' into ok/gitlab_webhook

This commit is contained in:
Ori Kotek
2023-09-05 18:15:51 +03:00
15 changed files with 885 additions and 219 deletions

View File

@ -5,6 +5,8 @@ from pr_agent.git_providers.github_provider import GithubProvider
from pr_agent.git_providers.gitlab_provider import GitLabProvider
from pr_agent.git_providers.local_git_provider import LocalGitProvider
from pr_agent.git_providers.azuredevops_provider import AzureDevopsProvider
from pr_agent.git_providers.gerrit_provider import GerritProvider
_GIT_PROVIDERS = {
'github': GithubProvider,
@ -12,7 +14,8 @@ _GIT_PROVIDERS = {
'bitbucket': BitbucketProvider,
'azure': AzureDevopsProvider,
'codecommit': CodeCommitProvider,
'local' : LocalGitProvider
'local' : LocalGitProvider,
'gerrit': GerritProvider,
}
def get_git_provider():

View File

@ -90,7 +90,11 @@ class CodeCommitClient:
):
differences.extend(page.get("differences", []))
except botocore.exceptions.ClientError as e:
raise ValueError(f"Failed to retrieve differences from CodeCommit PR #{self.pr_num}") from e
if e.response["Error"]["Code"] == 'RepositoryDoesNotExistException':
raise ValueError(f"CodeCommit cannot retrieve differences: Repository does not exist: {repo_name}") from e
raise ValueError(f"CodeCommit cannot retrieve differences for {source_commit}..{destination_commit}") from e
except Exception as e:
raise ValueError(f"CodeCommit cannot retrieve differences for {source_commit}..{destination_commit}") from e
output = []
for json in differences:
@ -122,6 +126,8 @@ class CodeCommitClient:
try:
response = self.boto_client.get_file(repositoryName=repo_name, commitSpecifier=sha_hash, filePath=file_path)
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == 'RepositoryDoesNotExistException':
raise ValueError(f"CodeCommit cannot retrieve PR: Repository does not exist: {repo_name}") from e
# if the file does not exist, but is flagged as optional, then return an empty string
if optional and e.response["Error"]["Code"] == 'FileDoesNotExistException':
return ""
@ -133,11 +139,12 @@ class CodeCommitClient:
return response.get("fileContent", "")
def get_pr(self, pr_number: int):
def get_pr(self, repo_name: str, pr_number: int):
"""
Get a information about a CodeCommit PR.
Args:
- repo_name: Name of the repository
- pr_number: The PR number you are requesting
Returns:
@ -155,6 +162,8 @@ class CodeCommitClient:
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == 'PullRequestDoesNotExistException':
raise ValueError(f"CodeCommit cannot retrieve PR: PR number does not exist: {pr_number}") from e
if e.response["Error"]["Code"] == 'RepositoryDoesNotExistException':
raise ValueError(f"CodeCommit cannot retrieve PR: Repository does not exist: {repo_name}") from e
raise ValueError(f"CodeCommit cannot retrieve PR: {pr_number}: boto client error") from e
except Exception as e:
raise ValueError(f"CodeCommit cannot retrieve PR: {pr_number}") from e
@ -201,7 +210,7 @@ class CodeCommitClient:
except Exception as e:
raise ValueError(f"Error calling publish_description") from e
def publish_comment(self, repo_name: str, pr_number: int, destination_commit: str, source_commit: str, comment: str):
def publish_comment(self, repo_name: str, pr_number: int, destination_commit: str, source_commit: str, comment: str, annotation_file: str = None, annotation_line: int = None):
"""
Publish a comment to a pull request
@ -210,7 +219,13 @@ class CodeCommitClient:
- pr_number: number of the pull request
- destination_commit: The commit hash you want to merge into (the "before" hash) (usually on the main or master branch)
- source_commit: The commit hash of the code you are adding (the "after" branch)
- pr_comment: comment
- comment: The comment you want to publish
- annotation_file: The file you want to annotate (optional)
- annotation_line: The line number you want to annotate (optional)
Comment annotations for CodeCommit are different than GitHub.
CodeCommit only designates the starting line number for the comment.
It does not support the ending line number to highlight a range of lines.
Returns:
- None
@ -223,13 +238,30 @@ class CodeCommitClient:
self._connect_boto_client()
try:
self.boto_client.post_comment_for_pull_request(
pullRequestId=str(pr_number),
repositoryName=repo_name,
beforeCommitId=destination_commit,
afterCommitId=source_commit,
content=comment,
)
# If the comment has code annotations,
# then set the file path and line number in the location dictionary
if annotation_file and annotation_line:
self.boto_client.post_comment_for_pull_request(
pullRequestId=str(pr_number),
repositoryName=repo_name,
beforeCommitId=destination_commit,
afterCommitId=source_commit,
content=comment,
location={
"filePath": annotation_file,
"filePosition": annotation_line,
"relativeFileVersion": "AFTER",
},
)
else:
# The comment does not have code annotations
self.boto_client.post_comment_for_pull_request(
pullRequestId=str(pr_number),
repositoryName=repo_name,
beforeCommitId=destination_commit,
afterCommitId=source_commit,
content=comment,
)
except botocore.exceptions.ClientError as e:
if e.response["Error"]["Code"] == 'RepositoryDoesNotExistException':
raise ValueError(f"Repository does not exist: {repo_name}") from e

View File

@ -180,10 +180,37 @@ class CodeCommitProvider(GitProvider):
comment=pr_comment,
)
except Exception as e:
raise ValueError(f"CodeCommit Cannot post comment for PR: {self.pr_num}") from e
raise ValueError(f"CodeCommit Cannot publish comment for PR: {self.pr_num}") from e
def publish_code_suggestions(self, code_suggestions: list) -> bool:
return [""] # not implemented yet
counter = 1
for suggestion in code_suggestions:
# Verify that each suggestion has the required keys
if not all(key in suggestion for key in ["body", "relevant_file", "relevant_lines_start"]):
logging.warning(f"Skipping code suggestion #{counter}: Each suggestion must have 'body', 'relevant_file', 'relevant_lines_start' keys")
continue
# Publish the code suggestion to CodeCommit
try:
logging.debug(f"Code Suggestion #{counter} in file: {suggestion['relevant_file']}: {suggestion['relevant_lines_start']}")
self.codecommit_client.publish_comment(
repo_name=self.repo_name,
pr_number=self.pr_num,
destination_commit=self.pr.destination_commit,
source_commit=self.pr.source_commit,
comment=suggestion["body"],
annotation_file=suggestion["relevant_file"],
annotation_line=suggestion["relevant_lines_start"],
)
except Exception as e:
raise ValueError(f"CodeCommit Cannot publish code suggestions for PR: {self.pr_num}") from e
counter += 1
# The calling function passes in a list of code suggestions, and this function publishes each suggestion one at a time.
# If we were to return False here, the calling function will attempt to publish the same list of code suggestions again, one at a time.
# Since this function publishes the suggestions one at a time anyway, we always return True here to avoid the retry.
return True
def publish_labels(self, labels):
return [""] # not implemented yet
@ -195,6 +222,7 @@ class CodeCommitProvider(GitProvider):
return "" # not implemented yet
def publish_inline_comment(self, body: str, relevant_file: str, relevant_line_in_file: str):
# https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/codecommit/client/post_comment_for_compared_commit.html
raise NotImplementedError("CodeCommit provider does not support publishing inline comments yet")
def create_inline_comment(self, body: str, relevant_file: str, relevant_line_in_file: str):
@ -255,9 +283,11 @@ class CodeCommitProvider(GitProvider):
return self.codecommit_client.get_file(self.repo_name, settings_filename, self.pr.source_commit, optional=True)
def add_eyes_reaction(self, issue_comment_id: int) -> Optional[int]:
logging.info("CodeCommit provider does not support eyes reaction yet")
return True
def remove_reaction(self, issue_comment_id: int, reaction_id: int) -> bool:
logging.info("CodeCommit provider does not support removing reactions yet")
return True
@staticmethod
@ -315,16 +345,16 @@ class CodeCommitProvider(GitProvider):
return re.match(r"^[a-z]{2}-(gov-)?[a-z]+-\d\.console\.aws\.amazon\.com$", hostname) is not None
def _get_pr(self):
response = self.codecommit_client.get_pr(self.pr_num)
response = self.codecommit_client.get_pr(self.repo_name, self.pr_num)
if len(response.targets) == 0:
raise ValueError(f"No files found in CodeCommit PR: {self.pr_num}")
# TODO: implement support for multiple commits in one CodeCommit PR
# for now, we are only using the first commit in the PR
# TODO: implement support for multiple targets in one CodeCommit PR
# for now, we are only using the first target in the PR
if len(response.targets) > 1:
logging.warning(
"Multiple commits in one PR is not supported for CodeCommit yet. Continuing, using the first commit only..."
"Multiple targets in one PR is not supported for CodeCommit yet. Continuing, using the first target only..."
)
# Return our object that mimics PullRequest class from the PyGithub library

View File

@ -0,0 +1,393 @@
import json
import logging
import os
import pathlib
import shutil
import subprocess
import uuid
from collections import Counter, namedtuple
from pathlib import Path
from tempfile import mkdtemp, NamedTemporaryFile
import requests
import urllib3.util
from git import Repo
from pr_agent.config_loader import get_settings
from pr_agent.git_providers.git_provider import GitProvider, FilePatchInfo, \
EDIT_TYPE
from pr_agent.git_providers.local_git_provider import PullRequestMimic
logger = logging.getLogger(__name__)
def _call(*command, **kwargs) -> (int, str, str):
res = subprocess.run(
command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=True,
**kwargs,
)
return res.stdout.decode()
def clone(url, directory):
logger.info("Cloning %s to %s", url, directory)
stdout = _call('git', 'clone', "--depth", "1", url, directory)
logger.info(stdout)
def fetch(url, refspec, cwd):
logger.info("Fetching %s %s", url, refspec)
stdout = _call(
'git', 'fetch', '--depth', '2', url, refspec,
cwd=cwd
)
logger.info(stdout)
def checkout(cwd):
logger.info("Checking out")
stdout = _call('git', 'checkout', "FETCH_HEAD", cwd=cwd)
logger.info(stdout)
def show(*args, cwd=None):
logger.info("Show")
return _call('git', 'show', *args, cwd=cwd)
def diff(*args, cwd=None):
logger.info("Diff")
patch = _call('git', 'diff', *args, cwd=cwd)
if not patch:
logger.warning("No changes found")
return
return patch
def reset_local_changes(cwd):
logger.info("Reset local changes")
_call('git', 'checkout', "--force", cwd=cwd)
def add_comment(url: urllib3.util.Url, refspec, message):
*_, patchset, changenum = refspec.rsplit("/")
message = "'" + message.replace("'", "'\"'\"'") + "'"
return _call(
"ssh",
"-p", str(url.port),
f"{url.auth}@{url.host}",
"gerrit", "review",
"--message", message,
# "--code-review", score,
f"{patchset},{changenum}",
)
def list_comments(url: urllib3.util.Url, refspec):
*_, patchset, _ = refspec.rsplit("/")
stdout = _call(
"ssh",
"-p", str(url.port),
f"{url.auth}@{url.host}",
"gerrit", "query",
"--comments",
"--current-patch-set", patchset,
"--format", "JSON",
)
change_set, *_ = stdout.splitlines()
return json.loads(change_set)["currentPatchSet"]["comments"]
def prepare_repo(url: urllib3.util.Url, project, refspec):
repo_url = (f"{url.scheme}://{url.auth}@{url.host}:{url.port}/{project}")
directory = pathlib.Path(mkdtemp())
clone(repo_url, directory),
fetch(repo_url, refspec, cwd=directory)
checkout(cwd=directory)
return directory
def adopt_to_gerrit_message(message):
lines = message.splitlines()
buf = []
for line in lines:
line = line.replace("*", "").replace("``", "`")
line = line.strip()
if line.startswith('#'):
buf.append("\n" +
line.replace('#', '').removesuffix(":").strip() +
":")
continue
elif line.startswith('-'):
buf.append(line.removeprefix('-').strip())
continue
else:
buf.append(line)
return "\n".join(buf).strip()
def add_suggestion(src_filename, context: str, start, end: int):
with (
NamedTemporaryFile("w", delete=False) as tmp,
open(src_filename, "r") as src
):
lines = src.readlines()
tmp.writelines(lines[:start - 1])
if context:
tmp.write(context)
tmp.writelines(lines[end:])
shutil.copy(tmp.name, src_filename)
os.remove(tmp.name)
def upload_patch(patch, path):
patch_server_endpoint = get_settings().get(
'gerrit.patch_server_endpoint')
patch_server_token = get_settings().get(
'gerrit.patch_server_token')
response = requests.post(
patch_server_endpoint,
json={
"content": patch,
"path": path,
},
headers={
"Content-Type": "application/json",
"Authorization": f"Bearer {patch_server_token}",
}
)
response.raise_for_status()
patch_server_endpoint = patch_server_endpoint.rstrip("/")
return patch_server_endpoint + "/" + path
class GerritProvider(GitProvider):
def __init__(self, key: str, incremental=False):
self.project, self.refspec = key.split(':')
assert self.project, "Project name is required"
assert self.refspec, "Refspec is required"
base_url = get_settings().get('gerrit.url')
assert base_url, "Gerrit URL is required"
user = get_settings().get('gerrit.user')
assert user, "Gerrit user is required"
parsed = urllib3.util.parse_url(base_url)
self.parsed_url = urllib3.util.parse_url(
f"{parsed.scheme}://{user}@{parsed.host}:{parsed.port}"
)
self.repo_path = prepare_repo(
self.parsed_url, self.project, self.refspec
)
self.repo = Repo(self.repo_path)
assert self.repo
self.pr = PullRequestMimic(self.get_pr_title(), self.get_diff_files())
def get_pr_title(self):
"""
Substitutes the branch-name as the PR-mimic title.
"""
return self.repo.branches[0].name
def get_issue_comments(self):
comments = list_comments(self.parsed_url, self.refspec)
Comments = namedtuple('Comments', ['reversed'])
Comment = namedtuple('Comment', ['body'])
return Comments([Comment(c['message']) for c in reversed(comments)])
def get_labels(self):
raise NotImplementedError(
'Getting labels is not implemented for the gerrit provider')
def add_eyes_reaction(self, issue_comment_id: int):
raise NotImplementedError(
'Adding reactions is not implemented for the gerrit provider')
def remove_reaction(self, issue_comment_id: int, reaction_id: int):
raise NotImplementedError(
'Removing reactions is not implemented for the gerrit provider')
def get_commit_messages(self):
return [self.repo.head.commit.message]
def get_repo_settings(self):
"""
TODO: Implement support of .pr_agent.toml
"""
return ""
def get_diff_files(self) -> list[FilePatchInfo]:
diffs = self.repo.head.commit.diff(
self.repo.head.commit.parents[0], # previous commit
create_patch=True,
R=True
)
diff_files = []
for diff_item in diffs:
if diff_item.a_blob is not None:
original_file_content_str = (
diff_item.a_blob.data_stream.read().decode('utf-8')
)
else:
original_file_content_str = "" # empty file
if diff_item.b_blob is not None:
new_file_content_str = diff_item.b_blob.data_stream.read(). \
decode('utf-8')
else:
new_file_content_str = "" # empty file
edit_type = EDIT_TYPE.MODIFIED
if diff_item.new_file:
edit_type = EDIT_TYPE.ADDED
elif diff_item.deleted_file:
edit_type = EDIT_TYPE.DELETED
elif diff_item.renamed_file:
edit_type = EDIT_TYPE.RENAMED
diff_files.append(
FilePatchInfo(
original_file_content_str,
new_file_content_str,
diff_item.diff.decode('utf-8'),
diff_item.b_path,
edit_type=edit_type,
old_filename=None
if diff_item.a_path == diff_item.b_path
else diff_item.a_path
)
)
self.diff_files = diff_files
return diff_files
def get_files(self):
diff_index = self.repo.head.commit.diff(
self.repo.head.commit.parents[0], # previous commit
R=True
)
# Get the list of changed files
diff_files = [item.a_path for item in diff_index]
return diff_files
def get_languages(self):
"""
Calculate percentage of languages in repository. Used for hunk
prioritisation.
"""
# Get all files in repository
filepaths = [Path(item.path) for item in
self.repo.tree().traverse() if item.type == 'blob']
# Identify language by file extension and count
lang_count = Counter(
ext.lstrip('.') for filepath in filepaths for ext in
[filepath.suffix.lower()])
# Convert counts to percentages
total_files = len(filepaths)
lang_percentage = {lang: count / total_files * 100 for lang, count
in lang_count.items()}
return lang_percentage
def get_pr_description_full(self):
return self.repo.head.commit.message
def get_user_id(self):
return self.repo.head.commit.author.email
def is_supported(self, capability: str) -> bool:
if capability in [
# 'get_issue_comments',
'create_inline_comment',
'publish_inline_comments',
'get_labels'
]:
return False
return True
def split_suggestion(self, msg) -> tuple[str, str]:
is_code_context = False
description = []
context = []
for line in msg.splitlines():
if line.startswith('```suggestion'):
is_code_context = True
continue
if line.startswith('```'):
is_code_context = False
continue
if is_code_context:
context.append(line)
else:
description.append(
line.replace('*', '')
)
return (
'\n'.join(description),
'\n'.join(context) + '\n' if context else ''
)
def publish_code_suggestions(self, code_suggestions: list):
msg = []
for suggestion in code_suggestions:
description, code = self.split_suggestion(suggestion['body'])
add_suggestion(
pathlib.Path(self.repo_path) / suggestion["relevant_file"],
code,
suggestion["relevant_lines_start"],
suggestion["relevant_lines_end"],
)
patch = diff(cwd=self.repo_path)
patch_id = uuid.uuid4().hex[0:4]
path = "/".join(["codium-ai", self.refspec, patch_id])
full_path = upload_patch(patch, path)
reset_local_changes(self.repo_path)
msg.append(f'* {description}\n{full_path}')
if msg:
add_comment(self.parsed_url, self.refspec, "\n".join(msg))
return True
def publish_comment(self, pr_comment: str, is_temporary: bool = False):
if not is_temporary:
msg = adopt_to_gerrit_message(pr_comment)
add_comment(self.parsed_url, self.refspec, msg)
def publish_description(self, pr_title: str, pr_body: str):
msg = adopt_to_gerrit_message(pr_body)
add_comment(self.parsed_url, self.refspec, pr_title + '\n' + msg)
def publish_inline_comments(self, comments: list[dict]):
raise NotImplementedError(
'Publishing inline comments is not implemented for the gerrit '
'provider')
def publish_inline_comment(self, body: str, relevant_file: str,
relevant_line_in_file: str):
raise NotImplementedError(
'Publishing inline comments is not implemented for the gerrit '
'provider')
def create_inline_comment(self, body: str, relevant_file: str,
relevant_line_in_file: str):
raise NotImplementedError(
'Creating inline comments is not implemented for the gerrit '
'provider')
def publish_labels(self, labels):
# Not applicable to the local git provider,
# but required by the interface
pass
def remove_initial_comment(self):
# remove repo, cloned in previous steps
# shutil.rmtree(self.repo_path)
pass
def get_pr_branch(self):
return self.repo.head

View File

@ -161,7 +161,6 @@ def get_main_pr_language(languages, files) -> str:
most_common_extension == 'scala' and top_language == 'scala' or \
most_common_extension == 'kt' and top_language == 'kotlin' or \
most_common_extension == 'pl' and top_language == 'perl' or \
most_common_extension == 'swift' and top_language == 'swift' or \
most_common_extension == top_language:
main_language_str = top_language

View File

@ -0,0 +1,78 @@
import copy
import logging
import sys
from enum import Enum
from json import JSONDecodeError
import uvicorn
from fastapi import APIRouter, FastAPI, HTTPException
from pydantic import BaseModel
from starlette.middleware import Middleware
from starlette_context import context
from starlette_context.middleware import RawContextMiddleware
from pr_agent.agent.pr_agent import PRAgent
from pr_agent.config_loader import global_settings, get_settings
logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
router = APIRouter()
class Action(str, Enum):
review = "review"
describe = "describe"
ask = "ask"
improve = "improve"
reflect = "reflect"
answer = "answer"
class Item(BaseModel):
refspec: str
project: str
msg: str
@router.post("/api/v1/gerrit/{action}")
async def handle_gerrit_request(action: Action, item: Item):
logging.debug("Received a Gerrit request")
context["settings"] = copy.deepcopy(global_settings)
if action == Action.ask:
if not item.msg:
return HTTPException(
status_code=400,
detail="msg is required for ask command"
)
await PRAgent().handle_request(
f"{item.project}:{item.refspec}",
f"/{item.msg.strip()}"
)
async def get_body(request):
try:
body = await request.json()
except JSONDecodeError as e:
logging.error("Error parsing request body", e)
return {}
return body
@router.get("/")
async def root():
return {"status": "ok"}
def start():
# to prevent adding help messages with the output
get_settings().set("CONFIG.CLI_MODE", True)
middleware = [Middleware(RawContextMiddleware)]
app = FastAPI(middleware=middleware)
app.include_router(router)
uvicorn.run(app, host="0.0.0.0", port=3000)
if __name__ == '__main__':
start()

View File

@ -84,4 +84,14 @@ polling_interval_seconds = 30
[local]
# LocalGitProvider settings - uncomment to use paths other than default
# description_path= "path/to/description.md"
# review_path= "path/to/review.md"
# review_path= "path/to/review.md"
[gerrit]
# endpoint to the gerrit service
# url = "ssh://gerrit.example.com:29418"
# user for gerrit authentication
# user = "ai-reviewer"
# patch server where patches will be saved
# patch_server_endpoint = "http://127.0.0.1:5000/patch"
# token to authenticate in the patch server
# patch_server_token = ""

View File

@ -68,12 +68,17 @@ Code suggestions:
type: string
description: |-
a code snippet showing the relevant code lines from a '__new hunk__' section.
It must be continuous, correctly formatted and indented, and without line numbers.
relevant lines:
type: string
It must be contiguous, correctly formatted and indented, and without line numbers.
relevant lines start:
type: integer
description: |-
the relevant lines from a '__new hunk__' section, in the format of 'start_line-end_line'.
For example: '10-15'. They should be derived from the hunk line numbers, and correspond to the 'existing code' snippet above.
The relevant line number from a '__new hunk__' section where the suggestion starts (inclusive).
Should be derived from the hunk line numbers, and correspond to the 'existing code' snippet above.
relevant lines end:
type: integer
description: |-
The relevant line number from a '__new hunk__' section where the suggestion ends (inclusive).
Should be derived from the hunk line numbers, and correspond to the 'existing code' snippet above.
improved code:
type: string
description: |-
@ -90,7 +95,8 @@ Code suggestions:
Add a docstring to func1()
existing code: |-
def func1():
relevant lines: '12-12'
relevant lines start: 12
relevant lines end: 12
improved code: |-
...
```

View File

@ -113,11 +113,8 @@ class PRCodeSuggestions:
if get_settings().config.verbosity_level >= 2:
logging.info(f"suggestion: {d}")
relevant_file = d['relevant file'].strip()
relevant_lines_str = d['relevant lines'].strip()
if ',' in relevant_lines_str: # handling 'relevant lines': '181, 190' or '178-184, 188-194'
relevant_lines_str = relevant_lines_str.split(',')[0]
relevant_lines_start = int(relevant_lines_str.split('-')[0]) # absolute position
relevant_lines_end = int(relevant_lines_str.split('-')[-1])
relevant_lines_start = int(d['relevant lines start']) # absolute position
relevant_lines_end = int(d['relevant lines end'])
content = d['suggestion content']
new_code_snippet = d['improved code']