From 02e0f958e784b6a8558907059a9aa8bd91f0fbc1 Mon Sep 17 00:00:00 2001 From: Ori Kotek Date: Thu, 31 Aug 2023 14:56:45 +0300 Subject: [PATCH 1/5] Add Gitlab webhook secret --- pr_agent/servers/gitlab_webhook.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pr_agent/servers/gitlab_webhook.py b/pr_agent/servers/gitlab_webhook.py index c9b623f7..6e48054f 100644 --- a/pr_agent/servers/gitlab_webhook.py +++ b/pr_agent/servers/gitlab_webhook.py @@ -15,6 +15,10 @@ router = APIRouter() @router.post("/webhook") async def gitlab_webhook(background_tasks: BackgroundTasks, request: Request): + if get_settings().get("GITLAB.SHARED_SECRET"): + secret = get_settings().get("GITLAB.SHARED_SECRET") + if not request.headers.get("X-Gitlab-Token") == secret: + return JSONResponse(status_code=status.HTTP_401_UNAUTHORIZED, content=jsonable_encoder({"message": "unauthorized"})) data = await request.json() if data.get('object_kind') == 'merge_request' and data['object_attributes'].get('action') in ['open', 'reopen']: logging.info(f"A merge request has been opened: {data['object_attributes'].get('title')}") From 4c6595148bc84fe4ddc0fe51241dfde205420e95 Mon Sep 17 00:00:00 2001 From: Ori Kotek Date: Thu, 31 Aug 2023 17:03:58 +0300 Subject: [PATCH 2/5] Add Gitlab webhook secret --- pr_agent/servers/gitlab_webhook.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pr_agent/servers/gitlab_webhook.py b/pr_agent/servers/gitlab_webhook.py index 6e48054f..cfc3a8a0 100644 --- a/pr_agent/servers/gitlab_webhook.py +++ b/pr_agent/servers/gitlab_webhook.py @@ -8,10 +8,13 @@ from starlette.background import BackgroundTasks from pr_agent.agent.pr_agent import PRAgent from pr_agent.config_loader import get_settings +from pr_agent.secret_providers import get_secret_provider app = FastAPI() router = APIRouter() +if get_settings().config.secret_provider: + secret_provider = get_secret_provider() @router.post("/webhook") async def gitlab_webhook(background_tasks: BackgroundTasks, request: Request): @@ -19,6 +22,9 @@ async def gitlab_webhook(background_tasks: BackgroundTasks, request: Request): secret = get_settings().get("GITLAB.SHARED_SECRET") if not request.headers.get("X-Gitlab-Token") == secret: return JSONResponse(status_code=status.HTTP_401_UNAUTHORIZED, content=jsonable_encoder({"message": "unauthorized"})) + gitlab_token = get_settings().get("GITLAB.PERSONAL_ACCESS_TOKEN", None) + if not gitlab_token: + return JSONResponse(status_code=status.HTTP_401_UNAUTHORIZED, content=jsonable_encoder({"message": "unauthorized"})) data = await request.json() if data.get('object_kind') == 'merge_request' and data['object_attributes'].get('action') in ['open', 'reopen']: logging.info(f"A merge request has been opened: {data['object_attributes'].get('title')}") @@ -36,9 +42,6 @@ def start(): gitlab_url = get_settings().get("GITLAB.URL", None) if not gitlab_url: raise ValueError("GITLAB.URL is not set") - gitlab_token = get_settings().get("GITLAB.PERSONAL_ACCESS_TOKEN", None) - if not gitlab_token: - raise ValueError("GITLAB.PERSONAL_ACCESS_TOKEN is not set") get_settings().config.git_provider = "gitlab" app = FastAPI() From 56e9493f7a244aa9fa734fdfec9adac5b747b5ec Mon Sep 17 00:00:00 2001 From: Ori Kotek Date: Mon, 4 Sep 2023 15:29:21 +0300 Subject: [PATCH 3/5] Add Gitlab webhook secret --- pr_agent/servers/gitlab_webhook.py | 31 +++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/pr_agent/servers/gitlab_webhook.py b/pr_agent/servers/gitlab_webhook.py index cfc3a8a0..565405b4 100644 --- a/pr_agent/servers/gitlab_webhook.py +++ b/pr_agent/servers/gitlab_webhook.py @@ -1,3 +1,5 @@ +import copy +import json import logging import uvicorn @@ -5,23 +7,38 @@ from fastapi import APIRouter, FastAPI, Request, status from fastapi.encoders import jsonable_encoder from fastapi.responses import JSONResponse from starlette.background import BackgroundTasks +from starlette.middleware import Middleware +from starlette_context import context +from starlette_context.middleware import RawContextMiddleware from pr_agent.agent.pr_agent import PRAgent -from pr_agent.config_loader import get_settings +from pr_agent.config_loader import get_settings, global_settings from pr_agent.secret_providers import get_secret_provider -app = FastAPI() router = APIRouter() -if get_settings().config.secret_provider: - secret_provider = get_secret_provider() +secret_provider = get_secret_provider() if get_settings().get("CONFIG.SECRET_PROVIDER") else None + @router.post("/webhook") async def gitlab_webhook(background_tasks: BackgroundTasks, request: Request): - if get_settings().get("GITLAB.SHARED_SECRET"): + if request.headers.get("X-Gitlab-Token") and secret_provider: + request_token = request.headers.get("X-Gitlab-Token") + secret = secret_provider.get_secret(request_token) + try: + secret_dict = json.loads(secret) + gitlab_token = secret_dict["gitlab_token"] + context["settings"] = copy.deepcopy(global_settings) + context["settings"].gitlab.personal_access_token = gitlab_token + except Exception as e: + logging.error(f"Failed to validate secret {request_token}: {e}") + return JSONResponse(status_code=status.HTTP_401_UNAUTHORIZED, content=jsonable_encoder({"message": "unauthorized"})) + elif get_settings().get("GITLAB.SHARED_SECRET"): secret = get_settings().get("GITLAB.SHARED_SECRET") if not request.headers.get("X-Gitlab-Token") == secret: return JSONResponse(status_code=status.HTTP_401_UNAUTHORIZED, content=jsonable_encoder({"message": "unauthorized"})) + else: + return JSONResponse(status_code=status.HTTP_401_UNAUTHORIZED, content=jsonable_encoder({"message": "unauthorized"})) gitlab_token = get_settings().get("GITLAB.PERSONAL_ACCESS_TOKEN", None) if not gitlab_token: return JSONResponse(status_code=status.HTTP_401_UNAUTHORIZED, content=jsonable_encoder({"message": "unauthorized"})) @@ -43,8 +60,8 @@ def start(): if not gitlab_url: raise ValueError("GITLAB.URL is not set") get_settings().config.git_provider = "gitlab" - - app = FastAPI() + middleware = [Middleware(RawContextMiddleware)] + app = FastAPI(middleware=middleware) app.include_router(router) uvicorn.run(app, host="0.0.0.0", port=3000) From 140760c517b17c68f2a8bafea26c9dadbd4030bc Mon Sep 17 00:00:00 2001 From: Ori Kotek Date: Mon, 4 Sep 2023 16:39:31 +0300 Subject: [PATCH 4/5] Add Gitlab webhook secret --- docker/Dockerfile | 4 ++++ pr_agent/servers/github_app.py | 2 ++ pr_agent/servers/gitlab_webhook.py | 8 ++++++++ 3 files changed, 14 insertions(+) diff --git a/docker/Dockerfile b/docker/Dockerfile index 4336cacc..951f846c 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -18,6 +18,10 @@ FROM base as github_polling ADD pr_agent pr_agent CMD ["python", "pr_agent/servers/github_polling.py"] +FROM base as gitlab_webhook +ADD pr_agent pr_agent +CMD ["python", "pr_agent/servers/gitlab_webhook.py"] + FROM base as test ADD requirements-dev.txt . RUN pip install -r requirements-dev.txt && rm requirements-dev.txt diff --git a/pr_agent/servers/github_app.py b/pr_agent/servers/github_app.py index 10584e54..c9f25124 100644 --- a/pr_agent/servers/github_app.py +++ b/pr_agent/servers/github_app.py @@ -98,6 +98,7 @@ async def handle_request(body: Dict[str, Any], event: str): api_url = body["comment"]["pull_request_url"] else: return {} + logging.info(body) logging.info(f"Handling comment because of event={event} and action={action}") comment_id = body.get("comment", {}).get("id") provider = get_git_provider()(pr_url=api_url) @@ -129,6 +130,7 @@ async def handle_request(body: Dict[str, Any], event: str): args = split_command[1:] other_args = update_settings_from_args(args) new_command = ' '.join([command] + other_args) + logging.info(body) logging.info(f"Performing command: {new_command}") await agent.handle_request(api_url, new_command) diff --git a/pr_agent/servers/gitlab_webhook.py b/pr_agent/servers/gitlab_webhook.py index 565405b4..8321cd60 100644 --- a/pr_agent/servers/gitlab_webhook.py +++ b/pr_agent/servers/gitlab_webhook.py @@ -1,6 +1,7 @@ import copy import json import logging +import sys import uvicorn from fastapi import APIRouter, FastAPI, Request, status @@ -15,6 +16,7 @@ from pr_agent.agent.pr_agent import PRAgent from pr_agent.config_loader import get_settings, global_settings from pr_agent.secret_providers import get_secret_provider +logging.basicConfig(stream=sys.stdout, level=logging.INFO) router = APIRouter() secret_provider = get_secret_provider() if get_settings().get("CONFIG.SECRET_PROVIDER") else None @@ -43,6 +45,7 @@ async def gitlab_webhook(background_tasks: BackgroundTasks, request: Request): if not gitlab_token: return JSONResponse(status_code=status.HTTP_401_UNAUTHORIZED, content=jsonable_encoder({"message": "unauthorized"})) data = await request.json() + logging.info(json.dumps(data)) if data.get('object_kind') == 'merge_request' and data['object_attributes'].get('action') in ['open', 'reopen']: logging.info(f"A merge request has been opened: {data['object_attributes'].get('title')}") url = data['object_attributes'].get('url') @@ -55,6 +58,11 @@ async def gitlab_webhook(background_tasks: BackgroundTasks, request: Request): background_tasks.add_task(PRAgent().handle_request, url, body) return JSONResponse(status_code=status.HTTP_200_OK, content=jsonable_encoder({"message": "success"})) + +@router.get("/") +async def root(): + return {"status": "ok"} + def start(): gitlab_url = get_settings().get("GITLAB.URL", None) if not gitlab_url: From 55b3c3fe5c39ccb95be31e209338a4dcff3b6cc3 Mon Sep 17 00:00:00 2001 From: Ori Kotek Date: Tue, 5 Sep 2023 18:31:29 +0300 Subject: [PATCH 5/5] Add Gitlab webhook installation documentation --- INSTALL.md | 21 +++++++++++++++++++++ README.md | 1 + 2 files changed, 22 insertions(+) diff --git a/INSTALL.md b/INSTALL.md index 74368ac0..5641c694 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -343,6 +343,27 @@ PYTHONPATH="/PATH/TO/PROJECTS/pr-agent" python pr_agent/cli.py \ review ``` +--- + +### Method 8 - Run a GitLab webhook server + +1. From the GitLab workspace or group, create an access token. Enable the "api" scope only. +2. Generate a random secret for your app, and save it for later. For example, you can use: + +``` +WEBHOOK_SECRET=$(python -c "import secrets; print(secrets.token_hex(10))") +``` +3. Follow the instructions to build the Docker image, setup a secrets file and deploy on your own server from [Method 5](#method-5-run-as-a-github-app). +4. In the secrets file, fill in the following: + - Your OpenAI key. + - In the [gitlab] section, fill in personal_access_token and shared_secret. The access token can be a personal access token, or a group or project access token. + - Set deployment_type to 'gitlab' in [configuration.toml](./pr_agent/settings/configuration.toml) +5. Create a webhook in GitLab. Set the URL to the URL of your app's server. Set the secret token to the generated secret from step 2. +In the "Trigger" section, check the ‘comments’ and ‘merge request events’ boxes. +6. Test your installation by opening a merge request or commenting or a merge request using one of CodiumAI's commands. + +--- + ### Appendix - **Debugging LLM API Calls** If you're testing your codium/pr-agent server, and need to see if calls were made successfully + the exact call logs, you can use the [LiteLLM Debugger tool](https://docs.litellm.ai/docs/debugging/hosted_debugging). diff --git a/README.md b/README.md index c6f14497..eff850e6 100644 --- a/README.md +++ b/README.md @@ -154,6 +154,7 @@ There are several ways to use PR-Agent: - Allowing you to automate the review process on your private or public repositories - [Method 6: Deploy as a Lambda Function](INSTALL.md#method-6---deploy-as-a-lambda-function) - [Method 7: AWS CodeCommit](INSTALL.md#method-7---aws-codecommit-setup) +- [Method 8: Run a GitLab webhook server](INSTALL.md#method-8---run-a-gitlab-webhook-server) ## How it works