admin-valentin/arcade-mcp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add Github Toolkit (#75)

Browse source 
### Adds the following tools to the Github Toolkit:

    1.	CreateIssueComment
	2.	SetStarred
	3.	CountStargazers
	4.	ListOrgRepositories
	5.	GetRepository
	6.	ListRepositoryActivities
	7.	ListReviewCommentsInARepository
	8.	ListPullRequests
	9.	GetPullRequest
	10.	UpdatePullRequest
	11.	ListPullRequestCommits
	12.	CreateReplyForReviewComment
	13.	ListReviewCommentsOnPullRequest
	14.	CreateReviewComment



Adds evals for all of these tools and unit tests.

---------

Co-authored-by: Sam Partee <sam@arcade-ai.com>
This commit is contained in:
Eric Gustin 2024-10-02 10:40:17 -07:00 committed by GitHub
parent bf53439b55
commit 7e352fbe91
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
21 changed files with 2537 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

51
arcade/arcade/cli/main.py
View file

@ -4,6 +4,7 @@ import readline
import threading
import uuid
import webbrowser
from pathlib import Path
from typing import Any, Optional
from urllib.parse import urlencode
@ -420,10 +421,29 @@ def evals(
config = _get_config_with_overrides(force_tls, force_no_tls, host, port)
models = models.split(",") # type: ignore[assignment]
eval_files = [f for f in os.listdir(directory) if f.startswith("eval_") and f.endswith(".py")]
directory_path = Path(directory).resolve()
if directory_path.is_dir():
eval_files = [
f
for f in directory_path.iterdir()
if f.is_file() and f.name.startswith("eval_") and f.name.endswith(".py")
]
elif directory_path.is_file():
eval_files = (
[directory_path]
if directory_path.name.startswith("eval_") and directory_path.name.endswith(".py")
else []
)
else:
console.print(f"Path not found: {directory_path}", style="bold red")
return
if not eval_files:
console.print("No evaluation files found.", style="bold yellow")
console.print(
"No evaluation files found. Filenames must start with 'eval_' and end with '.py'.",
style="bold yellow",
)
return
if show_details:
@ -438,14 +458,19 @@ def evals(
client = Arcade(api_key=config.api.key, base_url=config.engine_url)
log_engine_health(client)
for file in eval_files:
file_path = os.path.join(directory, file)
module_name = file[:-3] # Remove .py extension
for eval_file_path in eval_files:
module_name = eval_file_path.stem # filename without extension
spec = importlib.util.spec_from_file_location(module_name, file_path)
# Now we need to load the module from eval_file_path
file_path_str = str(eval_file_path)
module_name_str = module_name
# Load using importlib
spec = importlib.util.spec_from_file_location(module_name_str, file_path_str)
if spec is None:
console.print(f"Failed to load {file}", style="bold red")
console.print(f"Failed to load {eval_file_path}", style="bold red")
continue
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module) # type: ignore[union-attr]
@ -456,22 +481,26 @@ def evals(
]
if not eval_suites:
console.print(f"No @tool_eval functions found in {file}", style="bold yellow")
console.print(f"No @tool_eval functions found in {eval_file_path}", style="bold yellow")
continue
if show_details:
suite_label = "suite" if len(eval_suites) == 1 else "suites"
console.print(f"\nFound {len(eval_suites)} {suite_label} in {file}", style="bold")
console.print(
f"\nFound {len(eval_suites)} {suite_label} in {eval_file_path}", style="bold"
)
all_evaluations = []
for suite_func in eval_suites:
console.print(
Text.assemble(
("\nRunning evaluations in ", "bold"),
("Running evaluations in ", "bold"),
(suite_func.__name__, "bold blue"),
)
)
results = suite_func(config=config, models=models, max_concurrency=max_concurrent)
display_eval_results(results, show_details=show_details)
all_evaluations.append(results)
display_eval_results(all_evaluations, show_details=show_details)
@cli.command(help="Launch Arcade AI locally for tool dev", rich_help_panel="Launch")

85
arcade/arcade/cli/utils.py
View file

@ -67,11 +67,11 @@ def display_tool_messages(tool_messages: list[dict]) -> None:
if message["role"] == "assistant":
for tool_call in message.get("tool_calls", []):
console.print(
f"[bright_black]Called tool '{tool_call['function']['name']}' with parameters: {tool_call['function']['arguments']}[/bright_black]"
f"[bright_black][bold]Called tool '{tool_call['function']['name']}'[/bold]\n[bold]Parameters:[/bold]{tool_call['function']['arguments']}[/bright_black]"
)
elif message["role"] == "tool":
console.print(
f"[bright_black]Tool '{message['name']}' returned: {message['content']}[/bright_black]"
f"[bright_black][bold]'{message['name']}' tool returned:[/bold]{message['content']}[/bright_black]"
)
@ -197,7 +197,7 @@ def apply_config_overrides(
config.engine.tls = tls_input
def display_eval_results(results: list[dict[str, Any]], show_details: bool = False) -> None:
def display_eval_results(results: list[list[dict[str, Any]]], show_details: bool = False) -> None:
"""
Display evaluation results in a format inspired by pytest's output.
@ -210,47 +210,52 @@ def display_eval_results(results: list[dict[str, Any]], show_details: bool = Fal
total_warned = 0
total_cases = 0
for model_results in results:
model = model_results.get("model", "Unknown Model")
rubric = model_results.get("rubric", "Unknown Rubric")
cases = model_results.get("cases", [])
total_cases += len(cases)
console.print(f"\n[bold magenta]Model: {model}[/bold magenta]\n")
console.print(f"[bold magenta]{rubric}[/bold magenta]\n")
for case in cases:
evaluation = case["evaluation"]
status = (
"[green]PASSED[/green]"
if evaluation.passed
else "[yellow]WARNED[/yellow]"
if evaluation.warning
else "[red]FAILED[/red]"
)
if evaluation.passed:
total_passed += 1
elif evaluation.warning:
total_warned += 1
else:
total_failed += 1
# Display one-line summary for each case
console.print(f"{status} {case['name']} -- Score: {evaluation.score:.2f}")
for eval_suite in results:
for model_results in eval_suite:
model = model_results.get("model", "Unknown Model")
rubric = model_results.get("rubric", "Unknown Rubric")
cases = model_results.get("cases", [])
total_cases += len(cases)
console.print(f"[bold]Model:[/bold] [bold magenta]{model}[/bold magenta]")
if show_details:
# Show detailed information for each case
console.print(f"[bold]User Input:[/bold] {case['input']}\n")
console.print("[bold]Details:[/bold]")
console.print(_format_evaluation(evaluation))
console.print("-" * 80)
console.print(f"[bold magenta]{rubric}[/bold magenta]")
for case in cases:
evaluation = case["evaluation"]
status = (
"[green]PASSED[/green]"
if evaluation.passed
else "[yellow]WARNED[/yellow]"
if evaluation.warning
else "[red]FAILED[/red]"
)
if evaluation.passed:
total_passed += 1
elif evaluation.warning:
total_warned += 1
else:
total_failed += 1
# Display one-line summary for each case
console.print(f"{status} {case['name']} -- Score: {evaluation.score:.2f}")
if show_details:
# Show detailed information for each case
console.print(f"[bold]User Input:[/bold] {case['input']}\n")
console.print("[bold]Details:[/bold]")
console.print(_format_evaluation(evaluation))
console.print("-" * 80)
# Summary
console.print("\n[bold]Summary:[/bold]")
console.print(f"Total Cases: {total_cases}")
console.print(f"[green]Passed: {total_passed}[/green]")
console.print(f"[yellow]Warnings: {total_warned}[/yellow]")
console.print(f"[red]Failed: {total_failed}[/red]\n")
summary = (
f"[bold]Summary -- [/bold]Total: {total_cases} -- [green]Passed: {total_passed}[/green]"
)
if total_warned > 0:
summary += f" -- [yellow]Warnings: {total_warned}[/yellow]"
if total_failed > 0:
summary += f" -- [red]Failed: {total_failed}[/red]"
console.print(summary + "\n")
def _format_evaluation(evaluation: "EvaluationResult") -> str:

0
toolkits/github/arcade_github/__init__.py Normal file
View file

55
toolkits/github/arcade_github/tests/test_activity.py Normal file
View file

@ -0,0 +1,55 @@
from unittest.mock import AsyncMock, patch
import pytest
from arcade_github.tools.activity import set_starred
from httpx import Response
from arcade.core.errors import ToolExecutionError
@pytest.fixture
def mock_context():
context = AsyncMock()
context.authorization.token = "mock_token" # noqa: S105
return context
@pytest.fixture
def mock_client():
with patch("arcade_github.tools.activity.httpx.AsyncClient") as client:
yield client.return_value.__aenter__.return_value
@pytest.mark.asyncio
@pytest.mark.parametrize(
"starred,expected_message",
[
(True, "Successfully starred the repository owner/repo"),
(False, "Successfully unstarred the repository owner/repo"),
],
)
async def test_set_starred_success(mock_context, mock_client, starred, expected_message):
mock_client.put.return_value = mock_client.delete.return_value = Response(204)
result = await set_starred(mock_context, "owner", "repo", starred)
assert result == expected_message
@pytest.mark.asyncio
@pytest.mark.parametrize(
"status_code,error_message,expected_error",
[
(403, "Forbidden", "Error accessing.*: Forbidden"),
(404, "Not Found", "Error accessing.*: Resource not found"),
(500, "Internal Server Error", "Error accessing.*: Failed to process request"),
],
)
async def test_set_starred_errors(
mock_context, mock_client, status_code, error_message, expected_error
):
mock_client.put.return_value = mock_client.delete.return_value = Response(
status_code, json={"message": error_message}
)
with pytest.raises(ToolExecutionError, match=expected_error):
await set_starred(mock_context, "owner", "repo", True)

110
toolkits/github/arcade_github/tests/test_issues.py Normal file
View file

@ -0,0 +1,110 @@
from unittest.mock import AsyncMock, patch
import pytest
from arcade_github.tools.issues import create_issue, create_issue_comment
from httpx import Response
from arcade.core.errors import ToolExecutionError
@pytest.fixture
def mock_context():
context = AsyncMock()
context.authorization.token = "mock_token" # noqa: S105
return context
@pytest.fixture
def mock_client():
with patch("arcade_github.tools.issues.httpx.AsyncClient") as client:
yield client.return_value.__aenter__.return_value
@pytest.mark.asyncio
@pytest.mark.parametrize(
"status_code,error_message,expected_error,func,args",
[
(
422,
"Validation Failed",
"Error accessing.*: Validation failed",
create_issue,
("owner", "repo", "title"),
),
(
401,
"Unauthorized",
"Error accessing.*: Failed to process request",
create_issue_comment,
("owner", "repo", 1, "body"),
),
(
403,
"API rate limit exceeded",
"Error accessing.*: Forbidden",
create_issue_comment,
("owner", "repo", 1, "body"),
),
(
401,
"Bad credentials",
"Error accessing.*: Failed to process request",
create_issue,
("owner", "repo", "title"),
),
],
)
async def test_issue_errors(
mock_context, mock_client, status_code, error_message, expected_error, func, args
):
mock_client.post.return_value = Response(status_code, json={"message": error_message})
with pytest.raises(ToolExecutionError, match=expected_error):
await func(mock_context, *args)
@pytest.mark.asyncio
@pytest.mark.parametrize(
"func,args,response_json,expected_assertions",
[
(
create_issue,
("owner", "repo", "Test Issue", "This is a test issue"),
{
"id": 1,
"url": "https://api.github.com/repos/owner/repo/issues/1",
"title": "Test Issue",
"body": "This is a test issue",
"state": "open",
"html_url": "https://github.com/owner/repo/issues/1",
"created_at": "2023-05-01T12:00:00Z",
"updated_at": "2023-05-01T12:00:00Z",
"user": {"login": "testuser"},
"assignees": [],
"labels": [],
},
["Test Issue", "https://github.com/owner/repo/issues/1"],
),
(
create_issue_comment,
("owner", "repo", 1, "This is a test comment"),
{
"id": 1,
"url": "https://api.github.com/repos/owner/repo/issues/comments/1",
"body": "This is a test comment",
"user": {"login": "testuser"},
"created_at": "2023-05-01T12:00:00Z",
"updated_at": "2023-05-01T12:00:00Z",
},
["This is a test comment", "https://api.github.com/repos/owner/repo/issues/comments/1"],
),
],
)
async def test_issue_success(
mock_context, mock_client, func, args, response_json, expected_assertions
):
mock_client.post.return_value = Response(201, json=response_json)
result = await func(mock_context, *args)
for assertion in expected_assertions:
assert assertion in result

359
toolkits/github/arcade_github/tests/test_pull_requests.py Normal file
View file

@ -0,0 +1,359 @@
from unittest.mock import AsyncMock, patch
import pytest
from arcade_github.tools.models import (
DiffSide,
ReviewCommentSubjectType,
)
from arcade_github.tools.pull_requests import (
create_reply_for_review_comment,
create_review_comment,
get_pull_request,
list_pull_request_commits,
list_pull_requests,
list_review_comments_on_pull_request,
update_pull_request,
)
from httpx import Response
from arcade.core.errors import RetryableToolError, ToolExecutionError
@pytest.fixture
def mock_context():
context = AsyncMock()
context.authorization.token = "mock_token" # noqa: S105
return context
@pytest.fixture
def mock_client():
with patch("arcade_github.tools.pull_requests.httpx.AsyncClient") as client:
yield client.return_value.__aenter__.return_value
@pytest.mark.asyncio
@pytest.mark.parametrize(
"func,args,status_code,json_response,expected_result,error_message",
[
(list_pull_requests, ("owner", "repo"), 200, [], '{"pull_requests": []}', None),
(
get_pull_request,
("owner", "repo", 1),
404,
{"message": "Not Found"},
None,
"Error accessing.*: Resource not found",
),
(
update_pull_request,
("owner", "repo", 1, "New Title"),
409,
{"message": "Conflict"},
None,
"Error accessing.*: Failed to process request",
),
(
list_pull_request_commits,
("owner", "repo", 1),
500,
{"message": "Internal Server Error"},
None,
"Error accessing.*: Failed to process request",
),
(
list_review_comments_on_pull_request,
("owner", "repo", 1),
403,
{"message": "API rate limit exceeded"},
None,
"Error accessing.*: Forbidden",
),
],
)
async def test_pull_request_functions(
mock_context,
mock_client,
func,
args,
status_code,
json_response,
expected_result,
error_message,
):
mock_client.get.return_value = mock_client.post.return_value = (
mock_client.patch.return_value
) = Response(status_code, json=json_response)
if error_message:
with pytest.raises(ToolExecutionError, match=error_message):
await func(mock_context, *args)
else:
result = await func(mock_context, *args)
assert result == expected_result
@pytest.mark.asyncio
@pytest.mark.parametrize(
"func,args,json_response,expected_assertions",
[
(
list_pull_requests,
("owner", "repo"),
[
{
"number": 1,
"title": "Test PR",
"body": "This is a test PR",
"state": "open",
"html_url": "https://github.com/owner/repo/pull/1",
"created_at": "2023-05-01T12:00:00Z",
"updated_at": "2023-05-01T12:00:00Z",
"user": {"login": "testuser"},
"base": {"ref": "main"},
"head": {"ref": "feature-branch"},
}
],
["Test PR", "https://github.com/owner/repo/pull/1"],
),
(
update_pull_request,
("owner", "repo", 1, "Updated PR Title", "Updated PR body"),
{
"number": 1,
"title": "Updated PR Title",
"body": "Updated PR body",
"state": "open",
"html_url": "https://github.com/owner/repo/pull/1",
"created_at": "2023-05-01T12:00:00Z",
"updated_at": "2023-05-02T12:00:00Z",
"user": {"login": "testuser"},
},
["Updated PR Title", "Updated PR body"],
),
(
list_pull_request_commits,
("owner", "repo", 1),
[
{
"sha": "6dcb09b5b57875f334f61aebed695e2e4193db5e",
"commit": {
"author": {
"name": "Test Author",
"email": "author@example.com",
"date": "2023-05-01T12:00:00Z",
},
"message": "Test commit message",
},
}
],
["6dcb09b5b57875f334f61aebed695e2e4193db5e", "Test commit message"],
),
(
create_reply_for_review_comment,
("owner", "repo", 1, 42, "Thanks for the suggestion."),
{
"id": 123,
"body": "Thanks for the suggestion.",
"user": {"login": "testuser"},
"created_at": "2023-05-02T12:00:00Z",
"updated_at": "2023-05-02T12:00:00Z",
},
["Thanks for the suggestion.", "testuser"],
),
(
list_review_comments_on_pull_request,
("owner", "repo", 1),
[
{
"id": 1,
"body": "Great changes!",
"user": {"login": "reviewer1"},
"created_at": "2023-05-01T12:00:00Z",
"updated_at": "2023-05-01T12:00:00Z",
"path": "file1.txt",
"line": 5,
}
],
["Great changes!", "reviewer1", "file1.txt"],
),
(
get_pull_request,
("owner", "repo", 1, False, False),
{
"number": 1,
"title": "Test PR",
"body": "This is a test PR",
"state": "open",
"html_url": "https://github.com/owner/repo/pull/1",
"created_at": "2023-05-01T12:00:00Z",
"updated_at": "2023-05-01T12:00:00Z",
"user": {"login": "testuser"},
"base": {"ref": "main"},
"head": {"ref": "feature-branch"},
},
["Test PR", "https://github.com/owner/repo/pull/1"],
),
(
get_pull_request,
("owner", "repo", 1, True, False),
{
"number": 1,
"title": "Test PR",
"body": "This is a test PR",
"state": "open",
"html_url": "https://github.com/owner/repo/pull/1",
"created_at": "2023-05-01T12:00:00Z",
"updated_at": "2023-05-01T12:00:00Z",
"user": {"login": "testuser"},
"base": {"ref": "main"},
"head": {"ref": "feature-branch"},
"diff_content": "Sample diff content",
},
["Test PR", "https://github.com/owner/repo/pull/1", "diff_content"],
),
(
create_review_comment,
(
"owner",
"repo",
1,
"Great changes!",
"file1.txt",
"6dcb09b5b57875f334f61aebed695e2e4193db5e",
1,
2,
DiffSide.RIGHT,
None,
ReviewCommentSubjectType.LINE,
),
{
"id": 1,
"body": "Great changes!",
"path": "file1.txt",
"line": 2,
"side": "RIGHT",
"commit_id": "6dcb09b5b57875f334f61aebed695e2e4193db5e",
"user": {"login": "testuser"},
"created_at": "2023-05-01T12:00:00Z",
"updated_at": "2023-05-01T12:00:00Z",
"html_url": "https://github.com/owner/repo/pull/1#discussion_r1",
},
["Great changes!", "file1.txt", "6dcb09b5b57875f334f61aebed695e2e4193db5e"],
),
],
)
async def test_pull_request_functions_success(
mock_context, mock_client, func, args, json_response, expected_assertions
):
mock_client.get.return_value = mock_client.post.return_value = (
mock_client.patch.return_value
) = Response(200, json=json_response)
result = await func(mock_context, *args)
for assertion in expected_assertions:
assert assertion in result
@pytest.mark.asyncio
async def test_create_review_comment_file_subject_type(mock_context, mock_client):
mock_client.post.return_value = Response(
200,
json={
"id": 1,
"body": "File comment",
"path": "file1.txt",
"commit_id": "6dcb09b5b57875f334f61aebed695e2e4193db5e",
"user": {"login": "testuser"},
"created_at": "2023-05-01T12:00:00Z",
"updated_at": "2023-05-01T12:00:00Z",
"html_url": "https://github.com/owner/repo/pull/1#discussion_r1",
},
)
result = await create_review_comment(
mock_context,
"owner",
"repo",
1,
"File comment",
"file1.txt",
"6dcb09b5b57875f334f61aebed695e2e4193db5e",
subject_type=ReviewCommentSubjectType.FILE,
)
assert "File comment" in result
assert "file1.txt" in result
assert "6dcb09b5b57875f334f61aebed695e2e4193db5e" in result
assert "start_line" not in mock_client.post.call_args[1]["json"]
assert "end_line" not in mock_client.post.call_args[1]["json"]
@pytest.mark.asyncio
async def test_create_review_comment_missing_commit_id(mock_context, mock_client):
mock_client.get.return_value = Response(
200,
json=[{"sha": "latest_commit_sha"}],
)
mock_client.post.return_value = Response(
200,
json={
"id": 1,
"body": "Comment with auto-fetched commit ID",
"path": "file1.txt",
"commit_id": "latest_commit_sha",
"user": {"login": "testuser"},
"created_at": "2023-05-01T12:00:00Z",
"updated_at": "2023-05-01T12:00:00Z",
"html_url": "https://github.com/owner/repo/pull/1#discussion_r1",
},
)
result = await create_review_comment(
mock_context,
"owner",
"repo",
1,
"Comment with auto-fetched commit ID",
"file1.txt",
start_line=1,
end_line=2,
)
assert "Comment with auto-fetched commit ID" in result
assert "latest_commit_sha" in result
assert mock_client.get.called
assert mock_client.post.called
@pytest.mark.asyncio
async def test_create_review_comment_invalid_input(mock_context, mock_client):
with pytest.raises(
RetryableToolError, match="'start_line' and 'end_line' parameters are required"
):
await create_review_comment(
mock_context,
"owner",
"repo",
1,
"Invalid comment",
"file1.txt",
subject_type=ReviewCommentSubjectType.LINE,
)
@pytest.mark.asyncio
async def test_create_review_comment_no_commits(mock_context, mock_client):
mock_client.get.return_value = Response(200, json=[])
with pytest.raises(RetryableToolError, match="Failed to get the latest commit SHA"):
await create_review_comment(
mock_context,
"owner",
"repo",
1,
"Comment with no commits",
"file1.txt",
start_line=1,
end_line=2,
)

73
toolkits/github/arcade_github/tests/test_repositories.py Normal file
View file

@ -0,0 +1,73 @@
from unittest.mock import AsyncMock, patch
import pytest
from arcade_github.tools.models import RepoType
from arcade_github.tools.repositories import (
count_stargazers,
get_repository,
list_org_repositories,
list_repository_activities,
list_review_comments_in_a_repository,
)
from httpx import Response
from arcade.core.errors import ToolExecutionError
@pytest.fixture
def mock_context():
context = AsyncMock()
context.authorization.token = "mock_token" # noqa: S105
return context
@pytest.fixture
def mock_client():
with patch("arcade_github.tools.repositories.httpx.AsyncClient") as client:
yield client.return_value.__aenter__.return_value
@pytest.mark.asyncio
@pytest.mark.parametrize(
"status_code,error_message,expected_error",
[
(422, "Validation Failed", "Error accessing.*: Validation failed"),
(301, "Moved Permanently", "Error accessing.*: Moved permanently"),
(404, "Not Found", "Error accessing.*: Resource not found"),
(503, "Service Unavailable", "Error accessing.*: Service unavailable"),
(410, "Gone", "Error accessing.*: Gone"),
],
)
async def test_error_responses(
mock_context, mock_client, status_code, error_message, expected_error
):
mock_client.get.return_value = Response(status_code, json={"message": error_message})
mock_client.post.return_value = Response(status_code, json={"message": error_message})
with pytest.raises(ToolExecutionError, match=expected_error):
if status_code == 422:
await list_org_repositories(mock_context, "org", repo_type=RepoType.ALL)
elif status_code == 301:
await count_stargazers("owner", "repo")
elif status_code == 404:
await list_org_repositories(mock_context, "non_existent_org")
elif status_code == 503:
await get_repository(mock_context, "owner", "repo")
elif status_code == 410:
await list_review_comments_in_a_repository(mock_context, "owner", "repo")
@pytest.mark.asyncio
async def test_list_repository_activities_invalid_cursor(mock_context, mock_client):
mock_client.get.return_value = Response(422, json={"message": "Validation Failed"})
with pytest.raises(ToolExecutionError, match="Error accessing.*: Validation failed"):
await list_repository_activities(mock_context, "owner", "repo", before="invalid_cursor")
@pytest.mark.asyncio
async def test_count_stargazers_success(mock_client):
mock_client.get.return_value = Response(200, json={"stargazers_count": 42})
result = await count_stargazers("owner", "repo")
assert result == "The repository owner/repo has 42 stargazers."

0
toolkits/github/arcade_github/tools/__init__.py Normal file
View file

41
toolkits/github/arcade_github/tools/activity.py Normal file
View file

@ -0,0 +1,41 @@
from typing import Annotated
import httpx
from arcade.core.schema import ToolContext
from arcade.sdk import tool
from arcade.sdk.auth import GitHubApp
from arcade_github.tools.utils import get_github_json_headers, get_url, handle_github_response
# Implements https://docs.github.com/en/rest/activity/starring?apiVersion=2022-11-28#star-a-repository-for-the-authenticated-user and https://docs.github.com/en/rest/activity/starring?apiVersion=2022-11-28#unstar-a-repository-for-the-authenticated-user
# Example `arcade chat` usage: "star the vscode repo owned by microsoft"
@tool(requires_auth=GitHubApp())
async def set_starred(
context: ToolContext,
owner: Annotated[str, "The owner of the repository"],
name: Annotated[str, "The name of the repository"],
starred: Annotated[bool, "Whether to star the repository or not"],
) -> Annotated[
str, "A message indicating whether the repository was successfully starred or unstarred"
]:
"""
Star or un-star a GitHub repository.
For example, to star microsoft/vscode, you would use:
```
set_starred(owner="microsoft", name="vscode", starred=True)
```
"""
url = get_url("user_starred", owner=owner, repo=name)
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
if starred:
response = await client.put(url, headers=headers)
else:
response = await client.delete(url, headers=headers)
handle_github_response(response, url)
action = "starred" if starred else "unstarred"
return f"Successfully {action} the repository {owner}/{name}"

19
toolkits/github/arcade_github/tools/constants.py Normal file
View file

@ -0,0 +1,19 @@
# Base URL for GitHub API
GITHUB_API_BASE_URL = "https://api.github.com"
# Endpoint patterns
ENDPOINTS = {
"repo": "/repos/{owner}/{repo}",
"org_repos": "/orgs/{org}/repos",
"repo_activity": "/repos/{owner}/{repo}/activity",
"repo_pulls_comments": "/repos/{owner}/{repo}/pulls/comments",
"repo_issues": "/repos/{owner}/{repo}/issues",
"repo_issue_comments": "/repos/{owner}/{repo}/issues/{issue_number}/comments",
"repo_pulls": "/repos/{owner}/{repo}/pulls",
"repo_pull": "/repos/{owner}/{repo}/pulls/{pull_number}",
"repo_pull_commits": "/repos/{owner}/{repo}/pulls/{pull_number}/commits",
"repo_pull_comments": "/repos/{owner}/{repo}/pulls/{pull_number}/comments",
"repo_pull_comment_replies": "/repos/{owner}/{repo}/pulls/{pull_number}/comments/{comment_id}/replies",
"user_starred": "/user/starred/{owner}/{repo}",
"repo_stargazers": "/repos/{owner}/{repo}/stargazers",
}

137
toolkits/github/arcade_github/tools/issues.py Normal file
View file

@ -0,0 +1,137 @@
import json
from typing import Annotated, Optional
import httpx
from arcade.core.schema import ToolContext
from arcade.sdk import tool
from arcade.sdk.auth import GitHubApp
from arcade_github.tools.utils import (
get_github_json_headers,
get_url,
handle_github_response,
remove_none_values,
)
# Implements https://docs.github.com/en/rest/issues/issues?apiVersion=2022-11-28#create-an-issue
# Example `arcade chat` usage: "create an issue in the <REPO> repo owned by <OWNER> titled 'Found a bug' with the body 'I'm having a problem with this.' Assign it to <USER> and label it 'bug'"
@tool(requires_auth=GitHubApp())
async def create_issue(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
title: Annotated[str, "The title of the issue."],
body: Annotated[Optional[str], "The contents of the issue."] = None,
assignees: Annotated[Optional[list[str]], "Logins for Users to assign to this issue."] = None,
milestone: Annotated[
Optional[int], "The number of the milestone to associate this issue with."
] = None,
labels: Annotated[Optional[list[str]], "Labels to associate with this issue."] = None,
include_extra_data: Annotated[
bool,
"If true, return all the data available about the pull requests. This is a large payload and may impact performance - use with caution.",
] = False,
) -> Annotated[
str,
"A JSON string containing the created issue's details, including id, url, title, body, state, html_url, creation and update timestamps, user, assignees, and labels. If include_extra_data is True, returns all available data about the issue.",
]:
"""
Create an issue in a GitHub repository.
Example:
```
create_issue(owner="octocat", repo="Hello-World", title="Found a bug", body="I'm having a problem with this.", assignees=["octocat"], milestone=1, labels=["bug"])
```
"""
url = get_url("repo_issues", owner=owner, repo=repo)
data = {
"title": title,
"body": body,
"labels": labels,
"milestone": milestone,
"assignees": assignees,
}
data = remove_none_values(data)
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.post(url, headers=headers, json=data)
handle_github_response(response, url)
issue_data = response.json()
if include_extra_data:
return json.dumps(issue_data)
important_info = {
"id": issue_data.get("id"),
"url": issue_data.get("url"),
"title": issue_data.get("title"),
"body": issue_data.get("body"),
"state": issue_data.get("state"),
"html_url": issue_data.get("html_url"),
"created_at": issue_data.get("created_at"),
"updated_at": issue_data.get("updated_at"),
"user": issue_data.get("user", {}).get("login"),
"assignees": [assignee.get("login") for assignee in issue_data.get("assignees", [])],
"labels": [label.get("name") for label in issue_data.get("labels", [])],
}
return json.dumps(important_info)
# Implements https://docs.github.com/en/rest/issues/comments?apiVersion=2022-11-28#create-an-issue-comment
# Example `arcade chat` usage: "create a comment in the vscode repo owned by microsoft for issue 1347 that says 'Me too'"
@tool(requires_auth=GitHubApp())
async def create_issue_comment(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
issue_number: Annotated[int, "The number that identifies the issue."],
body: Annotated[str, "The contents of the comment."],
include_extra_data: Annotated[
bool,
"If true, return all the data available about the pull requests. This is a large payload and may impact performance - use with caution.",
] = False,
) -> Annotated[
str,
"A JSON string containing the created comment's details, including id, url, body, user, and creation and update timestamps. If include_extra_data is True, returns all available data about the comment.",
]:
"""
Create a comment on an issue in a GitHub repository.
Example:
```
create_issue_comment(owner="octocat", repo="Hello-World", issue_number=1347, body="Me too")
```
"""
url = get_url("repo_issue_comments", owner=owner, repo=repo, issue_number=issue_number)
data = {
"body": body,
}
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.post(url, headers=headers, json=data)
handle_github_response(response, url)
comment_data = response.json()
if include_extra_data:
return json.dumps(comment_data)
important_info = {
"id": comment_data.get("id"),
"url": comment_data.get("url"),
"body": comment_data.get("body"),
"user": comment_data.get("user", {}).get("login"),
"created_at": comment_data.get("created_at"),
"updated_at": comment_data.get("updated_at"),
}
return json.dumps(important_info)

98
toolkits/github/arcade_github/tools/models.py Normal file
View file

@ -0,0 +1,98 @@
from enum import Enum
# Pull Request specific
class PRSortProperty(str, Enum):
CREATED = "created"
UPDATED = "updated"
POPULARITY = "popularity"
LONG_RUNNING = "long-running"
class PRState(str, Enum):
OPEN = "open"
CLOSED = "closed"
ALL = "all"
class ReviewCommentSortProperty(str, Enum):
CREATED = "created"
UPDATED = "updated"
class ReviewCommentSubjectType(str, Enum):
FILE = "file"
LINE = "line"
class DiffSide(str, Enum):
"""
The side of the diff that the pull request's changes appear on.
Use LEFT for deletions that appear in red.
Use RIGHT for additions that appear in green or unchanged lines that appear in white and are shown for context
"""
LEFT = "LEFT"
RIGHT = "RIGHT"
# Repo specific
class RepoType(str, Enum):
"""
The types of repositories you want returned when listing organization repositories.
Default is all repositories.
"""
ALL = "all"
PUBLIC = "public"
PRIVATE = "private"
FORKS = "forks"
SOURCES = "sources"
MEMBER = "member"
class RepoSortProperty(str, Enum):
"""
The property to sort the results by when listing organization repositories.
Default is created.
"""
CREATED = "created"
UPDATED = "updated"
PUSHED = "pushed"
FULL_NAME = "full_name"
class RepoTimePeriod(str, Enum):
"""
The time period to filter by when listing repository activities.
"""
DAY = "day"
WEEK = "week"
MONTH = "month"
QUARTER = "quarter"
YEAR = "year"
class ActivityType(str, Enum):
"""
The activity type to filter by when listing repository activities.
"""
PUSH = "push"
FORCE_PUSH = "force_push"
BRANCH_CREATION = "branch_creation"
BRANCH_DELETION = "branch_deletion"
PR_MERGE = "pr_merge"
MERGE_QUEUE_MERGE = "merge_queue_merge"
class SortDirection(str, Enum):
"""
The order to sort by when listing organization repositories.
Default is asc.
"""
ASC = "asc"
DESC = "desc"

554
toolkits/github/arcade_github/tools/pull_requests.py Normal file
View file

@ -0,0 +1,554 @@
import json
from typing import Annotated, Optional
import httpx
from arcade.core.errors import RetryableToolError
from arcade.core.schema import ToolContext
from arcade.sdk import tool
from arcade.sdk.auth import GitHubApp
from arcade_github.tools.models import (
DiffSide,
PRSortProperty,
PRState,
ReviewCommentSortProperty,
ReviewCommentSubjectType,
SortDirection,
)
from arcade_github.tools.utils import (
get_github_diff_headers,
get_github_json_headers,
get_url,
handle_github_response,
remove_none_values,
)
# Implements https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#list-pull-requests
# Example `arcade chat` usage: "get all open PRs that <USER> has that are in the <OWNER>/<REPO> repo"
# TODO: Validate owner/repo combination is valid for the authenticated user. If not, return RetryableToolError with available repos.
# TODO: list repo's branches and validate base is in the list (or default to main). If not, return RetryableToolError with available branches.
@tool(requires_auth=GitHubApp())
async def list_pull_requests(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
state: Annotated[Optional[PRState], "The state of the pull requests to return."] = PRState.OPEN,
head: Annotated[
Optional[str],
"Filter pulls by head user or head organization and branch name in the format of user:ref-name or organization:ref-name.",
] = None,
base: Annotated[Optional[str], "Filter pulls by base branch name."] = "main",
sort: Annotated[
Optional[PRSortProperty], "The property to sort the results by."
] = PRSortProperty.CREATED,
direction: Annotated[Optional[SortDirection], "The direction of the sort."] = None,
per_page: Annotated[Optional[int], "The number of results per page (max 100)."] = 30,
page: Annotated[Optional[int], "The page number of the results to fetch."] = 1,
include_extra_data: Annotated[
bool,
"If true, return all the data available about the pull requests. This is a large payload and may impact performance - use with caution.",
] = False,
) -> Annotated[str, "JSON string containing a list of pull requests with their details"]:
"""
List pull requests in a GitHub repository.
Example:
```
list_pull_requests(owner="octocat", repo="Hello-World", state=PRState.OPEN, sort=PRSort.UPDATED)
```
"""
url = get_url("repo_pulls", owner=owner, repo=repo)
params = {
"base": base,
"state": state.value,
"sort": sort.value,
"per_page": min(max(1, per_page), 100), # clamp per_page to 1-100
"page": page,
"head": head,
"direction": direction, # Note: Github defaults to desc when sort is 'created' or not specified, otherwise defaults to asc
}
params = remove_none_values(params)
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.get(url, headers=headers, params=params)
handle_github_response(response, url)
pull_requests = response.json()
results = []
for pr in pull_requests:
if include_extra_data:
results.append(pr)
continue
results.append({
"number": pr.get("number"),
"title": pr.get("title"),
"body": pr.get("body"),
"state": pr.get("state"),
"html_url": pr.get("html_url"),
"diff_url": pr.get("diff_url"),
"created_at": pr.get("created_at"),
"updated_at": pr.get("updated_at"),
"user": pr.get("user", {}).get("login"),
"base": pr.get("base", {}).get("ref"),
"head": pr.get("head", {}).get("ref"),
})
return json.dumps({"pull_requests": results})
# Implements https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#get-a-pull-request
# Example `arcade chat` usage: "get the PR #72 in the <OWNER>/<REPO> repo. Include diff content in your response."
@tool(requires_auth=GitHubApp())
async def get_pull_request(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
pull_number: Annotated[int, "The number that identifies the pull request."],
include_diff_content: Annotated[
Optional[bool],
"If true, return the diff content of the pull request.",
] = False,
include_extra_data: Annotated[
Optional[bool],
"If true, return all the data available about the pull requests. This is a large payload and may impact performance - use with caution.",
] = False,
) -> Annotated[
str,
"JSON string containing details of the specified pull request, optionally including diff content",
]:
"""
Get details of a pull request in a GitHub repository.
Example:
```
get_pull_request(owner="octocat", repo="Hello-World", pull_number=1347)
```
"""
url = get_url("repo_pull", owner=owner, repo=repo, pull_number=pull_number)
headers = get_github_json_headers(context.authorization.token)
diff_headers = get_github_diff_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.get(url, headers=headers)
if include_diff_content:
diff_response = await client.get(url, headers=diff_headers)
handle_github_response(response, url)
if include_diff_content:
handle_github_response(diff_response, url)
pr_data = response.json()
if include_extra_data:
result = pr_data
if include_diff_content:
result["diff_content"] = diff_response.content.decode("utf-8")
return json.dumps(result)
important_info = {
"number": pr_data.get("number"),
"title": pr_data.get("title"),
"body": pr_data.get("body"),
"state": pr_data.get("state"),
"html_url": pr_data.get("html_url"),
"diff_url": pr_data.get("diff_url"),
"created_at": pr_data.get("created_at"),
"updated_at": pr_data.get("updated_at"),
"user": pr_data.get("user", {}).get("login"),
"base": pr_data.get("base", {}).get("ref"),
"head": pr_data.get("head", {}).get("ref"),
}
if include_diff_content:
important_info["diff_content"] = diff_response.content.decode("utf-8")
return json.dumps(important_info)
# Implements https://docs.github.com/en/rest/pulls/pulls?apiVersion=2022-11-28#update-a-pull-request
# Example `arcade chat` usage: "update PR #72 in the <OWNER>/<REPO> repo by changing the title to 'New Title' and setting the body to 'This PR description was added via arcade chat!'."
# TODO: Enable this tool to append to the PR contents instead of only replacing content.
@tool(requires_auth=GitHubApp())
async def update_pull_request(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
pull_number: Annotated[int, "The number that identifies the pull request."],
title: Annotated[Optional[str], "The title of the pull request."] = None,
body: Annotated[Optional[str], "The contents of the pull request."] = None,
state: Annotated[
Optional[PRState], "State of this Pull Request. Either open or closed."
] = None,
base: Annotated[
Optional[str], "The name of the branch you want your changes pulled into."
] = None,
maintainer_can_modify: Annotated[
Optional[bool], "Indicates whether maintainers can modify the pull request."
] = None,
) -> Annotated[str, "JSON string containing updated information about the pull request"]:
"""
Update a pull request in a GitHub repository.
Example:
```
update_pull_request(owner="octocat", repo="Hello-World", pull_number=1347, title="new title", body="updated body")
```
"""
url = get_url("repo_pull", owner=owner, repo=repo, pull_number=pull_number)
data = {
"title": title,
"body": body,
"state": state.value if state else None,
"base": base,
"maintainer_can_modify": maintainer_can_modify,
}
data = remove_none_values(data)
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.patch(url, headers=headers, json=data)
handle_github_response(response, url)
pr_data = response.json()
important_info = {
"url": pr_data.get("url"),
"id": pr_data.get("id"),
"html_url": pr_data.get("html_url"),
"number": pr_data.get("number"),
"state": pr_data.get("state"),
"title": pr_data.get("title"),
"user": pr_data.get("user", {}).get("login"),
"body": pr_data.get("body"),
"created_at": pr_data.get("created_at"),
"updated_at": pr_data.get("updated_at"),
}
return json.dumps(important_info)
# Implements https://docs.github.com/en/rest/pulls/commits?apiVersion=2022-11-28#list-commits-on-a-pull-request
# Example `arcade chat` usage: "list all of the commits for the PR 72 in the <OWNER>/<REPO> repo"
@tool(requires_auth=GitHubApp())
async def list_pull_request_commits(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
pull_number: Annotated[int, "The number that identifies the pull request."],
per_page: Annotated[Optional[int], "The number of results per page (max 100)."] = 30,
page: Annotated[Optional[int], "The page number of the results to fetch."] = 1,
include_extra_data: Annotated[
bool,
"If true, return all the data available about the pull requests. This is a large payload and may impact performance - use with caution.",
] = False,
) -> Annotated[str, "JSON string containing a list of commits for the specified pull request"]:
"""
List commits (from oldest to newest) on a pull request in a GitHub repository.
Example:
```
list_pull_request_commits(owner="octocat", repo="Hello-World", pull_number=1347)
```
"""
url = get_url("repo_pull_commits", owner=owner, repo=repo, pull_number=pull_number)
params = {
"per_page": max(1, min(100, per_page)), # clamp per_page to 1-100
"page": page,
}
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.get(url, headers=headers, params=params)
handle_github_response(response, url)
commits = response.json()
if include_extra_data:
return json.dumps({"commits": commits})
filtered_commits = []
for commit in commits:
filtered_commit = {
"sha": commit.get("sha"),
"html_url": commit.get("html_url"),
"diff_url": commit.get("html_url") + ".diff" if commit.get("html_url") else None,
"commit": {
"message": commit.get("commit", {}).get("message"),
"author": commit.get("commit", {}).get("author", {}).get("name"),
"committer": commit.get("commit", {}).get("committer", {}).get("name"),
"date": commit.get("commit", {}).get("committer", {}).get("date"),
},
"author": commit.get("author", {}).get("login"),
"committer": commit.get("committer", {}).get("login"),
}
filtered_commits.append(filtered_commit)
return json.dumps({"commits": filtered_commits})
# Implements https://docs.github.com/en/rest/pulls/comments?apiVersion=2022-11-28#create-a-reply-for-a-review-comment
# Example `arcade chat` usage: "create a reply to the review comment 1778019974 in arcadeai/arcade-ai for the PR 72 that says 'Thanks for the suggestion.'"
# Note: This tool requires the ID of the review comment to reply to. To obtain this ID, you should first call the `list_review_comments_on_pull_request` function.
# The returned JSON will contain the `id` field for each comment, which can be used as the `comment_id` parameter in this function.
@tool(requires_auth=GitHubApp())
async def create_reply_for_review_comment(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
pull_number: Annotated[int, "The number that identifies the pull request."],
comment_id: Annotated[int, "The unique identifier of the comment."],
body: Annotated[str, "The text of the review comment."],
) -> Annotated[str, "JSON string containing details of the created reply comment"]:
"""
Create a reply to a review comment for a pull request.
Example:
```
create_reply_for_review_comment(owner="octocat", repo="Hello-World", pull_number=1347, comment_id=42, body="Looks good to me!")
```
"""
url = get_url(
"repo_pull_comment_replies",
owner=owner,
repo=repo,
pull_number=pull_number,
comment_id=comment_id,
)
headers = get_github_json_headers(context.authorization.token)
data = {"body": body}
async with httpx.AsyncClient() as client:
response = await client.post(url, headers=headers, json=data)
handle_github_response(response, url)
return json.dumps(response.json())
# Implements https://docs.github.com/en/rest/pulls/comments?apiVersion=2022-11-28#list-review-comments-on-a-pull-request
# Example `arcade chat` usage: "list all of the review comments for PR 72 in <OWNER>/<REPO>"
@tool(requires_auth=GitHubApp())
async def list_review_comments_on_pull_request(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
pull_number: Annotated[int, "The number that identifies the pull request."],
sort: Annotated[
Optional[ReviewCommentSortProperty],
"The property to sort the results by. Can be one of: created, updated.",
] = ReviewCommentSortProperty.CREATED,
direction: Annotated[
Optional[SortDirection], "The direction to sort results. Can be one of: asc, desc."
] = SortDirection.DESC,
since: Annotated[
Optional[str],
"Only show results that were last updated after the given time. This is a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ.",
] = None,
per_page: Annotated[Optional[int], "The number of results per page (max 100)."] = 30,
page: Annotated[Optional[int], "The page number of the results to fetch."] = 1,
include_extra_data: Annotated[
bool,
"If true, return all the data available about the pull requests. This is a large payload and may impact performance - use with caution.",
] = False,
) -> Annotated[
str, "JSON string containing a list of review comments for the specified pull request"
]:
"""
List review comments on a pull request in a GitHub repository.
Example:
```
list_review_comments_on_pull_request(owner="octocat", repo="Hello-World", pull_number=1347)
```
"""
url = get_url("repo_pull_comments", owner=owner, repo=repo, pull_number=pull_number)
params = {
"sort": sort,
"direction": direction,
"per_page": max(1, min(100, per_page)), # clamp per_page to 1-100
"page": page,
"since": since,
}
params = remove_none_values(params)
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.get(url, headers=headers, params=params)
handle_github_response(response, url)
review_comments = response.json()
if include_extra_data:
return json.dumps(review_comments)
filtered_comments = []
for comment in review_comments:
filtered_comment = {
"id": comment.get("id"),
"url": comment.get("url"),
"diff_hunk": comment.get("diff_hunk"),
"path": comment.get("path"),
"position": comment.get("position"),
"original_position": comment.get("original_position"),
"commit_id": comment.get("commit_id"),
"original_commit_id": comment.get("original_commit_id"),
"in_reply_to_id": comment.get("in_reply_to_id"),
"user": comment.get("user", {}).get("login"),
"body": comment.get("body"),
"created_at": comment.get("created_at"),
"updated_at": comment.get("updated_at"),
"html_url": comment.get("html_url"),
"line": comment.get("line"),
"side": comment.get("side"),
"pull_request_url": comment.get("pull_request_url"),
}
filtered_comments.append(filtered_comment)
return json.dumps({"review_comments": filtered_comments})
# Implements https://docs.github.com/en/rest/pulls/comments?apiVersion=2022-11-28#create-a-review-comment-for-a-pull-request
# Example `arcade chat` usage: "create a review comment for PR 72 in <OWNER>/<REPO> that says 'Great stuff! This looks good to merge. Add the comment to README.md file.'"
# TODO: Verify that path parameter exists in the PR's files that have changed (Or should we allow for any file in the repo?). If not, then throw RetryableToolError with all valid file paths.
@tool(requires_auth=GitHubApp())
async def create_review_comment(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
pull_number: Annotated[int, "The number that identifies the pull request."],
body: Annotated[str, "The text of the review comment."],
path: Annotated[str, "The relative path to the file that necessitates a comment."],
commit_id: Annotated[
Optional[str],
"The SHA of the commit needing a comment. If not provided, the latest commit SHA of the PR's base branch will be used.",
] = None,
start_line: Annotated[
Optional[int],
"The start line of the range of lines in the pull request diff that the comment applies to. Required unless 'subject_type' is 'file'.",
] = None,
end_line: Annotated[
Optional[int],
"The end line of the range of lines in the pull request diff that the comment applies to. Required unless 'subject_type' is 'file'.",
] = None,
side: Annotated[
Optional[DiffSide],
"The side of the diff that the pull request's changes appear on. Use LEFT for deletions that appear in red. Use RIGHT for additions that appear in green or unchanged lines that appear in white and are shown for context",
] = DiffSide.RIGHT,
start_side: Annotated[
Optional[str], "The starting side of the diff that the comment applies to."
] = None,
subject_type: Annotated[
Optional[ReviewCommentSubjectType],
"The type of subject that the comment applies to. Can be one of: file, hunk, or line.",
] = ReviewCommentSubjectType.FILE,
include_extra_data: Annotated[
bool,
"If true, return all the data available about the review comment. This is a large payload and may impact performance - use with caution.",
] = False,
) -> Annotated[str, "JSON string containing details of the created review comment"]:
"""
Create a review comment for a pull request in a GitHub repository.
If the subject_type is not 'file', then the start_line and end_line parameters are required.
If the subject_type is 'file', then the start_line and end_line parameters are ignored.
If the commit_id is not provided, the latest commit SHA of the PR's base branch will be used.
Example:
```
create_review_comment(owner="octocat", repo="Hello-World", pull_number=1347, body="Great stuff!", commit_id="6dcb09b5b57875f334f61aebed695e2e4193db5e", path="file1.txt", line=2, side="RIGHT")
```
"""
# If the subject_type is 'file', then the line_range parameter is ignored
if subject_type == ReviewCommentSubjectType.FILE:
start_line, end_line = None, None
if (start_line is None or end_line is None) and subject_type != ReviewCommentSubjectType.FILE:
raise RetryableToolError(
"'start_line' and 'end_line' parameters are required when 'subject_type' parameter is not 'file'. Either provide both a start_line and end_line or set subject_type to 'file'."
)
# Ensure the line range goes from lowest to highest
if start_line is not None and end_line is not None:
start_line, end_line = (min(start_line, end_line), max(start_line, end_line))
# Get the latest commit SHA of the PR's base branch and use that for the commit_id
if not commit_id:
commits_json = await list_pull_request_commits(context, owner, repo, pull_number)
commits_data = json.loads(commits_json)
commits = commits_data.get("commits", [])
latest_commit = commits[-1] if commits else {}
commit_id = latest_commit.get("sha")
if not commit_id:
raise RetryableToolError(
f"Failed to get the latest commit SHA of PR {pull_number} in repo {repo} owned by {owner}. Does the PR exist?"
)
url = get_url("repo_pull_comments", owner=owner, repo=repo, pull_number=pull_number)
data = {
"body": body,
"commit_id": commit_id,
"path": path,
"side": side,
"line": end_line if end_line else None,
"start_line": start_line
if start_line and start_line != end_line
else None, # Only send start_line when using multi-line comments
"start_side": start_side,
}
data = remove_none_values(data)
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.post(url, headers=headers, json=data)
handle_github_response(response, url)
comment_data = response.json()
if include_extra_data:
return json.dumps(comment_data)
important_info = {
"id": comment_data.get("id"),
"url": comment_data.get("url"),
"body": comment_data.get("body"),
"path": comment_data.get("path"),
"line": comment_data.get("line"),
"side": comment_data.get("side"),
"commit_id": comment_data.get("commit_id"),
"user": comment_data.get("user", {}).get("login"),
"created_at": comment_data.get("created_at"),
"updated_at": comment_data.get("updated_at"),
"html_url": comment_data.get("html_url"),
}
return json.dumps(important_info)

343
toolkits/github/arcade_github/tools/repositories.py Normal file
View file

@ -0,0 +1,343 @@
import json
from typing import Annotated, Optional
import httpx
from arcade.core.schema import ToolContext
from arcade.sdk import tool
from arcade.sdk.auth import GitHubApp
from arcade_github.tools.models import (
ActivityType,
RepoSortProperty,
RepoTimePeriod,
RepoType,
ReviewCommentSortProperty,
SortDirection,
)
from arcade_github.tools.utils import (
get_github_json_headers,
get_url,
handle_github_response,
remove_none_values,
)
# Implements https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#get-a-repository and returns only the stargazers_count field.
# Example arcade chat usage: "How many stargazers does the <OWNER>/<REPO> repo have?"
@tool(requires_auth=GitHubApp())
async def count_stargazers(
owner: Annotated[str, "The owner of the repository"],
name: Annotated[str, "The name of the repository"],
) -> Annotated[int, "The number of stargazers (stars) for the specified repository"]:
"""Count the number of stargazers (stars) for a GitHub repository.
For example, to count the number of stars for microsoft/vscode, you would use:
```
count_stargazers(owner="microsoft", name="vscode")
```
"""
url = get_url("repo", owner=owner, repo=name)
async with httpx.AsyncClient() as client:
response = await client.get(url)
handle_github_response(response, url)
data = response.json()
stargazers_count = data.get("stargazers_count", 0)
return f"The repository {owner}/{name} has {stargazers_count} stargazers."
# Implements https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-organization-repositories
# Example arcade chat usage: "List all repositories for the <ORG> organization. Sort by creation date in descending order."
@tool(requires_auth=GitHubApp())
async def list_org_repositories(
context: ToolContext,
org: Annotated[str, "The organization name. The name is not case sensitive"],
repo_type: Annotated[RepoType, "The types of repositories you want returned."] = RepoType.ALL,
sort: Annotated[
RepoSortProperty, "The property to sort the results by"
] = RepoSortProperty.CREATED,
sort_direction: Annotated[SortDirection, "The order to sort by"] = SortDirection.ASC,
per_page: Annotated[Optional[int], "The number of results per page"] = 30,
page: Annotated[Optional[int], "The page number of the results to fetch"] = 1,
include_extra_data: Annotated[
bool,
"If true, return all the data available about the pull requests. This is a large payload and may impact performance - use with caution.",
] = False,
) -> Annotated[
dict[str, list[dict]],
"A dictionary with key 'repositories' containing a list of repositories, each with details such as name, full_name, html_url, description, clone_url, private status, creation/update/push timestamps, and star/watcher/fork counts",
]:
"""List repositories for the specified organization."""
url = get_url("org_repos", org=org)
params = {
"type": repo_type.value,
"sort": sort.value,
"direction": sort_direction.value,
"per_page": per_page,
"page": page,
}
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.get(url, headers=headers, params=params)
handle_github_response(response, url)
repos = response.json()
if include_extra_data:
return {"repositories": repos}
results = []
for repo in repos:
results.append({
"name": repo["name"],
"full_name": repo["full_name"],
"html_url": repo["html_url"],
"description": repo["description"],
"clone_url": repo["clone_url"],
"private": repo["private"],
"created_at": repo["created_at"],
"updated_at": repo["updated_at"],
"pushed_at": repo["pushed_at"],
"stargazers_count": repo["stargazers_count"],
"watchers_count": repo["watchers_count"],
"forks_count": repo["forks_count"],
})
return {"repositories": results}
# Implements https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#get-a-repository
# Example arcade chat usage: "Tell me about the <OWNER>/<REPO> repo."
@tool(requires_auth=GitHubApp())
async def get_repository(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
include_extra_data: Annotated[
bool,
"If true, return all the data available about the pull requests. This is a large payload and may impact performance - use with caution.",
] = False,
) -> Annotated[
dict,
"A dictionary containing repository details such as name, full_name, html_url, description, clone_url, private status, creation/update/push timestamps, and star/watcher/fork counts",
]:
"""Get a repository.
Retrieves detailed information about a repository using the GitHub API.
Example:
```
get_repository(owner="octocat", repo="Hello-World")
```
"""
url = get_url("repo", owner=owner, repo=repo)
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.get(url, headers=headers)
handle_github_response(response, url)
repo_data = response.json()
if include_extra_data:
return json.dumps(repo_data)
return {
"name": repo_data["name"],
"full_name": repo_data["full_name"],
"html_url": repo_data["html_url"],
"description": repo_data["description"],
"clone_url": repo_data["clone_url"],
"private": repo_data["private"],
"created_at": repo_data["created_at"],
"updated_at": repo_data["updated_at"],
"pushed_at": repo_data["pushed_at"],
"stargazers_count": repo_data["stargazers_count"],
"watchers_count": repo_data["watchers_count"],
"forks_count": repo_data["forks_count"],
}
# Implements https://docs.github.com/en/rest/repos/repos?apiVersion=2022-11-28#list-repository-activities
# Example arcade chat usage: "List all merges into main for the <OWNER>/<REPO> repo in the last week by <USER>"
@tool(requires_auth=GitHubApp())
async def list_repository_activities(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
direction: Annotated[
Optional[SortDirection], "The direction to sort the results by."
] = SortDirection.DESC,
per_page: Annotated[Optional[int], "The number of results per page (max 100)."] = 30,
before: Annotated[
Optional[str],
"A cursor (unique identifier, e.g., a SHA of a commit) to search for results before this cursor.",
] = None,
after: Annotated[
Optional[str],
"A cursor (unique identifier, e.g., a SHA of a commit) to search for results after this cursor.",
] = None,
ref: Annotated[
Optional[str],
"The Git reference for the activities you want to list. The ref for a branch can be formatted either as refs/heads/BRANCH_NAME or BRANCH_NAME, where BRANCH_NAME is the name of your branch.",
] = None,
actor: Annotated[
Optional[str], "The GitHub username to filter by the actor who performed the activity."
] = None,
time_period: Annotated[Optional[RepoTimePeriod], "The time period to filter by."] = None,
activity_type: Annotated[Optional[ActivityType], "The activity type to filter by."] = None,
include_extra_data: Annotated[
bool,
"If true, return all the data available about the pull requests. This is a large payload and may impact performance - use with caution.",
] = False,
) -> Annotated[
str,
"A JSON string containing a dictionary with key 'activities', which is a list of repository activities. Each activity includes id, node_id, before and after states, ref, timestamp, activity_type, and actor information",
]:
"""List repository activities.
Retrieves a detailed history of changes to a repository, such as pushes, merges, force pushes, and branch changes,
and associates these changes with commits and users.
Example:
```
list_repository_activities(
owner="octocat",
repo="Hello-World",
per_page=10,
activity_type="force_push"
)
```
"""
url = get_url("repo_activity", owner=owner, repo=repo)
params = {
"direction": direction.value,
"per_page": min(100, per_page), # The API only allows up to 100 per page
"before": before,
"after": after,
"ref": ref,
"actor": actor,
"time_period": time_period,
"activity_type": activity_type,
}
params = remove_none_values(params)
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.get(url, headers=headers, params=params)
handle_github_response(response, url)
activities = response.json()
if include_extra_data:
return json.dumps({"activities": activities})
results = []
for activity in activities:
results.append({
"id": activity["id"],
"node_id": activity["node_id"],
"before": activity.get("before"),
"after": activity.get("after"),
"ref": activity.get("ref"),
"timestamp": activity.get("timestamp"),
"activity_type": activity.get("activity_type"),
"actor": activity.get("actor", {}).get("login") if activity.get("actor") else None,
})
return json.dumps({"activities": results})
# Implements https://docs.github.com/en/rest/pulls/comments?apiVersion=2022-11-28#list-review-comments-in-a-repository
# Example arcade chat usage: "List all review comments for the <OWNER>/<REPO> repo. Sort by update date in descending order."
# TODO: Improve the 'since' input parameter such that language model can more easily specify a valid date/time.
@tool(requires_auth=GitHubApp())
async def list_review_comments_in_a_repository(
context: ToolContext,
owner: Annotated[str, "The account owner of the repository. The name is not case sensitive."],
repo: Annotated[
str,
"The name of the repository without the .git extension. The name is not case sensitive.",
],
sort: Annotated[
Optional[ReviewCommentSortProperty], "Can be one of: created, updated."
] = ReviewCommentSortProperty.CREATED,
direction: Annotated[
Optional[SortDirection],
"The direction to sort results. Ignored without sort parameter. Can be one of: asc, desc.",
] = SortDirection.DESC,
since: Annotated[
Optional[str],
"Only show results that were last updated after the given time. This is a timestamp in ISO 8601 format: YYYY-MM-DDTHH:MM:SSZ.",
] = None,
per_page: Annotated[Optional[int], "The number of results per page (max 100)."] = 30,
page: Annotated[Optional[int], "The page number of the results to fetch."] = 1,
include_extra_data: Annotated[
bool,
"If true, return all the data available about the pull requests. This is a large payload and may impact performance - use with caution.",
] = False,
) -> Annotated[
str,
"A JSON string containing a dictionary with key 'review_comments', which is a list of review comments. Each comment includes id, url, diff_hunk, path, position details, commit information, user, body, timestamps, and related URLs",
]:
"""
List review comments in a GitHub repository.
Example:
```
list_review_comments(owner="octocat", repo="Hello-World", sort="created", direction="asc")
```
"""
url = get_url("repo_pulls_comments", owner=owner, repo=repo)
params = {
"per_page": min(max(1, per_page), 100), # clamp per_page to 1-100
"page": page,
"sort": sort,
"direction": direction,
"since": since,
}
params = remove_none_values(params)
headers = get_github_json_headers(context.authorization.token)
async with httpx.AsyncClient() as client:
response = await client.get(url, headers=headers, params=params)
handle_github_response(response, url)
review_comments = response.json()
if include_extra_data:
return json.dumps({"review_comments": review_comments})
else:
important_info = [
{
"id": comment["id"],
"url": comment["url"],
"diff_hunk": comment["diff_hunk"],
"path": comment["path"],
"position": comment["position"],
"original_position": comment["original_position"],
"commit_id": comment["commit_id"],
"original_commit_id": comment["original_commit_id"],
"in_reply_to_id": comment.get("in_reply_to_id"),
"user": comment["user"]["login"],
"body": comment["body"],
"created_at": comment["created_at"],
"updated_at": comment["updated_at"],
"html_url": comment["html_url"],
"line": comment["line"],
"side": comment["side"],
"pull_request_url": comment["pull_request_url"],
}
for comment in review_comments
]
return json.dumps({"review_comments": important_info})

79
toolkits/github/arcade_github/tools/utils.py Normal file
View file

@ -0,0 +1,79 @@
from arcade.core.errors import ToolExecutionError
from arcade_github.tools.constants import ENDPOINTS, GITHUB_API_BASE_URL
def handle_github_response(response: dict, url: str) -> None:
"""
Handle GitHub API response and raise appropriate exceptions for non-200 status codes.
:param response: The response object from the GitHub API
:param url: The URL of the API endpoint
:raises ToolExecutionError: If the response status code is not 200
"""
if 200 <= response.status_code < 300:
return
error_messages = {
301: "Moved permanently. The repository has moved.",
304: "Not modified. The requested resource hasn't been modified since the last request.",
403: "Forbidden. You do not have access to this resource.",
404: "Resource not found. The requested resource does not exist.",
410: "Gone. The requested resource is no longer available.",
422: "Validation failed or the endpoint has been spammed.",
503: "Service unavailable. The server is temporarily unable to handle the request.",
}
error_message = error_messages.get(
response.status_code, f"Failed to process request. Status code: {response.status_code}"
)
raise ToolExecutionError(f"Error accessing '{url}': {error_message}")
def get_github_json_headers(token: str) -> dict:
"""
Generate common headers for GitHub API requests.
:param token: The authorization token
:return: A dictionary of headers
"""
return {
"Accept": "application/vnd.github+json",
"Authorization": f"Bearer {token}",
"X-GitHub-Api-Version": "2022-11-28",
}
def get_github_diff_headers(token: str) -> dict:
"""
Generate headers for GitHub API requests for diff content.
:param token: The authorization token
:return: A dictionary of headers
"""
return {
"Accept": "application/vnd.github.diff",
"Authorization": f"Bearer {token}",
"X-GitHub-Api-Version": "2022-11-28",
}
def remove_none_values(params: dict) -> dict:
"""
Remove None values from a dictionary.
:param params: The dictionary to clean
:return: A new dictionary with None values removed
"""
return {k: v for k, v in params.items() if v is not None}
def get_url(endpoint: str, **kwargs) -> str:
"""
Get the full URL for a given endpoint.
:param endpoint: The endpoint key from ENDPOINTS
:param kwargs: The parameters to format the URL with
:return: The full URL
"""
return f"{GITHUB_API_BASE_URL}{ENDPOINTS[endpoint].format(**kwargs)}"

74
toolkits/github/evals/eval_github_activity.py Normal file
View file

@ -0,0 +1,74 @@
import arcade_github
from arcade_github.tools.activity import set_starred
from arcade.core.catalog import ToolCatalog
from arcade.sdk.eval import (
BinaryCritic,
EvalRubric,
EvalSuite,
tool_eval,
)
# Evaluation rubric
rubric = EvalRubric(
fail_threshold=0.9,
warn_threshold=0.95,
)
catalog = ToolCatalog()
# Register the GitHub tools
catalog.add_module(arcade_github)
@tool_eval()
def github_activity_eval_suite() -> EvalSuite:
"""Evaluation suite for GitHub Activity tools."""
suite = EvalSuite(
name="GitHub Activity Tools Evaluation Suite",
system_message="You are an AI assistant that helps users interact with GitHub repositories using the provided tools.",
catalog=catalog,
rubric=rubric,
)
# Set Starred
suite.add_case(
name="Star a repository",
user_message="Star the test repository that is owned by ArcadeAI.",
expected_tool_calls=[
(
set_starred,
{
"owner": "ArcadeAI",
"name": "test",
"starred": True,
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.3),
BinaryCritic(critic_field="name", weight=0.3),
BinaryCritic(critic_field="starred", weight=0.4),
],
)
suite.add_case(
name="Unstar a repository",
user_message="Unstar the ArcadeAI/test repository.",
expected_tool_calls=[
(
set_starred,
{
"owner": "ArcadeAI",
"name": "test",
"starred": False,
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.3),
BinaryCritic(critic_field="name", weight=0.3),
BinaryCritic(critic_field="starred", weight=0.4),
],
)
return suite

91
toolkits/github/evals/eval_github_issues.py Normal file
View file

@ -0,0 +1,91 @@
import arcade_github
from arcade_github.tools.issues import (
create_issue,
create_issue_comment,
)
from arcade.core.catalog import ToolCatalog
from arcade.sdk.eval import (
BinaryCritic,
EvalRubric,
EvalSuite,
SimilarityCritic,
tool_eval,
)
# Evaluation rubric
rubric = EvalRubric(
fail_threshold=0.9,
warn_threshold=0.95,
)
catalog = ToolCatalog()
# Register the GitHub tools
catalog.add_module(arcade_github)
@tool_eval()
def github_issues_eval_suite() -> EvalSuite:
"""Evaluation suite for GitHub Issues tools."""
suite = EvalSuite(
name="GitHub Issues Tools Evaluation Suite",
system_message="You are an AI assistant that helps users interact with GitHub issues using the provided tools.",
catalog=catalog,
rubric=rubric,
)
# Create Issue
suite.add_case(
name="Create a new issue",
user_message="Create a new issue in the 'ArcadeAI/arcade-ai' repository with the title 'Bug: Login not working' and description 'Users are unable to log in to the application.' Assign the issue to TestUser, add it to milestone 1, and add the labels 'bug', and 'critical'.",
expected_tool_calls=[
(
create_issue,
{
"owner": "ArcadeAI",
"repo": "arcade-ai",
"title": "Bug: Login not working",
"body": "Users are unable to log in to the application.",
"assignees": ["TestUser"],
"milestone": 1,
"labels": ["bug", "critical"],
"include_extra_data": False,
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.2),
BinaryCritic(critic_field="repo", weight=0.2),
SimilarityCritic(critic_field="title", weight=0.2),
SimilarityCritic(critic_field="body", weight=0.1),
BinaryCritic(critic_field="assignees", weight=0.1),
BinaryCritic(critic_field="milestone", weight=0.1),
BinaryCritic(critic_field="labels", weight=0.1),
],
)
# Create Issue Comment
suite.add_case(
name="Add a comment to an existing issue",
user_message="Add a comment to issue #42 in the 'ArcadeAI/test' repository saying 'This issue is being investigated by the dev team.'",
expected_tool_calls=[
(
create_issue_comment,
{
"owner": "ArcadeAI",
"repo": "test",
"issue_number": 42,
"body": "This issue is being investigated by the dev team.",
"include_extra_data": False,
},
)
],
critics=[
SimilarityCritic(critic_field="owner", weight=0.2),
SimilarityCritic(critic_field="repo", weight=0.2),
BinaryCritic(critic_field="issue_number", weight=0.3),
SimilarityCritic(critic_field="body", weight=0.2),
],
)
return suite

245
toolkits/github/evals/eval_github_pull_requests.py Normal file
View file

@ -0,0 +1,245 @@
import arcade_github
from arcade_github.tools.models import DiffSide, ReviewCommentSubjectType # Add these imports
from arcade_github.tools.pull_requests import (
create_reply_for_review_comment,
create_review_comment, # Add this import
get_pull_request,
list_pull_request_commits,
list_pull_requests,
list_review_comments_on_pull_request,
update_pull_request,
)
from arcade.core.catalog import ToolCatalog
from arcade.sdk.eval import (
BinaryCritic,
EvalRubric,
EvalSuite,
SimilarityCritic,
tool_eval,
)
# Evaluation rubric
rubric = EvalRubric(
fail_threshold=0.9,
warn_threshold=0.95,
)
catalog = ToolCatalog()
# Register the GitHub tools
catalog.add_module(arcade_github)
@tool_eval()
def github_pull_requests_eval_suite() -> EvalSuite:
"""Evaluation suite for GitHub Pull Requests tools."""
suite = EvalSuite(
name="GitHub Pull Requests Tools Evaluation Suite",
system_message="You are an AI assistant that helps users interact with GitHub pull requests using the provided tools.",
catalog=catalog,
rubric=rubric,
)
# List Pull Requests
suite.add_case(
name="List all open pull requests",
user_message="List all open pull requests in the test repository under the ArcadeAI account that are proposing to merge into main.",
expected_tool_calls=[
(
list_pull_requests,
{
"owner": "ArcadeAI",
"repo": "test",
"state": "open",
"base": "main",
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.2),
BinaryCritic(critic_field="repo", weight=0.2),
BinaryCritic(critic_field="state", weight=0.2),
BinaryCritic(critic_field="base", weight=0.1),
],
)
# Get Pull Request
suite.add_case(
name="Get details of a pull request",
user_message="Get diff of pull request #72 in the 'ArcadeAI/test' repository. Include all the data that is available in your response.",
expected_tool_calls=[
(
get_pull_request,
{
"owner": "ArcadeAI",
"repo": "test",
"pull_number": 72,
"include_diff_content": True,
"include_extra_data": True,
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.2),
BinaryCritic(critic_field="repo", weight=0.2),
BinaryCritic(critic_field="pull_number", weight=0.3),
BinaryCritic(critic_field="include_extra_data", weight=0.1),
BinaryCritic(critic_field="include_diff_content", weight=0.2),
],
)
# Update Pull Request
suite.add_case(
name="Update a pull request",
user_message="Update the title of pull request #72 in the 'ArcadeAI/test' repository to 'Updated Title'.",
expected_tool_calls=[
(
update_pull_request,
{
"owner": "ArcadeAI",
"repo": "test",
"pull_number": 72,
"title": "Updated Title",
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.2),
BinaryCritic(critic_field="repo", weight=0.2),
BinaryCritic(critic_field="pull_number", weight=0.3),
BinaryCritic(critic_field="title", weight=0.3),
],
)
# List Pull Request Commits
suite.add_case(
name="List commits on a pull request",
user_message="List all commits for PR 72 in the test repository under ArcadeAI.",
expected_tool_calls=[
(
list_pull_request_commits,
{
"owner": "ArcadeAI",
"repo": "test",
"pull_number": 72,
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.2),
BinaryCritic(critic_field="repo", weight=0.2),
BinaryCritic(critic_field="pull_number", weight=0.3),
],
)
# Create Reply for Review Comment
suite.add_case(
name="Create a reply to a review comment",
user_message="Create a reply to the review comment 1778019974 in 'ArcadeAI/test' for pr 72 saying 'Thanks for the suggestion.'",
expected_tool_calls=[
(
create_reply_for_review_comment,
{
"owner": "ArcadeAI",
"repo": "test",
"pull_number": 72,
"comment_id": 1778019974,
"body": "Thanks for the suggestion.",
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.2),
BinaryCritic(critic_field="repo", weight=0.2),
BinaryCritic(critic_field="pull_number", weight=0.2),
BinaryCritic(critic_field="comment_id", weight=0.2),
SimilarityCritic(critic_field="body", weight=0.2),
],
)
# List Review Comments on Pull Request
suite.add_case(
name="List all review comments on a pull request",
user_message="List review comments for pr 72 in the ArcadeAI/test repo. Sort by updated time in ascending order.",
expected_tool_calls=[
(
list_review_comments_on_pull_request,
{
"owner": "ArcadeAI",
"repo": "test",
"pull_number": 72,
"sort": "updated",
"direction": "asc",
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.2),
BinaryCritic(critic_field="repo", weight=0.2),
BinaryCritic(critic_field="pull_number", weight=0.2),
BinaryCritic(critic_field="sort", weight=0.2),
BinaryCritic(critic_field="direction", weight=0.2),
],
)
# Create Review Comment
suite.add_case(
name="Create a review comment on a pull request file",
user_message="Create a review comment on pr 72 in the 'ArcadeAI/test' repo. The comment should be on the file 'README.md' and says 'nit: you misspelled the word 'intelligence'",
expected_tool_calls=[
(
create_review_comment,
{
"owner": "ArcadeAI",
"repo": "test",
"pull_number": 72,
"body": "nit: you misspelled the word 'intelligence'",
"path": "README.md",
"subject_type": ReviewCommentSubjectType.FILE,
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.15),
BinaryCritic(critic_field="repo", weight=0.15),
BinaryCritic(critic_field="pull_number", weight=0.2),
SimilarityCritic(critic_field="body", weight=0.1),
BinaryCritic(critic_field="path", weight=0.2),
BinaryCritic(critic_field="subject_type", weight=0.2),
],
)
# Create Review Comment with Line Numbers
suite.add_case(
name="Create a review comment on specific lines of a pull request",
user_message="Create a review comment on pull request #72 in the 'ArcadeAI/test' repository. The comment should be on the file 'src/main.py', lines 10-15, and say 'Move these to constants.py.'",
expected_tool_calls=[
(
create_review_comment,
{
"owner": "ArcadeAI",
"repo": "test",
"pull_number": 72,
"body": "Move these to constants.py.",
"path": "src/main.py",
"start_line": 10,
"end_line": 15,
"side": DiffSide.RIGHT,
"subject_type": ReviewCommentSubjectType.LINE,
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.1),
BinaryCritic(critic_field="repo", weight=0.1),
BinaryCritic(critic_field="pull_number", weight=0.15),
SimilarityCritic(critic_field="body", weight=0.15),
BinaryCritic(critic_field="path", weight=0.1),
BinaryCritic(critic_field="start_line", weight=0.1),
BinaryCritic(critic_field="end_line", weight=0.1),
BinaryCritic(critic_field="side", weight=0.1),
BinaryCritic(critic_field="subject_type", weight=0.1),
],
)
return suite

156
toolkits/github/evals/eval_github_repositories.py Normal file
View file

@ -0,0 +1,156 @@
import arcade_github
from arcade_github.tools.repositories import (
count_stargazers,
get_repository,
list_org_repositories,
list_repository_activities,
list_review_comments_in_a_repository,
)
from arcade.core.catalog import ToolCatalog
from arcade.sdk.eval import (
BinaryCritic,
EvalRubric,
EvalSuite,
tool_eval,
)
# Evaluation rubric
rubric = EvalRubric(
fail_threshold=0.9,
warn_threshold=0.95,
)
catalog = ToolCatalog()
# Register the GitHub tools
catalog.add_module(arcade_github)
@tool_eval()
def github_repositories_eval_suite() -> EvalSuite:
"""Evaluation suite for GitHub Repositories tools."""
suite = EvalSuite(
name="GitHub Repositories Tools Evaluation Suite",
system_message="You are an AI assistant that helps users interact with GitHub repositories using the provided tools.",
catalog=catalog,
rubric=rubric,
)
# Count Stargazers
suite.add_case(
name="Count stargazers of a repository",
user_message="How many stargazers does the ArcadeAI/test repo have?",
expected_tool_calls=[
(
count_stargazers,
{
"owner": "ArcadeAI",
"name": "test",
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.5),
BinaryCritic(critic_field="name", weight=0.5),
],
)
# List an Organization's Repositories
suite.add_case(
name="List repositories in an organization",
user_message="List all repos in the ArcadeAI org, sorted by creation date in descending order.",
expected_tool_calls=[
(
list_org_repositories,
{
"org": "ArcadeAI",
"repo_type": "all",
"sort": "created",
"sort_direction": "desc",
},
)
],
critics=[
BinaryCritic(critic_field="org", weight=0.1),
BinaryCritic(critic_field="repo_type", weight=0.1),
BinaryCritic(critic_field="sort", weight=0.1),
BinaryCritic(critic_field="sort_direction", weight=0.1),
],
)
# Get Repository
suite.add_case(
name="Get details of a repository",
user_message="Tell me about the test repo owned by ArcadeAI.",
expected_tool_calls=[
(
get_repository,
{
"owner": "ArcadeAI",
"repo": "test",
"include_extra_data": False,
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.3),
BinaryCritic(critic_field="repo", weight=0.3),
],
)
# List Repository Activities
suite.add_case(
name="List activities in a repository",
user_message="List all PR merges in the 'ArcadeAI/test' repository that were performed by TestUser in the last month",
expected_tool_calls=[
(
list_repository_activities,
{
"owner": "ArcadeAI",
"repo": "test",
"direction": "desc",
"per_page": 30,
"actor": "TestUser",
"time_period": "month",
"activity_type": "pr_merge",
"include_extra_data": False,
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.1),
BinaryCritic(critic_field="repo", weight=0.1),
BinaryCritic(critic_field="direction", weight=0.1),
BinaryCritic(critic_field="actor", weight=0.1),
BinaryCritic(critic_field="time_period", weight=0.1),
BinaryCritic(critic_field="activity_type", weight=0.1),
],
)
# List Review Comments in a Repository
suite.add_case(
name="List review comments in a repository",
user_message="List all review comments in the 'ArcadeAI/test' repository, sorted by creation date in descending order.",
expected_tool_calls=[
(
list_review_comments_in_a_repository,
{
"owner": "ArcadeAI",
"repo": "test",
"sort": "created",
"direction": "desc",
"per_page": 30,
"page": 1,
"include_extra_data": False,
},
)
],
critics=[
BinaryCritic(critic_field="owner", weight=0.2),
BinaryCritic(critic_field="repo", weight=0.2),
BinaryCritic(critic_field="sort", weight=0.1),
BinaryCritic(critic_field="direction", weight=0.1),
],
)
return suite

17
toolkits/github/pyproject.toml Normal file
View file

@ -0,0 +1,17 @@
[tool.poetry]
name = "arcade_github"
version = "0.1.0"
description = "LLM tools for interacting with Github"
authors = ["Eric Gustin <eric@arcade-ai.com>"]
[tool.poetry.dependencies]
python = "^3.10"
arcade-ai = "^0.1.0"
httpx = "^0.27.2"
[tool.poetry.dev-dependencies]
pytest = "^8.3.0"
[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

2
toolkits/google/arcade_google/tools/gmail.py
View file

@ -376,7 +376,7 @@ def process_messages(service, messages):
try:
email_data = service.users().messages().get(userId="me", id=msg["id"]).execute()
email_details = parse_email(email_data)
emails += email_details if email_details else []
emails += [email_details] if email_details else []
except HttpError as e:
print(f"Error reading email {msg['id']}: {e}")
return emails