admin-valentin/arcade-mcp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add Google Calendar Tools (#71)

Browse source 
This PR adds four new tools to the Google ToolKit
* `create_event`
* `list_events`
* `update_event`
* `delete_event`


I also improved an error log when tools are being registered by the
actor.


This PR also sneaks in an eval for gmail


Here is a sample conversation that shows the tools and their
capabilities and limitiations:

![image](https://github.com/user-attachments/assets/a70e97d7-2c4e-4f6e-aebd-1fcaaf6c36b9)
This commit is contained in:
Eric Gustin 2024-10-02 10:24:11 -07:00 committed by GitHub
parent 13563871cb
commit bf53439b55
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 1033 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
arcade/arcade/core/catalog.py
View file

@ -175,6 +175,7 @@ class ToolCatalog(BaseModel):
try:
module = import_module(module_name)
tool_func = getattr(module, tool_name)
self.add_tool(tool_func, toolkit, module)
except AttributeError:
raise ToolDefinitionError(
@ -182,8 +183,10 @@ class ToolCatalog(BaseModel):
)
except ImportError as e:
raise ToolDefinitionError(f"Could not import module {module_name}. Reason: {e}")
self.add_tool(tool_func, toolkit, module)
except TypeError as e:
raise ToolDefinitionError(
f"Type error encountered while adding tool {tool_name} from {module_name}. Reason: {e}"
)
def __getitem__(self, name: FullyQualifiedName) -> MaterializedTool:
return self.get_tool(name)

35
arcade/tests/core/test_catalog.py
View file

@ -1,6 +1,9 @@
from unittest.mock import MagicMock, patch
import pytest
from arcade.core.catalog import ToolCatalog
from arcade.core.errors import ToolDefinitionError
from arcade.core.schema import FullyQualifiedName
from arcade.core.toolkit import Toolkit
from arcade.sdk import tool
@ -67,3 +70,35 @@ def test_get_tool(toolkit_version: str | None, expected_tool):
tool = catalog.get_tool(fq_name)
assert tool.tool == expected_tool
def test_add_toolkit_type_error():
catalog = ToolCatalog()
# Create a mock toolkit with an invalid tool
class InvalidTool:
pass
mock_toolkit = Toolkit(
name="mock_toolkit",
description="A mock toolkit",
version="0.0.1",
package_name="mock_toolkit",
)
mock_toolkit.tools = {"mock_module": ["invalid_tool"]}
# Mock the import_module and getattr functions
with (
patch("arcade.core.catalog.import_module") as mock_import,
patch("arcade.core.catalog.getattr") as mock_getattr,
):
mock_import.return_value = MagicMock()
mock_getattr.return_value = InvalidTool()
# Assert that ToolDefinitionError is raised with the correct message
with pytest.raises(ToolDefinitionError) as exc_info:
catalog.add_toolkit(mock_toolkit)
assert "Type error encountered while adding tool invalid_tool from mock_module" in str(
exc_info.value
)

346
toolkits/google/arcade_google/tools/calendar.py Normal file
View file

@ -0,0 +1,346 @@
from datetime import datetime, timedelta
from typing import Annotated
from zoneinfo import ZoneInfo
from google.oauth2.credentials import Credentials
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from arcade.core.errors import RetryableToolError, ToolExecutionError
from arcade.core.schema import ToolContext
from arcade.sdk import tool
from arcade.sdk.auth import Google
from arcade_google.tools.models import Day, EventVisibility, SendUpdatesOptions, TimeSlot
from arcade_google.tools.utils import _update_datetime
@tool(
requires_auth=Google(
scopes=[
"https://www.googleapis.com/auth/calendar.readonly",
"https://www.googleapis.com/auth/calendar.events",
],
)
)
async def create_event(
context: ToolContext,
summary: Annotated[str, "The title of the event"],
start_date: Annotated[Day, "The day that the event starts"],
start_time: Annotated[TimeSlot, "The time of the day that the event starts"],
end_date: Annotated[Day, "The day that the event ends"],
end_time: Annotated[TimeSlot, "The time of the day that the event ends"],
calendar_id: Annotated[
str, "The ID of the calendar to create the event in, usually 'primary'"
] = "primary",
description: Annotated[str | None, "The description of the event"] = None,
location: Annotated[str | None, "The location of the event"] = None,
visibility: Annotated[EventVisibility, "The visibility of the event"] = EventVisibility.DEFAULT,
attendee_emails: Annotated[
list[str] | None,
"The list of attendee emails. Must be valid email addresses e.g., username@domain.com",
] = None,
) -> Annotated[dict, "A dictionary containing the created event details"]:
"""Create a new event/meeting/sync/meetup in the specified calendar."""
service = build("calendar", "v3", credentials=Credentials(context.authorization.token))
try:
# Get the calendar's time zone
calendar = service.calendars().get(calendarId=calendar_id).execute()
time_zone = calendar["timeZone"]
# Convert enum values to datetime objects
start_datetime = datetime.combine(start_date.to_date(time_zone), start_time.to_time())
end_datetime = datetime.combine(end_date.to_date(time_zone), end_time.to_time())
event = {
"summary": summary,
"description": description,
"location": location,
"start": {"dateTime": start_datetime.isoformat(), "timeZone": time_zone},
"end": {"dateTime": end_datetime.isoformat(), "timeZone": time_zone},
"visibility": visibility.value,
}
if attendee_emails:
event["attendees"] = [{"email": email} for email in attendee_emails]
created_event = service.events().insert(calendarId=calendar_id, body=event).execute()
except HttpError as e:
raise ToolExecutionError(
f"HttpError during execution of '{create_event.__name__}' tool.", str(e)
)
except Exception as e:
raise ToolExecutionError(
f"Unexpected Error encountered during execution of '{create_event.__name__}' tool.",
str(e),
)
else:
return {"event": created_event}
@tool(
requires_auth=Google(
scopes=["https://www.googleapis.com/auth/calendar.events.readonly"],
)
)
async def list_events(
context: ToolContext,
min_day: Annotated[
Day, "Filter by events that end on or after this day. Combined with min_time_slot"
],
min_time_slot: Annotated[
TimeSlot, "Filter by events that end after this time. Combined with min_day"
],
max_day: Annotated[
Day, "Filter by events that start on or before this day. Combined with max_time_slot"
],
max_time_slot: Annotated[
TimeSlot, "Filter by events that start before this time. Combined with max_day"
],
calendar_id: Annotated[str, "The ID of the calendar to list events from"] = "primary",
max_results: Annotated[int, "The maximum number of events to return"] = 10,
) -> Annotated[dict, "A dictionary containing the list of events"]:
"""
List events from the specified calendar within the given date range.
min_day and min_time_slot are combined to form the lower bound (exclusive) for an event's end time to filter by
max_day and max_time_slot are combined to form the upper bound (exclusive) for an event's start time to filter by
For example:
If min_day is set to Day.TODAY and min_time_slot is set to TimeSlot._09:00,
and max_day is set to Day.TOMORROW and max_time_slot is set to TimeSlot._17:00,
the function will return events that:
1. End after 09:00 today (exclusive)
2. Start before 17:00 tomorrow (exclusive)
This means an event starting at 08:00 today and ending at 10:00 today would be included,
but an event starting at 17:00 tomorrow would not be included.
"""
service = build("calendar", "v3", credentials=Credentials(context.authorization.token))
try:
# Get the calendar's time zone
calendar = service.calendars().get(calendarId=calendar_id).execute()
time_zone = calendar["timeZone"]
# Convert enum values to datetime with timezone offset
start_datetime = datetime.combine(
min_day.to_date(time_zone), min_time_slot.to_time()
).astimezone(ZoneInfo(time_zone))
end_datetime = datetime.combine(
max_day.to_date(time_zone), max_time_slot.to_time()
).astimezone(ZoneInfo(time_zone))
if start_datetime > end_datetime:
start_datetime, end_datetime = end_datetime, start_datetime
events_result = (
service.events()
.list(
calendarId=calendar_id,
timeMin=start_datetime.isoformat(),
timeMax=end_datetime.isoformat(),
maxResults=max_results,
singleEvents=True,
orderBy="startTime",
)
.execute()
)
items_keys = [
"attachments",
"attendees",
"creator",
"description",
"end",
"eventType",
"htmlLink",
"id",
"location",
"organizer",
"start",
"summary",
"visibility",
]
events = [
{key: event[key] for key in items_keys if key in event}
for event in events_result.get("items", [])
]
return {"events_count": len(events), "events": events}
except HttpError as e:
raise ToolExecutionError(
f"HttpError during execution of '{list_events.__name__}' tool.", str(e)
)
except Exception as e:
raise ToolExecutionError(
f"Unexpected Error encountered during execution of '{list_events.__name__}' tool.",
str(e),
)
@tool(
requires_auth=Google(
scopes=["https://www.googleapis.com/auth/calendar"],
)
)
async def update_event(
context: ToolContext,
event_id: Annotated[str, "The ID of the event to update"],
updated_start_day: Annotated[
Day | None,
"The updated day that the event starts. Combined with updated_start_time to form the new start time",
] = None,
updated_start_time: Annotated[
TimeSlot | None,
"The updated time that the event starts. Combined with updated_start_day to form the new start time",
] = None,
updated_end_day: Annotated[
Day | None,
"The updated day that the event ends. Combined with updated_end_time to form the new end time",
] = None,
updated_end_time: Annotated[TimeSlot | None, "The updated time that the event ends"] = None,
updated_calendar_id: Annotated[
str | None, "The updated ID of the calendar containing the event"
] = None,
updated_summary: Annotated[str | None, "The updated title of the event"] = None,
updated_description: Annotated[str | None, "The updated description of the event"] = None,
updated_location: Annotated[str | None, "The updated location of the event"] = None,
updated_visibility: Annotated[EventVisibility | None, "The visibility of the event"] = None,
attendee_emails_to_add: Annotated[
list[str] | None,
"The list of updated attendee emails to add. Must be valid email addresses e.g., username@domain.com",
] = None,
attendee_emails_to_remove: Annotated[
list[str] | None,
"The list of attendee emails to remove. Must be valid email addresses e.g., username@domain.com",
] = None,
send_updates: Annotated[
SendUpdatesOptions, "Guests who should receive notifications about the event update"
] = SendUpdatesOptions.ALL,
) -> Annotated[
str,
"A string containing the updated event details, including the event ID, update timestamp, and a link to view the updated event",
]:
"""
Update an existing event in the specified calendar with the provided details.
Only the provided fields will be updated; others will remain unchanged.
`updated_start_day` and `updated_start_time` must be provided together.
`updated_end_day` and `updated_end_time` must be provided together.
"""
service = build("calendar", "v3", credentials=Credentials(context.authorization.token))
try:
calendar = service.calendars().get(calendarId="primary").execute()
time_zone = calendar["timeZone"]
try:
event = service.events().get(calendarId="primary", eventId=event_id).execute()
except HttpError: # TODO: This is a first pass. We should do better.
valid_events_with_id = (
service.events()
.list(
calendarId="primary",
timeMin=(datetime.now() - timedelta(days=2)).isoformat(),
timeMax=(datetime.now() - timedelta(days=2)).isoformat(),
maxResults=50,
singleEvents=True,
orderBy="startTime",
)
.execute()
)
raise RetryableToolError(
f"Event with ID {event_id} not found.",
additional_prompt_content=f"Here is list of valid events. The event_id parameter must match one of these: {valid_events_with_id}",
retry_after_ms=1000,
developer_message=f"Event with ID {event_id} not found. Please try again with a valid event ID.",
)
update_fields = {
"start": _update_datetime(updated_start_day, updated_start_time, time_zone),
"end": _update_datetime(updated_end_day, updated_end_time, time_zone),
"calendarId": updated_calendar_id,
"sendUpdates": send_updates.value if send_updates else None,
"summary": updated_summary,
"description": updated_description,
"location": updated_location,
"visibility": updated_visibility.value if updated_visibility else None,
}
event.update({k: v for k, v in update_fields.items() if v is not None})
if attendee_emails_to_remove:
event["attendees"] = [
attendee
for attendee in event.get("attendees", [])
if attendee.get("email", "") not in attendee_emails_to_remove
]
if attendee_emails_to_add:
event["attendees"] = event.get("attendees", []) + [
{"email": email} for email in attendee_emails_to_add
]
updated_event = (
service.events()
.update(
calendarId="primary",
eventId=event_id,
sendUpdates=send_updates.value,
body=event,
)
.execute()
)
return f"Event with ID {event_id} successfully updated at {updated_event['updated']}. View updated event at {updated_event['htmlLink']}"
except HttpError as e:
raise ToolExecutionError(
f"HttpError during execution of '{update_event.__name__}' tool.", str(e)
)
except Exception as e:
raise ToolExecutionError(
f"Unexpected Error encountered during execution of '{update_event.__name__}' tool.",
str(e),
)
@tool(
requires_auth=Google(
scopes=["https://www.googleapis.com/auth/calendar.events"],
)
)
async def delete_event(
context: ToolContext,
event_id: Annotated[str, "The ID of the event to delete"],
calendar_id: Annotated[str, "The ID of the calendar containing the event"] = "primary",
send_updates: Annotated[
SendUpdatesOptions, "Specifies which attendees to notify about the deletion"
] = SendUpdatesOptions.ALL,
) -> Annotated[str, "A string containing the deletion confirmation message"]:
"""Delete an event from Google Calendar."""
service = build("calendar", "v3", credentials=Credentials(context.authorization.token))
try:
service.events().delete(
calendarId=calendar_id, eventId=event_id, sendUpdates=send_updates.value
).execute()
notification_message = ""
if send_updates == SendUpdatesOptions.ALL:
notification_message = "Notifications were sent to all attendees."
elif send_updates == SendUpdatesOptions.EXTERNAL_ONLY:
notification_message = "Notifications were sent to external attendees only."
elif send_updates == SendUpdatesOptions.NONE:
notification_message = "No notifications were sent to attendees."
except HttpError as e:
raise ToolExecutionError(
f"HttpError during execution of '{delete_event.__name__}' tool.", str(e)
)
except Exception as e:
raise ToolExecutionError(
f"Unexpected Error encountered during execution of '{delete_event.__name__}' tool.",
str(e),
)
else:
return f"Event with ID '{event_id}' successfully deleted from calendar '{calendar_id}'. {notification_message}"

232
toolkits/google/arcade_google/tools/models.py Normal file
View file

@ -0,0 +1,232 @@
from datetime import datetime, timedelta
from enum import Enum
from zoneinfo import ZoneInfo
# ---------------------------------------------------------------------------- #
# Google Calendar Models and Enums
# ---------------------------------------------------------------------------- #
class DateRange(Enum):
TODAY = "today"
TOMORROW = "tomorrow"
THIS_WEEK = "this_week"
NEXT_WEEK = "next_week"
THIS_MONTH = "this_month"
NEXT_MONTH = "next_month"
def to_date_range(self):
today = datetime.now().date()
if self == DateRange.TODAY:
return today, today + timedelta(days=1)
elif self == DateRange.TOMORROW:
return today + timedelta(days=1), today + timedelta(days=2)
elif self == DateRange.THIS_WEEK:
start = today - timedelta(days=today.weekday())
return start, start + timedelta(days=7)
elif self == DateRange.NEXT_WEEK:
start = today + timedelta(days=7 - today.weekday())
return start, start + timedelta(days=7)
elif self == DateRange.THIS_MONTH:
start = today.replace(day=1)
next_month = start + timedelta(days=32)
end = next_month.replace(day=1)
return start, end
elif self == DateRange.NEXT_MONTH:
start = (today.replace(day=1) + timedelta(days=32)).replace(day=1)
next_month = start + timedelta(days=32)
end = next_month.replace(day=1)
return start, end
def to_datetime_range(self, time_zone_name: str | None = None) -> tuple[datetime, datetime]:
start_date, end_date = self.to_date_range()
# time_zone = ZoneInfo(time_zone_name)
start_datetime = datetime.combine(
start_date, datetime.min.time()
) # .replace(tzinfo=time_zone)
end_datetime = datetime.combine(end_date, datetime.min.time()) # .replace(tzinfo=time_zone)
return start_datetime, end_datetime
class Day(Enum):
# TODO: THere are obvious limitations here. We should do better and support any date.
YESTERDAY = "yesterday"
TODAY = "today"
TOMORROW = "tomorrow"
THIS_SUNDAY = "this_sunday"
THIS_MONDAY = "this_monday"
THIS_TUESDAY = "this_tuesday"
THIS_WEDNESDAY = "this_wednesday"
THIS_THURSDAY = "this_thursday"
THIS_FRIDAY = "this_friday"
THIS_SATURDAY = "this_saturday"
NEXT_SUNDAY = "next_sunday"
NEXT_MONDAY = "next_monday"
NEXT_TUESDAY = "next_tuesday"
NEXT_WEDNESDAY = "next_wednesday"
NEXT_THURSDAY = "next_thursday"
NEXT_FRIDAY = "next_friday"
NEXT_SATURDAY = "next_saturday"
def to_date(self, time_zone_name: str):
time_zone = ZoneInfo(time_zone_name)
today = datetime.now(time_zone).date()
weekday = today.weekday()
if self == Day.YESTERDAY:
return today - timedelta(days=1)
elif self == Day.TODAY:
return today
elif self == Day.TOMORROW:
return today + timedelta(days=1)
day_offsets = {
Day.THIS_SUNDAY: 6,
Day.THIS_MONDAY: 0,
Day.THIS_TUESDAY: 1,
Day.THIS_WEDNESDAY: 2,
Day.THIS_THURSDAY: 3,
Day.THIS_FRIDAY: 4,
Day.THIS_SATURDAY: 5,
}
if self in day_offsets:
return today + timedelta(days=(day_offsets[self] - weekday) % 7)
next_week_offsets = {
Day.NEXT_SUNDAY: 6,
Day.NEXT_MONDAY: 0,
Day.NEXT_TUESDAY: 1,
Day.NEXT_WEDNESDAY: 2,
Day.NEXT_THURSDAY: 3,
Day.NEXT_FRIDAY: 4,
Day.NEXT_SATURDAY: 5,
}
if self in next_week_offsets:
return today + timedelta(days=(next_week_offsets[self] - weekday + 7) % 7)
raise ValueError(f"Invalid Day enum value: {self}")
class TimeSlot(Enum):
_0000 = "00:00"
_0015 = "00:15"
_0030 = "00:30"
_0045 = "00:45"
_0100 = "01:00"
_0115 = "01:15"
_0130 = "01:30"
_0145 = "01:45"
_0200 = "02:00"
_0215 = "02:15"
_0230 = "02:30"
_0245 = "02:45"
_0300 = "03:00"
_0315 = "03:15"
_0330 = "03:30"
_0345 = "03:45"
_0400 = "04:00"
_0415 = "04:15"
_0430 = "04:30"
_0445 = "04:45"
_0500 = "05:00"
_0515 = "05:15"
_0530 = "05:30"
_0545 = "05:45"
_0600 = "06:00"
_0615 = "06:15"
_0630 = "06:30"
_0645 = "06:45"
_0700 = "07:00"
_0715 = "07:15"
_0730 = "07:30"
_0745 = "07:45"
_0800 = "08:00"
_0815 = "08:15"
_0830 = "08:30"
_0845 = "08:45"
_0900 = "09:00"
_0915 = "09:15"
_0930 = "09:30"
_0945 = "09:45"
_1000 = "10:00"
_1015 = "10:15"
_1030 = "10:30"
_1045 = "10:45"
_1100 = "11:00"
_1115 = "11:15"
_1130 = "11:30"
_1145 = "11:45"
_1200 = "12:00"
_1215 = "12:15"
_1230 = "12:30"
_1245 = "12:45"
_1300 = "13:00"
_1315 = "13:15"
_1330 = "13:30"
_1345 = "13:45"
_1400 = "14:00"
_1415 = "14:15"
_1430 = "14:30"
_1445 = "14:45"
_1500 = "15:00"
_1515 = "15:15"
_1530 = "15:30"
_1545 = "15:45"
_1600 = "16:00"
_1615 = "16:15"
_1630 = "16:30"
_1645 = "16:45"
_1700 = "17:00"
_1715 = "17:15"
_1730 = "17:30"
_1745 = "17:45"
_1800 = "18:00"
_1815 = "18:15"
_1830 = "18:30"
_1845 = "18:45"
_1900 = "19:00"
_1915 = "19:15"
_1930 = "19:30"
_1945 = "19:45"
_2000 = "20:00"
_2015 = "20:15"
_2030 = "20:30"
_2045 = "20:45"
_2100 = "21:00"
_2115 = "21:15"
_2130 = "21:30"
_2145 = "21:45"
_2200 = "22:00"
_2215 = "22:15"
_2230 = "22:30"
_2245 = "22:45"
_2300 = "23:00"
_2315 = "23:15"
_2330 = "23:30"
_2345 = "23:45"
def to_time(self):
return datetime.strptime(self.value, "%H:%M").time()
class EventVisibility(Enum):
DEFAULT = "default"
PUBLIC = "public"
PRIVATE = "private"
CONFIDENTIAL = "confidential"
class EventType(Enum):
BIRTHDAY = "birthday" # Special all-day events with an annual recurrence.
DEFAULT = "default" # Regular events
FOCUS_TIME = "focusTime" # Focus time events
FROM_GMAIL = "fromGmail" # Events from Gmail
OUT_OF_OFFICE = "outOfOffice" # Out of office events
WORKING_LOCATION = "workingLocation" # Working location events
class SendUpdatesOptions(Enum):
NONE = "none" # No notifications are sent
ALL = "all" # Notifications are sent to all guests
EXTERNAL_ONLY = "externalOnly" # Notifications are sent to non-Google Calendar guests only.

20
toolkits/google/arcade_google/tools/utils.py
View file

@ -6,6 +6,8 @@ from typing import Any, Optional
from bs4 import BeautifulSoup
from arcade_google.tools.models import Day, TimeSlot
class DateRange(Enum):
TODAY = "today"
@ -176,6 +178,24 @@ def _clean_text(text: str) -> str:
return text
def _update_datetime(day: Day | None, time: TimeSlot | None, time_zone: str) -> dict | None:
"""
Update the datetime for a Google Calendar event.
Args:
day (Day | None): The day of the event.
time (TimeSlot | None): The time of the event.
time_zone (str): The time zone of the event.
Returns:
dict | None: The updated datetime for the event.
"""
if day and time:
dt = datetime.combine(day.to_date(time_zone), time.to_time())
return {"dateTime": dt.isoformat(), "timeZone": time_zone}
return None
def build_query_string(sender, recipient, subject, body, date_range):
"""
Helper function to build a query string for Gmail list_emails_by_header tool.

176
toolkits/google/evals/eval_google_calendar.py Normal file
View file

@ -0,0 +1,176 @@
import arcade_google
from arcade_google.tools.calendar import create_event, delete_event, list_events, update_event
from arcade_google.tools.models import Day, EventVisibility, TimeSlot
from arcade.core.catalog import ToolCatalog
from arcade.sdk.eval import (
BinaryCritic,
EvalRubric,
EvalSuite,
tool_eval,
)
# Evaluation rubric
rubric = EvalRubric(
fail_threshold=0.9,
warn_threshold=0.95,
)
catalog = ToolCatalog()
catalog.add_module(arcade_google)
history_after_list_events = [
{"role": "user", "content": "do i have any events on my calendar for today?"},
{
"role": "assistant",
"content": "Please go to this URL and authorize the action: \n[Link](https://accounts.google.com/o/oauth2/v2/auth?)",
},
{
"role": "assistant",
"content": "",
"tool_calls": [
{
"id": "call_uHdRlr4z7sFm39ZrPsE5wcdT",
"type": "function",
"function": {
"name": "Google_ListEvents",
"arguments": '{"max_day":"today","max_time_slot":"23:45","min_day":"today","min_time_slot":"00:00"}',
},
}
],
},
{
"role": "tool",
"content": '{"events_count": 3, "events": [{"creator": {"email": "john@example.com", "self": true}, "description": "1:1 meeting with Joe", "end": {"dateTime": "2024-09-26T00:15:00-07:00", "timeZone": "America/Los_Angeles"}, "eventType": "default", "htmlLink": "https://www.google.com/calendar/event?eid=01234", "id": "10009199283838467", "location": "622 Rainbow Ave, South San Francisco, CA 94080, USA", "organizer": {"email": "john@example.com", "self": true}, "start": {"dateTime": "2024-09-25T23:15:00-07:00", "timeZone": "America/Los_Angeles"}, "summary": "1:1 meeting"}, {"attendees": [{"email": "joe@example.com", "responseStatus": "accepted"}], "creator": {"email": "john@example.com", "self": true}, "description": "This is just a test", "end": {"dateTime": "2024-09-26T14:00:00-07:00", "timeZone": "America/Los_Angeles"}, "eventType": "default", "htmlLink": "https://www.google.com/calendar/event?eid=OXB2OGFwcmZraWk1N234324", "id": "00099992228181818181", "organizer": {"email": "john@example.com", "self": true}, "start": {"dateTime": "2024-09-26T12:00:00-07:00", "timeZone": "America/Los_Angeles"}, "summary": "API test"}, {"attendees": [{"email": "henry@example.com", "responseStatus": "needsAction"}], "creator": {"email": "john@example.com", "self": true}, "end": {"dateTime": "2024-09-26T17:00:00-07:00", "timeZone": "America/Los_Angeles"}, "eventType": "default", "htmlLink": "https://www.google.com/calendar/event?eid=Z3I1ZzE4b324534556", "id": "gr5g18lf88tfpp3vkareukkc7g", "location": "611 Rainbow Road", "organizer": {"email": "john@example.com", "self": true}, "start": {"dateTime": "2024-09-26T15:00:00-07:00", "timeZone": "America/Los_Angeles"}, "summary": "Focus Time", "visibility": "public"}]}',
"tool_call_id": "call_uHdRlr4z7sFm39ZrPsE5wcdT",
"name": "Google_ListEvents",
},
{
"role": "assistant",
"content": "Yes, you have three events on your calendar for today:\n\n1. **Event:** Test2\n - **Time:** 23:15 - 00:15 (PST)\n - **Location:** 611 Gateway Blvd, South San Francisco, CA 94080, USA\n - **Description:** 1:1 meeting with Joe\n 2. **Event:** API Test\n - **Time:** 12:00 - 14:00 (PST)\n **Description:** This is just a test\n - [View Event](https://www.google.com/calendar/event?eid=OXB2OGFwcmZraWk1NGUwa24xaTNya2lvZDggZXJpY0BhcmNhZGUtYWkuY29t)\n\n3. **Event:** Focus Time\n - **Time:** 15:00 - 17:00 (PST)\n - **Location:** 611 Rainbow Road\n - **Visibility:** Public\n - [View Event](https://www.google.com/calendar/event?eid=Z3I1ZzE4bGY4OHRmcHAzdmthcmV1a2tjN2cgZXJpY0BhcmNhZGUtYWkuY29t)\n\nIf you need more details or help with anything else, feel free to ask!",
},
]
@tool_eval()
def calendar_eval_suite() -> EvalSuite:
"""Create an evaluation suite for Calendar tools."""
suite = EvalSuite(
name="Calendar Tools Evaluation",
system_message="You are an AI assistant that can create and list events using the provided tools.",
catalog=catalog,
rubric=rubric,
)
# Cases for create_event
suite.add_case(
name="Create calendar event",
user_message="Create a meeting for 'Team Meeting' starting next thursday from 11:45pm to 12:15am. Invite johndoe@example.com",
expected_tool_calls=[
(
create_event,
{
"summary": "Team Meeting",
"start_date": Day.NEXT_THURSDAY.value,
"start_time": TimeSlot._2345.value,
"end_date": Day.NEXT_FRIDAY.value,
"end_time": TimeSlot._0015.value,
"calendar_id": "primary",
"attendee_emails": ["johndoe@example.com"],
"description": None,
"location": None,
"visibility": EventVisibility.DEFAULT,
},
)
],
critics=[
BinaryCritic(critic_field="summary", weight=0.15),
BinaryCritic(critic_field="start_date", weight=0.15),
BinaryCritic(critic_field="start_time", weight=0.15),
BinaryCritic(critic_field="end_date", weight=0.15),
BinaryCritic(critic_field="end_time", weight=0.15),
BinaryCritic(critic_field="attendee_emails", weight=0.15),
BinaryCritic(critic_field="description", weight=0.1),
],
)
# Cases for list_events
suite.add_case(
name="List calendar events",
user_message="Do I have any events on my calendar today?",
expected_tool_calls=[
(
list_events,
{
"min_day": Day.TODAY.value,
"min_time_slot": TimeSlot._0000.value,
"max_day": Day.TOMORROW.value,
"max_time_slot": TimeSlot._0000.value,
"calendar_id": "primary",
"event_types": None,
"max_results": 10,
},
)
],
critics=[
BinaryCritic(critic_field="min_day", weight=0.1),
BinaryCritic(critic_field="min_time_slot", weight=0.1),
BinaryCritic(critic_field="max_day", weight=0.1),
BinaryCritic(critic_field="max_time_slot", weight=0.1),
BinaryCritic(critic_field="calendar_id", weight=0.1),
BinaryCritic(critic_field="event_types", weight=0.1),
BinaryCritic(critic_field="max_results", weight=0.1),
],
)
# Cases for update_event
suite.add_case(
name="Update a calendar event",
user_message="Oh no! I cant make it to the API Test since i have lunch with an old friend at that time. Change the meeting to 3pm to 4pm please.",
expected_tool_calls=[
(
update_event,
{
"event_id": "00099992228181818181",
"updated_start_day": Day.TODAY.value,
"updated_start_time": TimeSlot._1500.value,
"updated_end_day": Day.TODAY.value,
"updated_end_time": TimeSlot._1600.value,
},
)
],
critics=[
BinaryCritic(critic_field="event_id", weight=0.2),
BinaryCritic(critic_field="updated_start_day", weight=0.1),
BinaryCritic(critic_field="updated_start_time", weight=0.1),
BinaryCritic(critic_field="updated_end_day", weight=0.1),
BinaryCritic(critic_field="updated_end_time", weight=0.1),
],
additional_messages=history_after_list_events,
)
# Cases for delete_event
suite.add_case(
name="Delete a calendar event",
user_message="I don't need to have focus time today. Please delete it from my calendar. Don't send any notifications.",
expected_tool_calls=[
(
delete_event,
{
"event_id": "gr5g18lf88tfpp3vkareukkc7g",
"calendar_id": "primary",
"send_updates": "none",
},
)
],
critics=[
BinaryCritic(critic_field="event_id", weight=0.6),
BinaryCritic(critic_field="calendar_id", weight=0.2),
BinaryCritic(critic_field="send_updates", weight=0.2),
],
additional_messages=history_after_list_events,
)
return suite

60
toolkits/google/evals/eval_google_gmail.py Normal file
View file

@ -0,0 +1,60 @@
import arcade_google
from arcade_google.tools.gmail import (
send_email,
)
from arcade.core.catalog import ToolCatalog
from arcade.sdk.eval import (
BinaryCritic,
EvalRubric,
EvalSuite,
SimilarityCritic,
tool_eval,
)
# Evaluation rubric
rubric = EvalRubric(
fail_threshold=0.9,
warn_threshold=0.95,
)
catalog = ToolCatalog()
catalog.add_module(arcade_google)
@tool_eval()
def gmail_eval_suite() -> EvalSuite:
"""Create an evaluation suite for Gmail tools."""
suite = EvalSuite(
name="Gmail Tools Evaluation",
system_message="You are an AI assistant that can send and manage emails using the provided tools.",
catalog=catalog,
rubric=rubric,
)
suite.add_case(
name="Send email to user with clear username",
user_message="Send a email to johndoe@example.com saying 'Hello, can we meet at 3 PM?'. CC his boss janedoe@example.com",
expected_tool_calls=[
(
send_email,
{
"subject": "Meeting Request",
"body": "Hello, can we meet at 3 PM?",
"recipient": "johndoe@example.com",
"cc": ["janedoe@example.com"],
"bcc": None,
},
)
],
critics=[
SimilarityCritic(critic_field="subject", weight=0.125),
SimilarityCritic(critic_field="body", weight=0.25),
BinaryCritic(critic_field="recipient", weight=0.25),
BinaryCritic(critic_field="cc", weight=0.25),
BinaryCritic(critic_field="bcc", weight=0.125),
],
)
return suite

159
toolkits/google/tests/test_calendar.py Normal file
View file

@ -0,0 +1,159 @@
from unittest.mock import MagicMock, patch
import pytest
from arcade_google.tools.calendar import create_event, delete_event, list_events, update_event
from arcade_google.tools.models import Day, EventVisibility, SendUpdatesOptions, TimeSlot
from googleapiclient.errors import HttpError
from arcade.core.errors import ToolExecutionError
from arcade.core.schema import ToolAuthorizationContext, ToolContext
@pytest.fixture
def mock_context():
mock_auth = ToolAuthorizationContext(token="fake-token") # noqa: S106
return ToolContext(authorization=mock_auth)
@pytest.mark.asyncio
@patch("arcade_google.tools.calendar.build")
async def test_create_event(mock_build, mock_context):
mock_service = MagicMock()
mock_build.return_value = mock_service
# Mock the calendar's time zone
mock_service.calendars().get().execute.return_value = {"timeZone": "America/Los_Angeles"}
# Case: HttpError
mock_service.events().insert().execute.side_effect = HttpError(
resp=MagicMock(status=400),
content=b'{"error": {"message": "Invalid request"}}',
)
with pytest.raises(ToolExecutionError):
await create_event(
context=mock_context,
summary="Test Event",
start_date=Day.TODAY,
start_time=TimeSlot._1615,
end_date=Day.TODAY,
end_time=TimeSlot._1715,
description="Test Description",
location="Test Location",
visibility=EventVisibility.PRIVATE,
attendee_emails=["test@example.com"],
)
@pytest.mark.asyncio
@patch("arcade_google.tools.calendar.build")
async def test_list_events(mock_build, mock_context):
mock_service = MagicMock()
mock_build.return_value = mock_service
# Mock the calendar's time zone
mock_service.calendars().get().execute.return_value = {"timeZone": "America/Los_Angeles"}
# Case: min time is after max time. list_events tool should swap the times and still return the events
mock_events_list_response = {
"items": [
{
"creator": {"email": "example@arcade-ai.com", "self": True},
"end": {"dateTime": "2024-09-27T01:00:00-07:00", "timeZone": "America/Los_Angeles"},
"eventType": "default",
"htmlLink": "https://www.google.com/calendar/event?eid=N2pmYjZ0ZmNnMGNydG5scmhkY2JvZWc4OGIgZXJpY0BhcmNhZGUtYWku",
"id": "7jfb6tfcg0crtnlrhdcboeg88b",
"organizer": {"email": "example@arcade-ai.com", "self": True},
"start": {
"dateTime": "2024-09-27T00:00:00-07:00",
"timeZone": "America/Los_Angeles",
},
"summary": "teST",
},
{
"creator": {"email": "example@arcade-ai.com", "self": True},
"end": {"dateTime": "2024-09-27T17:00:00-07:00", "timeZone": "America/Los_Angeles"},
"eventType": "default",
"htmlLink": "https://www.google.com/calendar/event?eid=MjZvYnRoc2xtMWMzbG5mdG10bzk4cDcxaGMgZXJpY0BhcmNhZGUtYWku",
"id": "26obthslm1c3lnftmto98p71hc",
"organizer": {"email": "example@arcade-ai.com", "self": True},
"start": {
"dateTime": "2024-09-27T14:00:00-07:00",
"timeZone": "America/Los_Angeles",
},
"summary": "New Event",
},
]
}
expected_tool_response = {
"events_count": len(mock_events_list_response["items"]),
"events": mock_events_list_response["items"],
}
mock_service.events().list().execute.return_value = mock_events_list_response
message = await list_events(
context=mock_context,
min_day=Day.TODAY,
min_time_slot=TimeSlot._1615,
max_day=Day.TODAY,
max_time_slot=TimeSlot._1515,
)
assert message == expected_tool_response
# Case: HttpError
mock_service.events().list().execute.side_effect = HttpError(
resp=MagicMock(status=400),
content=b'{"error": {"message": "Invalid request"}}',
)
with pytest.raises(ToolExecutionError):
await list_events(
context=mock_context,
min_day=Day.TODAY,
min_time_slot=TimeSlot._1615,
max_day=Day.TOMORROW,
max_time_slot=TimeSlot._1815,
)
@pytest.mark.asyncio
@patch("arcade_google.tools.calendar.build")
async def test_update_event(mock_build, mock_context):
mock_service = MagicMock()
mock_build.return_value = mock_service
mock_service.events().update().execute.side_effect = HttpError(
resp=MagicMock(status=400),
content=b'{"error": {"message": "Event not found"}}',
)
with pytest.raises(ToolExecutionError):
await update_event(
context=mock_context,
event_id="1234567890",
updated_start_day=Day.NEXT_FRIDAY,
updated_start_time=TimeSlot._0015,
updated_end_day=Day.NEXT_FRIDAY,
updated_end_time=TimeSlot._0115,
updated_summary="Updated Event",
updated_description="Updated Description",
updated_location="Updated Location",
updated_visibility=EventVisibility.PRIVATE,
attendee_emails_to_add=["test@example.com"],
attendee_emails_to_remove=["test@example2.com"],
send_updates=SendUpdatesOptions.ALL,
)
@pytest.mark.asyncio
@patch("arcade_google.tools.calendar.build")
async def test_delete_event(mock_build, mock_context):
mock_service = MagicMock()
mock_build.return_value = mock_service
mock_service.events().delete().execute.side_effect = HttpError(
resp=MagicMock(status=400),
content=b'{"error": {"message": "Event not found"}}',
)
with pytest.raises(ToolExecutionError):
await delete_event(
context=mock_context,
event_id="nonexistent_event",
)