diff --git a/arcade/arcade/core/catalog.py b/arcade/arcade/core/catalog.py index ba9fd566..ee64252f 100644 --- a/arcade/arcade/core/catalog.py +++ b/arcade/arcade/core/catalog.py @@ -175,6 +175,7 @@ class ToolCatalog(BaseModel): try: module = import_module(module_name) tool_func = getattr(module, tool_name) + self.add_tool(tool_func, toolkit, module) except AttributeError: raise ToolDefinitionError( @@ -182,8 +183,10 @@ class ToolCatalog(BaseModel): ) except ImportError as e: raise ToolDefinitionError(f"Could not import module {module_name}. Reason: {e}") - - self.add_tool(tool_func, toolkit, module) + except TypeError as e: + raise ToolDefinitionError( + f"Type error encountered while adding tool {tool_name} from {module_name}. Reason: {e}" + ) def __getitem__(self, name: FullyQualifiedName) -> MaterializedTool: return self.get_tool(name) diff --git a/arcade/tests/core/test_catalog.py b/arcade/tests/core/test_catalog.py index 70288176..4a894a97 100644 --- a/arcade/tests/core/test_catalog.py +++ b/arcade/tests/core/test_catalog.py @@ -1,6 +1,9 @@ +from unittest.mock import MagicMock, patch + import pytest from arcade.core.catalog import ToolCatalog +from arcade.core.errors import ToolDefinitionError from arcade.core.schema import FullyQualifiedName from arcade.core.toolkit import Toolkit from arcade.sdk import tool @@ -67,3 +70,35 @@ def test_get_tool(toolkit_version: str | None, expected_tool): tool = catalog.get_tool(fq_name) assert tool.tool == expected_tool + + +def test_add_toolkit_type_error(): + catalog = ToolCatalog() + + # Create a mock toolkit with an invalid tool + class InvalidTool: + pass + + mock_toolkit = Toolkit( + name="mock_toolkit", + description="A mock toolkit", + version="0.0.1", + package_name="mock_toolkit", + ) + mock_toolkit.tools = {"mock_module": ["invalid_tool"]} + + # Mock the import_module and getattr functions + with ( + patch("arcade.core.catalog.import_module") as mock_import, + patch("arcade.core.catalog.getattr") as mock_getattr, + ): + mock_import.return_value = MagicMock() + mock_getattr.return_value = InvalidTool() + + # Assert that ToolDefinitionError is raised with the correct message + with pytest.raises(ToolDefinitionError) as exc_info: + catalog.add_toolkit(mock_toolkit) + + assert "Type error encountered while adding tool invalid_tool from mock_module" in str( + exc_info.value + ) diff --git a/toolkits/google/arcade_google/tools/calendar.py b/toolkits/google/arcade_google/tools/calendar.py new file mode 100644 index 00000000..19f3ab51 --- /dev/null +++ b/toolkits/google/arcade_google/tools/calendar.py @@ -0,0 +1,346 @@ +from datetime import datetime, timedelta +from typing import Annotated +from zoneinfo import ZoneInfo + +from google.oauth2.credentials import Credentials +from googleapiclient.discovery import build +from googleapiclient.errors import HttpError + +from arcade.core.errors import RetryableToolError, ToolExecutionError +from arcade.core.schema import ToolContext +from arcade.sdk import tool +from arcade.sdk.auth import Google +from arcade_google.tools.models import Day, EventVisibility, SendUpdatesOptions, TimeSlot +from arcade_google.tools.utils import _update_datetime + + +@tool( + requires_auth=Google( + scopes=[ + "https://www.googleapis.com/auth/calendar.readonly", + "https://www.googleapis.com/auth/calendar.events", + ], + ) +) +async def create_event( + context: ToolContext, + summary: Annotated[str, "The title of the event"], + start_date: Annotated[Day, "The day that the event starts"], + start_time: Annotated[TimeSlot, "The time of the day that the event starts"], + end_date: Annotated[Day, "The day that the event ends"], + end_time: Annotated[TimeSlot, "The time of the day that the event ends"], + calendar_id: Annotated[ + str, "The ID of the calendar to create the event in, usually 'primary'" + ] = "primary", + description: Annotated[str | None, "The description of the event"] = None, + location: Annotated[str | None, "The location of the event"] = None, + visibility: Annotated[EventVisibility, "The visibility of the event"] = EventVisibility.DEFAULT, + attendee_emails: Annotated[ + list[str] | None, + "The list of attendee emails. Must be valid email addresses e.g., username@domain.com", + ] = None, +) -> Annotated[dict, "A dictionary containing the created event details"]: + """Create a new event/meeting/sync/meetup in the specified calendar.""" + + service = build("calendar", "v3", credentials=Credentials(context.authorization.token)) + + try: + # Get the calendar's time zone + calendar = service.calendars().get(calendarId=calendar_id).execute() + time_zone = calendar["timeZone"] + + # Convert enum values to datetime objects + start_datetime = datetime.combine(start_date.to_date(time_zone), start_time.to_time()) + end_datetime = datetime.combine(end_date.to_date(time_zone), end_time.to_time()) + + event = { + "summary": summary, + "description": description, + "location": location, + "start": {"dateTime": start_datetime.isoformat(), "timeZone": time_zone}, + "end": {"dateTime": end_datetime.isoformat(), "timeZone": time_zone}, + "visibility": visibility.value, + } + + if attendee_emails: + event["attendees"] = [{"email": email} for email in attendee_emails] + + created_event = service.events().insert(calendarId=calendar_id, body=event).execute() + + except HttpError as e: + raise ToolExecutionError( + f"HttpError during execution of '{create_event.__name__}' tool.", str(e) + ) + except Exception as e: + raise ToolExecutionError( + f"Unexpected Error encountered during execution of '{create_event.__name__}' tool.", + str(e), + ) + else: + return {"event": created_event} + + +@tool( + requires_auth=Google( + scopes=["https://www.googleapis.com/auth/calendar.events.readonly"], + ) +) +async def list_events( + context: ToolContext, + min_day: Annotated[ + Day, "Filter by events that end on or after this day. Combined with min_time_slot" + ], + min_time_slot: Annotated[ + TimeSlot, "Filter by events that end after this time. Combined with min_day" + ], + max_day: Annotated[ + Day, "Filter by events that start on or before this day. Combined with max_time_slot" + ], + max_time_slot: Annotated[ + TimeSlot, "Filter by events that start before this time. Combined with max_day" + ], + calendar_id: Annotated[str, "The ID of the calendar to list events from"] = "primary", + max_results: Annotated[int, "The maximum number of events to return"] = 10, +) -> Annotated[dict, "A dictionary containing the list of events"]: + """ + List events from the specified calendar within the given date range. + + min_day and min_time_slot are combined to form the lower bound (exclusive) for an event's end time to filter by + max_day and max_time_slot are combined to form the upper bound (exclusive) for an event's start time to filter by + + For example: + If min_day is set to Day.TODAY and min_time_slot is set to TimeSlot._09:00, + and max_day is set to Day.TOMORROW and max_time_slot is set to TimeSlot._17:00, + the function will return events that: + 1. End after 09:00 today (exclusive) + 2. Start before 17:00 tomorrow (exclusive) + This means an event starting at 08:00 today and ending at 10:00 today would be included, + but an event starting at 17:00 tomorrow would not be included. + """ + service = build("calendar", "v3", credentials=Credentials(context.authorization.token)) + + try: + # Get the calendar's time zone + calendar = service.calendars().get(calendarId=calendar_id).execute() + time_zone = calendar["timeZone"] + + # Convert enum values to datetime with timezone offset + start_datetime = datetime.combine( + min_day.to_date(time_zone), min_time_slot.to_time() + ).astimezone(ZoneInfo(time_zone)) + end_datetime = datetime.combine( + max_day.to_date(time_zone), max_time_slot.to_time() + ).astimezone(ZoneInfo(time_zone)) + + if start_datetime > end_datetime: + start_datetime, end_datetime = end_datetime, start_datetime + + events_result = ( + service.events() + .list( + calendarId=calendar_id, + timeMin=start_datetime.isoformat(), + timeMax=end_datetime.isoformat(), + maxResults=max_results, + singleEvents=True, + orderBy="startTime", + ) + .execute() + ) + + items_keys = [ + "attachments", + "attendees", + "creator", + "description", + "end", + "eventType", + "htmlLink", + "id", + "location", + "organizer", + "start", + "summary", + "visibility", + ] + + events = [ + {key: event[key] for key in items_keys if key in event} + for event in events_result.get("items", []) + ] + + return {"events_count": len(events), "events": events} + except HttpError as e: + raise ToolExecutionError( + f"HttpError during execution of '{list_events.__name__}' tool.", str(e) + ) + except Exception as e: + raise ToolExecutionError( + f"Unexpected Error encountered during execution of '{list_events.__name__}' tool.", + str(e), + ) + + +@tool( + requires_auth=Google( + scopes=["https://www.googleapis.com/auth/calendar"], + ) +) +async def update_event( + context: ToolContext, + event_id: Annotated[str, "The ID of the event to update"], + updated_start_day: Annotated[ + Day | None, + "The updated day that the event starts. Combined with updated_start_time to form the new start time", + ] = None, + updated_start_time: Annotated[ + TimeSlot | None, + "The updated time that the event starts. Combined with updated_start_day to form the new start time", + ] = None, + updated_end_day: Annotated[ + Day | None, + "The updated day that the event ends. Combined with updated_end_time to form the new end time", + ] = None, + updated_end_time: Annotated[TimeSlot | None, "The updated time that the event ends"] = None, + updated_calendar_id: Annotated[ + str | None, "The updated ID of the calendar containing the event" + ] = None, + updated_summary: Annotated[str | None, "The updated title of the event"] = None, + updated_description: Annotated[str | None, "The updated description of the event"] = None, + updated_location: Annotated[str | None, "The updated location of the event"] = None, + updated_visibility: Annotated[EventVisibility | None, "The visibility of the event"] = None, + attendee_emails_to_add: Annotated[ + list[str] | None, + "The list of updated attendee emails to add. Must be valid email addresses e.g., username@domain.com", + ] = None, + attendee_emails_to_remove: Annotated[ + list[str] | None, + "The list of attendee emails to remove. Must be valid email addresses e.g., username@domain.com", + ] = None, + send_updates: Annotated[ + SendUpdatesOptions, "Guests who should receive notifications about the event update" + ] = SendUpdatesOptions.ALL, +) -> Annotated[ + str, + "A string containing the updated event details, including the event ID, update timestamp, and a link to view the updated event", +]: + """ + Update an existing event in the specified calendar with the provided details. + Only the provided fields will be updated; others will remain unchanged. + + `updated_start_day` and `updated_start_time` must be provided together. + `updated_end_day` and `updated_end_time` must be provided together. + """ + service = build("calendar", "v3", credentials=Credentials(context.authorization.token)) + + try: + calendar = service.calendars().get(calendarId="primary").execute() + time_zone = calendar["timeZone"] + + try: + event = service.events().get(calendarId="primary", eventId=event_id).execute() + except HttpError: # TODO: This is a first pass. We should do better. + valid_events_with_id = ( + service.events() + .list( + calendarId="primary", + timeMin=(datetime.now() - timedelta(days=2)).isoformat(), + timeMax=(datetime.now() - timedelta(days=2)).isoformat(), + maxResults=50, + singleEvents=True, + orderBy="startTime", + ) + .execute() + ) + raise RetryableToolError( + f"Event with ID {event_id} not found.", + additional_prompt_content=f"Here is list of valid events. The event_id parameter must match one of these: {valid_events_with_id}", + retry_after_ms=1000, + developer_message=f"Event with ID {event_id} not found. Please try again with a valid event ID.", + ) + + update_fields = { + "start": _update_datetime(updated_start_day, updated_start_time, time_zone), + "end": _update_datetime(updated_end_day, updated_end_time, time_zone), + "calendarId": updated_calendar_id, + "sendUpdates": send_updates.value if send_updates else None, + "summary": updated_summary, + "description": updated_description, + "location": updated_location, + "visibility": updated_visibility.value if updated_visibility else None, + } + + event.update({k: v for k, v in update_fields.items() if v is not None}) + + if attendee_emails_to_remove: + event["attendees"] = [ + attendee + for attendee in event.get("attendees", []) + if attendee.get("email", "") not in attendee_emails_to_remove + ] + if attendee_emails_to_add: + event["attendees"] = event.get("attendees", []) + [ + {"email": email} for email in attendee_emails_to_add + ] + + updated_event = ( + service.events() + .update( + calendarId="primary", + eventId=event_id, + sendUpdates=send_updates.value, + body=event, + ) + .execute() + ) + return f"Event with ID {event_id} successfully updated at {updated_event['updated']}. View updated event at {updated_event['htmlLink']}" + except HttpError as e: + raise ToolExecutionError( + f"HttpError during execution of '{update_event.__name__}' tool.", str(e) + ) + except Exception as e: + raise ToolExecutionError( + f"Unexpected Error encountered during execution of '{update_event.__name__}' tool.", + str(e), + ) + + +@tool( + requires_auth=Google( + scopes=["https://www.googleapis.com/auth/calendar.events"], + ) +) +async def delete_event( + context: ToolContext, + event_id: Annotated[str, "The ID of the event to delete"], + calendar_id: Annotated[str, "The ID of the calendar containing the event"] = "primary", + send_updates: Annotated[ + SendUpdatesOptions, "Specifies which attendees to notify about the deletion" + ] = SendUpdatesOptions.ALL, +) -> Annotated[str, "A string containing the deletion confirmation message"]: + """Delete an event from Google Calendar.""" + service = build("calendar", "v3", credentials=Credentials(context.authorization.token)) + + try: + service.events().delete( + calendarId=calendar_id, eventId=event_id, sendUpdates=send_updates.value + ).execute() + + notification_message = "" + if send_updates == SendUpdatesOptions.ALL: + notification_message = "Notifications were sent to all attendees." + elif send_updates == SendUpdatesOptions.EXTERNAL_ONLY: + notification_message = "Notifications were sent to external attendees only." + elif send_updates == SendUpdatesOptions.NONE: + notification_message = "No notifications were sent to attendees." + except HttpError as e: + raise ToolExecutionError( + f"HttpError during execution of '{delete_event.__name__}' tool.", str(e) + ) + except Exception as e: + raise ToolExecutionError( + f"Unexpected Error encountered during execution of '{delete_event.__name__}' tool.", + str(e), + ) + else: + return f"Event with ID '{event_id}' successfully deleted from calendar '{calendar_id}'. {notification_message}" diff --git a/toolkits/google/arcade_google/tools/models.py b/toolkits/google/arcade_google/tools/models.py new file mode 100644 index 00000000..f47e3afb --- /dev/null +++ b/toolkits/google/arcade_google/tools/models.py @@ -0,0 +1,232 @@ +from datetime import datetime, timedelta +from enum import Enum +from zoneinfo import ZoneInfo + + +# ---------------------------------------------------------------------------- # +# Google Calendar Models and Enums +# ---------------------------------------------------------------------------- # +class DateRange(Enum): + TODAY = "today" + TOMORROW = "tomorrow" + THIS_WEEK = "this_week" + NEXT_WEEK = "next_week" + THIS_MONTH = "this_month" + NEXT_MONTH = "next_month" + + def to_date_range(self): + today = datetime.now().date() + if self == DateRange.TODAY: + return today, today + timedelta(days=1) + elif self == DateRange.TOMORROW: + return today + timedelta(days=1), today + timedelta(days=2) + elif self == DateRange.THIS_WEEK: + start = today - timedelta(days=today.weekday()) + return start, start + timedelta(days=7) + elif self == DateRange.NEXT_WEEK: + start = today + timedelta(days=7 - today.weekday()) + return start, start + timedelta(days=7) + elif self == DateRange.THIS_MONTH: + start = today.replace(day=1) + next_month = start + timedelta(days=32) + end = next_month.replace(day=1) + return start, end + elif self == DateRange.NEXT_MONTH: + start = (today.replace(day=1) + timedelta(days=32)).replace(day=1) + next_month = start + timedelta(days=32) + end = next_month.replace(day=1) + return start, end + + def to_datetime_range(self, time_zone_name: str | None = None) -> tuple[datetime, datetime]: + start_date, end_date = self.to_date_range() + # time_zone = ZoneInfo(time_zone_name) + start_datetime = datetime.combine( + start_date, datetime.min.time() + ) # .replace(tzinfo=time_zone) + end_datetime = datetime.combine(end_date, datetime.min.time()) # .replace(tzinfo=time_zone) + return start_datetime, end_datetime + + +class Day(Enum): + # TODO: THere are obvious limitations here. We should do better and support any date. + YESTERDAY = "yesterday" + TODAY = "today" + TOMORROW = "tomorrow" + THIS_SUNDAY = "this_sunday" + THIS_MONDAY = "this_monday" + THIS_TUESDAY = "this_tuesday" + THIS_WEDNESDAY = "this_wednesday" + THIS_THURSDAY = "this_thursday" + THIS_FRIDAY = "this_friday" + THIS_SATURDAY = "this_saturday" + NEXT_SUNDAY = "next_sunday" + NEXT_MONDAY = "next_monday" + NEXT_TUESDAY = "next_tuesday" + NEXT_WEDNESDAY = "next_wednesday" + NEXT_THURSDAY = "next_thursday" + NEXT_FRIDAY = "next_friday" + NEXT_SATURDAY = "next_saturday" + + def to_date(self, time_zone_name: str): + time_zone = ZoneInfo(time_zone_name) + today = datetime.now(time_zone).date() + weekday = today.weekday() + + if self == Day.YESTERDAY: + return today - timedelta(days=1) + elif self == Day.TODAY: + return today + elif self == Day.TOMORROW: + return today + timedelta(days=1) + + day_offsets = { + Day.THIS_SUNDAY: 6, + Day.THIS_MONDAY: 0, + Day.THIS_TUESDAY: 1, + Day.THIS_WEDNESDAY: 2, + Day.THIS_THURSDAY: 3, + Day.THIS_FRIDAY: 4, + Day.THIS_SATURDAY: 5, + } + + if self in day_offsets: + return today + timedelta(days=(day_offsets[self] - weekday) % 7) + + next_week_offsets = { + Day.NEXT_SUNDAY: 6, + Day.NEXT_MONDAY: 0, + Day.NEXT_TUESDAY: 1, + Day.NEXT_WEDNESDAY: 2, + Day.NEXT_THURSDAY: 3, + Day.NEXT_FRIDAY: 4, + Day.NEXT_SATURDAY: 5, + } + + if self in next_week_offsets: + return today + timedelta(days=(next_week_offsets[self] - weekday + 7) % 7) + + raise ValueError(f"Invalid Day enum value: {self}") + + +class TimeSlot(Enum): + _0000 = "00:00" + _0015 = "00:15" + _0030 = "00:30" + _0045 = "00:45" + _0100 = "01:00" + _0115 = "01:15" + _0130 = "01:30" + _0145 = "01:45" + _0200 = "02:00" + _0215 = "02:15" + _0230 = "02:30" + _0245 = "02:45" + _0300 = "03:00" + _0315 = "03:15" + _0330 = "03:30" + _0345 = "03:45" + _0400 = "04:00" + _0415 = "04:15" + _0430 = "04:30" + _0445 = "04:45" + _0500 = "05:00" + _0515 = "05:15" + _0530 = "05:30" + _0545 = "05:45" + _0600 = "06:00" + _0615 = "06:15" + _0630 = "06:30" + _0645 = "06:45" + _0700 = "07:00" + _0715 = "07:15" + _0730 = "07:30" + _0745 = "07:45" + _0800 = "08:00" + _0815 = "08:15" + _0830 = "08:30" + _0845 = "08:45" + _0900 = "09:00" + _0915 = "09:15" + _0930 = "09:30" + _0945 = "09:45" + _1000 = "10:00" + _1015 = "10:15" + _1030 = "10:30" + _1045 = "10:45" + _1100 = "11:00" + _1115 = "11:15" + _1130 = "11:30" + _1145 = "11:45" + _1200 = "12:00" + _1215 = "12:15" + _1230 = "12:30" + _1245 = "12:45" + _1300 = "13:00" + _1315 = "13:15" + _1330 = "13:30" + _1345 = "13:45" + _1400 = "14:00" + _1415 = "14:15" + _1430 = "14:30" + _1445 = "14:45" + _1500 = "15:00" + _1515 = "15:15" + _1530 = "15:30" + _1545 = "15:45" + _1600 = "16:00" + _1615 = "16:15" + _1630 = "16:30" + _1645 = "16:45" + _1700 = "17:00" + _1715 = "17:15" + _1730 = "17:30" + _1745 = "17:45" + _1800 = "18:00" + _1815 = "18:15" + _1830 = "18:30" + _1845 = "18:45" + _1900 = "19:00" + _1915 = "19:15" + _1930 = "19:30" + _1945 = "19:45" + _2000 = "20:00" + _2015 = "20:15" + _2030 = "20:30" + _2045 = "20:45" + _2100 = "21:00" + _2115 = "21:15" + _2130 = "21:30" + _2145 = "21:45" + _2200 = "22:00" + _2215 = "22:15" + _2230 = "22:30" + _2245 = "22:45" + _2300 = "23:00" + _2315 = "23:15" + _2330 = "23:30" + _2345 = "23:45" + + def to_time(self): + return datetime.strptime(self.value, "%H:%M").time() + + +class EventVisibility(Enum): + DEFAULT = "default" + PUBLIC = "public" + PRIVATE = "private" + CONFIDENTIAL = "confidential" + + +class EventType(Enum): + BIRTHDAY = "birthday" # Special all-day events with an annual recurrence. + DEFAULT = "default" # Regular events + FOCUS_TIME = "focusTime" # Focus time events + FROM_GMAIL = "fromGmail" # Events from Gmail + OUT_OF_OFFICE = "outOfOffice" # Out of office events + WORKING_LOCATION = "workingLocation" # Working location events + + +class SendUpdatesOptions(Enum): + NONE = "none" # No notifications are sent + ALL = "all" # Notifications are sent to all guests + EXTERNAL_ONLY = "externalOnly" # Notifications are sent to non-Google Calendar guests only. diff --git a/toolkits/google/arcade_google/tools/utils.py b/toolkits/google/arcade_google/tools/utils.py index 63c0a1b7..33f55b49 100644 --- a/toolkits/google/arcade_google/tools/utils.py +++ b/toolkits/google/arcade_google/tools/utils.py @@ -6,6 +6,8 @@ from typing import Any, Optional from bs4 import BeautifulSoup +from arcade_google.tools.models import Day, TimeSlot + class DateRange(Enum): TODAY = "today" @@ -176,6 +178,24 @@ def _clean_text(text: str) -> str: return text +def _update_datetime(day: Day | None, time: TimeSlot | None, time_zone: str) -> dict | None: + """ + Update the datetime for a Google Calendar event. + + Args: + day (Day | None): The day of the event. + time (TimeSlot | None): The time of the event. + time_zone (str): The time zone of the event. + + Returns: + dict | None: The updated datetime for the event. + """ + if day and time: + dt = datetime.combine(day.to_date(time_zone), time.to_time()) + return {"dateTime": dt.isoformat(), "timeZone": time_zone} + return None + + def build_query_string(sender, recipient, subject, body, date_range): """ Helper function to build a query string for Gmail list_emails_by_header tool. diff --git a/toolkits/google/evals/eval_google_calendar.py b/toolkits/google/evals/eval_google_calendar.py new file mode 100644 index 00000000..c7accdcc --- /dev/null +++ b/toolkits/google/evals/eval_google_calendar.py @@ -0,0 +1,176 @@ +import arcade_google +from arcade_google.tools.calendar import create_event, delete_event, list_events, update_event +from arcade_google.tools.models import Day, EventVisibility, TimeSlot + +from arcade.core.catalog import ToolCatalog +from arcade.sdk.eval import ( + BinaryCritic, + EvalRubric, + EvalSuite, + tool_eval, +) + +# Evaluation rubric +rubric = EvalRubric( + fail_threshold=0.9, + warn_threshold=0.95, +) + + +catalog = ToolCatalog() +catalog.add_module(arcade_google) + + +history_after_list_events = [ + {"role": "user", "content": "do i have any events on my calendar for today?"}, + { + "role": "assistant", + "content": "Please go to this URL and authorize the action: \n[Link](https://accounts.google.com/o/oauth2/v2/auth?)", + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_uHdRlr4z7sFm39ZrPsE5wcdT", + "type": "function", + "function": { + "name": "Google_ListEvents", + "arguments": '{"max_day":"today","max_time_slot":"23:45","min_day":"today","min_time_slot":"00:00"}', + }, + } + ], + }, + { + "role": "tool", + "content": '{"events_count": 3, "events": [{"creator": {"email": "john@example.com", "self": true}, "description": "1:1 meeting with Joe", "end": {"dateTime": "2024-09-26T00:15:00-07:00", "timeZone": "America/Los_Angeles"}, "eventType": "default", "htmlLink": "https://www.google.com/calendar/event?eid=01234", "id": "10009199283838467", "location": "622 Rainbow Ave, South San Francisco, CA 94080, USA", "organizer": {"email": "john@example.com", "self": true}, "start": {"dateTime": "2024-09-25T23:15:00-07:00", "timeZone": "America/Los_Angeles"}, "summary": "1:1 meeting"}, {"attendees": [{"email": "joe@example.com", "responseStatus": "accepted"}], "creator": {"email": "john@example.com", "self": true}, "description": "This is just a test", "end": {"dateTime": "2024-09-26T14:00:00-07:00", "timeZone": "America/Los_Angeles"}, "eventType": "default", "htmlLink": "https://www.google.com/calendar/event?eid=OXB2OGFwcmZraWk1N234324", "id": "00099992228181818181", "organizer": {"email": "john@example.com", "self": true}, "start": {"dateTime": "2024-09-26T12:00:00-07:00", "timeZone": "America/Los_Angeles"}, "summary": "API test"}, {"attendees": [{"email": "henry@example.com", "responseStatus": "needsAction"}], "creator": {"email": "john@example.com", "self": true}, "end": {"dateTime": "2024-09-26T17:00:00-07:00", "timeZone": "America/Los_Angeles"}, "eventType": "default", "htmlLink": "https://www.google.com/calendar/event?eid=Z3I1ZzE4b324534556", "id": "gr5g18lf88tfpp3vkareukkc7g", "location": "611 Rainbow Road", "organizer": {"email": "john@example.com", "self": true}, "start": {"dateTime": "2024-09-26T15:00:00-07:00", "timeZone": "America/Los_Angeles"}, "summary": "Focus Time", "visibility": "public"}]}', + "tool_call_id": "call_uHdRlr4z7sFm39ZrPsE5wcdT", + "name": "Google_ListEvents", + }, + { + "role": "assistant", + "content": "Yes, you have three events on your calendar for today:\n\n1. **Event:** Test2\n - **Time:** 23:15 - 00:15 (PST)\n - **Location:** 611 Gateway Blvd, South San Francisco, CA 94080, USA\n - **Description:** 1:1 meeting with Joe\n 2. **Event:** API Test\n - **Time:** 12:00 - 14:00 (PST)\n **Description:** This is just a test\n - [View Event](https://www.google.com/calendar/event?eid=OXB2OGFwcmZraWk1NGUwa24xaTNya2lvZDggZXJpY0BhcmNhZGUtYWkuY29t)\n\n3. **Event:** Focus Time\n - **Time:** 15:00 - 17:00 (PST)\n - **Location:** 611 Rainbow Road\n - **Visibility:** Public\n - [View Event](https://www.google.com/calendar/event?eid=Z3I1ZzE4bGY4OHRmcHAzdmthcmV1a2tjN2cgZXJpY0BhcmNhZGUtYWkuY29t)\n\nIf you need more details or help with anything else, feel free to ask!", + }, +] + + +@tool_eval() +def calendar_eval_suite() -> EvalSuite: + """Create an evaluation suite for Calendar tools.""" + suite = EvalSuite( + name="Calendar Tools Evaluation", + system_message="You are an AI assistant that can create and list events using the provided tools.", + catalog=catalog, + rubric=rubric, + ) + + # Cases for create_event + suite.add_case( + name="Create calendar event", + user_message="Create a meeting for 'Team Meeting' starting next thursday from 11:45pm to 12:15am. Invite johndoe@example.com", + expected_tool_calls=[ + ( + create_event, + { + "summary": "Team Meeting", + "start_date": Day.NEXT_THURSDAY.value, + "start_time": TimeSlot._2345.value, + "end_date": Day.NEXT_FRIDAY.value, + "end_time": TimeSlot._0015.value, + "calendar_id": "primary", + "attendee_emails": ["johndoe@example.com"], + "description": None, + "location": None, + "visibility": EventVisibility.DEFAULT, + }, + ) + ], + critics=[ + BinaryCritic(critic_field="summary", weight=0.15), + BinaryCritic(critic_field="start_date", weight=0.15), + BinaryCritic(critic_field="start_time", weight=0.15), + BinaryCritic(critic_field="end_date", weight=0.15), + BinaryCritic(critic_field="end_time", weight=0.15), + BinaryCritic(critic_field="attendee_emails", weight=0.15), + BinaryCritic(critic_field="description", weight=0.1), + ], + ) + + # Cases for list_events + suite.add_case( + name="List calendar events", + user_message="Do I have any events on my calendar today?", + expected_tool_calls=[ + ( + list_events, + { + "min_day": Day.TODAY.value, + "min_time_slot": TimeSlot._0000.value, + "max_day": Day.TOMORROW.value, + "max_time_slot": TimeSlot._0000.value, + "calendar_id": "primary", + "event_types": None, + "max_results": 10, + }, + ) + ], + critics=[ + BinaryCritic(critic_field="min_day", weight=0.1), + BinaryCritic(critic_field="min_time_slot", weight=0.1), + BinaryCritic(critic_field="max_day", weight=0.1), + BinaryCritic(critic_field="max_time_slot", weight=0.1), + BinaryCritic(critic_field="calendar_id", weight=0.1), + BinaryCritic(critic_field="event_types", weight=0.1), + BinaryCritic(critic_field="max_results", weight=0.1), + ], + ) + + # Cases for update_event + suite.add_case( + name="Update a calendar event", + user_message="Oh no! I cant make it to the API Test since i have lunch with an old friend at that time. Change the meeting to 3pm to 4pm please.", + expected_tool_calls=[ + ( + update_event, + { + "event_id": "00099992228181818181", + "updated_start_day": Day.TODAY.value, + "updated_start_time": TimeSlot._1500.value, + "updated_end_day": Day.TODAY.value, + "updated_end_time": TimeSlot._1600.value, + }, + ) + ], + critics=[ + BinaryCritic(critic_field="event_id", weight=0.2), + BinaryCritic(critic_field="updated_start_day", weight=0.1), + BinaryCritic(critic_field="updated_start_time", weight=0.1), + BinaryCritic(critic_field="updated_end_day", weight=0.1), + BinaryCritic(critic_field="updated_end_time", weight=0.1), + ], + additional_messages=history_after_list_events, + ) + + # Cases for delete_event + suite.add_case( + name="Delete a calendar event", + user_message="I don't need to have focus time today. Please delete it from my calendar. Don't send any notifications.", + expected_tool_calls=[ + ( + delete_event, + { + "event_id": "gr5g18lf88tfpp3vkareukkc7g", + "calendar_id": "primary", + "send_updates": "none", + }, + ) + ], + critics=[ + BinaryCritic(critic_field="event_id", weight=0.6), + BinaryCritic(critic_field="calendar_id", weight=0.2), + BinaryCritic(critic_field="send_updates", weight=0.2), + ], + additional_messages=history_after_list_events, + ) + + return suite diff --git a/toolkits/google/evals/eval_google_gmail.py b/toolkits/google/evals/eval_google_gmail.py new file mode 100644 index 00000000..15220197 --- /dev/null +++ b/toolkits/google/evals/eval_google_gmail.py @@ -0,0 +1,60 @@ +import arcade_google +from arcade_google.tools.gmail import ( + send_email, +) + +from arcade.core.catalog import ToolCatalog +from arcade.sdk.eval import ( + BinaryCritic, + EvalRubric, + EvalSuite, + SimilarityCritic, + tool_eval, +) + +# Evaluation rubric +rubric = EvalRubric( + fail_threshold=0.9, + warn_threshold=0.95, +) + + +catalog = ToolCatalog() +catalog.add_module(arcade_google) + + +@tool_eval() +def gmail_eval_suite() -> EvalSuite: + """Create an evaluation suite for Gmail tools.""" + suite = EvalSuite( + name="Gmail Tools Evaluation", + system_message="You are an AI assistant that can send and manage emails using the provided tools.", + catalog=catalog, + rubric=rubric, + ) + + suite.add_case( + name="Send email to user with clear username", + user_message="Send a email to johndoe@example.com saying 'Hello, can we meet at 3 PM?'. CC his boss janedoe@example.com", + expected_tool_calls=[ + ( + send_email, + { + "subject": "Meeting Request", + "body": "Hello, can we meet at 3 PM?", + "recipient": "johndoe@example.com", + "cc": ["janedoe@example.com"], + "bcc": None, + }, + ) + ], + critics=[ + SimilarityCritic(critic_field="subject", weight=0.125), + SimilarityCritic(critic_field="body", weight=0.25), + BinaryCritic(critic_field="recipient", weight=0.25), + BinaryCritic(critic_field="cc", weight=0.25), + BinaryCritic(critic_field="bcc", weight=0.125), + ], + ) + + return suite diff --git a/toolkits/google/tests/test_calendar.py b/toolkits/google/tests/test_calendar.py new file mode 100644 index 00000000..f06fbc22 --- /dev/null +++ b/toolkits/google/tests/test_calendar.py @@ -0,0 +1,159 @@ +from unittest.mock import MagicMock, patch + +import pytest +from arcade_google.tools.calendar import create_event, delete_event, list_events, update_event +from arcade_google.tools.models import Day, EventVisibility, SendUpdatesOptions, TimeSlot +from googleapiclient.errors import HttpError + +from arcade.core.errors import ToolExecutionError +from arcade.core.schema import ToolAuthorizationContext, ToolContext + + +@pytest.fixture +def mock_context(): + mock_auth = ToolAuthorizationContext(token="fake-token") # noqa: S106 + return ToolContext(authorization=mock_auth) + + +@pytest.mark.asyncio +@patch("arcade_google.tools.calendar.build") +async def test_create_event(mock_build, mock_context): + mock_service = MagicMock() + mock_build.return_value = mock_service + + # Mock the calendar's time zone + mock_service.calendars().get().execute.return_value = {"timeZone": "America/Los_Angeles"} + + # Case: HttpError + mock_service.events().insert().execute.side_effect = HttpError( + resp=MagicMock(status=400), + content=b'{"error": {"message": "Invalid request"}}', + ) + + with pytest.raises(ToolExecutionError): + await create_event( + context=mock_context, + summary="Test Event", + start_date=Day.TODAY, + start_time=TimeSlot._1615, + end_date=Day.TODAY, + end_time=TimeSlot._1715, + description="Test Description", + location="Test Location", + visibility=EventVisibility.PRIVATE, + attendee_emails=["test@example.com"], + ) + + +@pytest.mark.asyncio +@patch("arcade_google.tools.calendar.build") +async def test_list_events(mock_build, mock_context): + mock_service = MagicMock() + mock_build.return_value = mock_service + # Mock the calendar's time zone + mock_service.calendars().get().execute.return_value = {"timeZone": "America/Los_Angeles"} + + # Case: min time is after max time. list_events tool should swap the times and still return the events + mock_events_list_response = { + "items": [ + { + "creator": {"email": "example@arcade-ai.com", "self": True}, + "end": {"dateTime": "2024-09-27T01:00:00-07:00", "timeZone": "America/Los_Angeles"}, + "eventType": "default", + "htmlLink": "https://www.google.com/calendar/event?eid=N2pmYjZ0ZmNnMGNydG5scmhkY2JvZWc4OGIgZXJpY0BhcmNhZGUtYWku", + "id": "7jfb6tfcg0crtnlrhdcboeg88b", + "organizer": {"email": "example@arcade-ai.com", "self": True}, + "start": { + "dateTime": "2024-09-27T00:00:00-07:00", + "timeZone": "America/Los_Angeles", + }, + "summary": "teST", + }, + { + "creator": {"email": "example@arcade-ai.com", "self": True}, + "end": {"dateTime": "2024-09-27T17:00:00-07:00", "timeZone": "America/Los_Angeles"}, + "eventType": "default", + "htmlLink": "https://www.google.com/calendar/event?eid=MjZvYnRoc2xtMWMzbG5mdG10bzk4cDcxaGMgZXJpY0BhcmNhZGUtYWku", + "id": "26obthslm1c3lnftmto98p71hc", + "organizer": {"email": "example@arcade-ai.com", "self": True}, + "start": { + "dateTime": "2024-09-27T14:00:00-07:00", + "timeZone": "America/Los_Angeles", + }, + "summary": "New Event", + }, + ] + } + expected_tool_response = { + "events_count": len(mock_events_list_response["items"]), + "events": mock_events_list_response["items"], + } + mock_service.events().list().execute.return_value = mock_events_list_response + message = await list_events( + context=mock_context, + min_day=Day.TODAY, + min_time_slot=TimeSlot._1615, + max_day=Day.TODAY, + max_time_slot=TimeSlot._1515, + ) + assert message == expected_tool_response + + # Case: HttpError + mock_service.events().list().execute.side_effect = HttpError( + resp=MagicMock(status=400), + content=b'{"error": {"message": "Invalid request"}}', + ) + + with pytest.raises(ToolExecutionError): + await list_events( + context=mock_context, + min_day=Day.TODAY, + min_time_slot=TimeSlot._1615, + max_day=Day.TOMORROW, + max_time_slot=TimeSlot._1815, + ) + + +@pytest.mark.asyncio +@patch("arcade_google.tools.calendar.build") +async def test_update_event(mock_build, mock_context): + mock_service = MagicMock() + mock_build.return_value = mock_service + mock_service.events().update().execute.side_effect = HttpError( + resp=MagicMock(status=400), + content=b'{"error": {"message": "Event not found"}}', + ) + + with pytest.raises(ToolExecutionError): + await update_event( + context=mock_context, + event_id="1234567890", + updated_start_day=Day.NEXT_FRIDAY, + updated_start_time=TimeSlot._0015, + updated_end_day=Day.NEXT_FRIDAY, + updated_end_time=TimeSlot._0115, + updated_summary="Updated Event", + updated_description="Updated Description", + updated_location="Updated Location", + updated_visibility=EventVisibility.PRIVATE, + attendee_emails_to_add=["test@example.com"], + attendee_emails_to_remove=["test@example2.com"], + send_updates=SendUpdatesOptions.ALL, + ) + + +@pytest.mark.asyncio +@patch("arcade_google.tools.calendar.build") +async def test_delete_event(mock_build, mock_context): + mock_service = MagicMock() + mock_build.return_value = mock_service + mock_service.events().delete().execute.side_effect = HttpError( + resp=MagicMock(status=400), + content=b'{"error": {"message": "Event not found"}}', + ) + + with pytest.raises(ToolExecutionError): + await delete_event( + context=mock_context, + event_id="nonexistent_event", + )