diff --git a/toolkits/google/arcade_google/critics.py b/toolkits/google/arcade_google/critics.py new file mode 100644 index 00000000..a9538b88 --- /dev/null +++ b/toolkits/google/arcade_google/critics.py @@ -0,0 +1,41 @@ +from collections.abc import Collection +from typing import Any + +from arcade.sdk.eval import DatetimeCritic + + +class DatetimeOrNoneCritic(DatetimeCritic): + """ + A critic that evaluates the closeness of datetime values within a specified tolerance or whether + it's a None value. + + Attributes: + tolerance: Acceptable timedelta between expected and actual datetimes. + max_difference: Maximum timedelta for a partial score. + """ + + def evaluate(self, expected: Any, actual: Any) -> dict[str, Any]: + if actual is None: + return {"match": True, "score": self.weight} + return super().evaluate(expected, actual) + + +class AnyDatetimeCritic(DatetimeCritic): + """ + A critic that evaluates the closeness of datetime values within a list of expected values. + """ + + def evaluate(self, expected: Any, actual: Any) -> dict[str, Any]: + if not isinstance(expected, Collection): + expected = [expected] + for expected_value in expected: + critic = DatetimeCritic( + critic_field=self.critic_field, + weight=self.weight, + tolerance=self.tolerance, + max_difference=self.max_difference, + ) + result = critic.evaluate(expected_value, actual) + if result["match"]: + return result + return {"match": False, "score": 0} diff --git a/toolkits/google/arcade_google/exceptions.py b/toolkits/google/arcade_google/exceptions.py index 88937f1f..a925406b 100644 --- a/toolkits/google/arcade_google/exceptions.py +++ b/toolkits/google/arcade_google/exceptions.py @@ -1,3 +1,8 @@ +from zoneinfo import available_timezones + +from arcade.sdk.errors import RetryableToolError + + class GoogleToolError(Exception): """Base exception for Google tool errors.""" @@ -13,6 +18,12 @@ class GoogleToolError(Exception): return base_message +class RetryableGoogleToolError(RetryableToolError): + """Raised when there's an error in a Google tool that can be retried.""" + + pass + + class GoogleServiceError(GoogleToolError): """Raised when there's an error building or using the Google service.""" @@ -31,6 +42,22 @@ class GoogleCalendarToolError(GoogleToolError): pass +class InvalidTimezoneError(RetryableGoogleToolError): + """Raised when a timezone is provided that is not supported by Python's zoneinfo.""" + + def __init__(self, timezone_str: str): + self.timezone_str = timezone_str + available_timezones_msg = ( + "Here is a list of valid timezones (from Python's zoneinfo.available_timezones()): " + f"{available_timezones()}" + ) + super().__init__( + f"Invalid timezone: '{timezone_str}'", + developer_message=available_timezones_msg, + additional_prompt_content=available_timezones_msg, + ) + + class GoogleDriveToolError(GoogleToolError): """Raised when there's an error in the Google Drive tools.""" diff --git a/toolkits/google/arcade_google/models.py b/toolkits/google/arcade_google/models.py index c80d8b83..152705c1 100644 --- a/toolkits/google/arcade_google/models.py +++ b/toolkits/google/arcade_google/models.py @@ -18,36 +18,59 @@ class DateRange(Enum): THIS_MONTH = "this_month" NEXT_MONTH = "next_month" - def to_date_range(self) -> tuple[date, date]: - today = datetime.now().date() - if self == DateRange.TODAY: - return today, today + timedelta(days=1) - elif self == DateRange.TOMORROW: - return today + timedelta(days=1), today + timedelta(days=2) - elif self == DateRange.THIS_WEEK: - start = today - timedelta(days=today.weekday()) - return start, start + timedelta(days=7) - elif self == DateRange.NEXT_WEEK: - start = today + timedelta(days=7 - today.weekday()) - return start, start + timedelta(days=7) - elif self == DateRange.THIS_MONTH: - start = today.replace(day=1) - next_month = start + timedelta(days=32) - end = next_month.replace(day=1) - return start, end - elif self == DateRange.NEXT_MONTH: - start = (today.replace(day=1) + timedelta(days=32)).replace(day=1) - next_month = start + timedelta(days=32) - end = next_month.replace(day=1) - return start, end + def to_datetime_range( + self, + start_time: Optional[time] = None, + end_time: Optional[time] = None, + time_zone: Optional[ZoneInfo] = None, + today: Optional[date] = None, + ) -> tuple[datetime, datetime]: + """ + Convert a DateRange enum value to a tuple with two datetime objects representing the start + and end of the date range. + + :param start_time: The start time of the date range. Defaults to the current time. + :param end_time: The end time of the date range. Defaults to 23:59:59. + :param time_zone: The time zone to use for the date range. Defaults to UTC. + :param today: Today's date. Defaults to the current date provided by `datetime.now().date()` + """ + start_time = start_time or datetime.now().time() + end_time = end_time or time(23, 59, 59) + today = today or datetime.now().date() + + if self == DateRange.TODAY: + start_date, end_date = today, today + elif self == DateRange.TOMORROW: + start_date, end_date = today + timedelta(days=1), today + timedelta(days=1) + elif self == DateRange.THIS_WEEK: + start_date = today - timedelta(days=today.weekday()) + end_date = start_date + timedelta(days=6) + elif self == DateRange.NEXT_WEEK: + start_date = today + timedelta(days=7 - today.weekday()) + end_date = start_date + timedelta(days=6) + elif self == DateRange.THIS_MONTH: + start_date = today.replace(day=1) + next_month = start_date + timedelta(days=31) + end_date = next_month.replace(day=1) - timedelta(days=1) + elif self == DateRange.NEXT_MONTH: + start_date = (today.replace(day=1) + timedelta(days=31)).replace(day=1) + next_month = start_date + timedelta(days=31) + end_date = next_month.replace(day=1) - timedelta(days=1) + else: + raise ValueError( + f"DateRange enum value: {self} is not supported for date range conversion" + ) + + start_time = start_time or time(0, 0, 0) + end_time = end_time or time(23, 59, 59) + + start_datetime = datetime.combine(start_date, start_time) + end_datetime = datetime.combine(end_date, end_time) + + if time_zone: + start_datetime = start_datetime.replace(tzinfo=time_zone) + end_datetime = end_datetime.replace(tzinfo=time_zone) - def to_datetime_range(self, time_zone_name: str | None = None) -> tuple[datetime, datetime]: - start_date, end_date = self.to_date_range() - # time_zone = ZoneInfo(time_zone_name) - start_datetime = datetime.combine( - start_date, datetime.min.time() - ) # .replace(tzinfo=time_zone) - end_datetime = datetime.combine(end_date, datetime.min.time()) # .replace(tzinfo=time_zone) return start_datetime, end_datetime diff --git a/toolkits/google/arcade_google/tools/calendar.py b/toolkits/google/arcade_google/tools/calendar.py index 1efd5bce..bf3237c2 100644 --- a/toolkits/google/arcade_google/tools/calendar.py +++ b/toolkits/google/arcade_google/tools/calendar.py @@ -1,5 +1,7 @@ +import json from datetime import datetime, timedelta -from typing import Annotated, Any +from typing import Annotated, Any, Optional +from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from arcade.sdk import ToolContext, tool from arcade.sdk.auth import Google @@ -9,7 +11,10 @@ from googleapiclient.discovery import build from googleapiclient.errors import HttpError from arcade_google.models import EventVisibility, SendUpdatesOptions -from arcade_google.utils import parse_datetime +from arcade_google.utils import ( + compute_free_time_intersection, + parse_datetime, +) @tool( @@ -356,3 +361,140 @@ async def delete_event( f"Event with ID '{event_id}' successfully deleted from calendar '{calendar_id}'. " f"{notification_message}" ) + + +# TODO: would be nice to have a "min_slot_duration" parameter +# TODO: find a way to have "include_weekends" parameter without confusing LLMs +@tool( + requires_auth=Google( + scopes=["https://www.googleapis.com/auth/calendar.readonly"], + ), +) +async def find_time_slots_when_everyone_is_free( + context: ToolContext, + email_addresses: Annotated[ + Optional[list[str]], + "The list of email addresses from people in the same organization domain (apart from the " + "currently logged in user) to search for free time slots. Defaults to None, which will " + "return free time slots for the current user only.", + ] = None, + start_date: Annotated[ + Optional[str], + "The start date to search for time slots in the format 'YYYY-MM-DD'. Defaults to today's " + "date. It will search starting from this date at the time 00:00:00.", + ] = None, + end_date: Annotated[ + Optional[str], + "The end date to search for time slots in the format 'YYYY-MM-DD'. Defaults to seven days " + "from the start date. It will search until this date at the time 23:59:59.", + ] = None, + start_time_boundary: Annotated[ + str, + "Will return free slots in any given day starting from this time in the format 'HH:MM'. " + "Defaults to '08:00', which is a usual business hour start time.", + ] = "08:00", + end_time_boundary: Annotated[ + str, + "Will return free slots in any given day until this time in the format 'HH:MM'. " + "Defaults to '18:00', which is a usual business hour end time.", + ] = "18:00", +) -> Annotated[ + dict, + "A dictionary with the free slots and the timezone in which time slots are represented.", +]: + """ + Provides time slots when everyone is free within a given date range and time boundaries. + """ + credentials = Credentials( + context.authorization.token if context.authorization and context.authorization.token else "" + ) + + # Build google api services + oauth_service = build("oauth2", "v2", credentials=credentials) + calendar_service = build("calendar", "v3", credentials=credentials) + + email_addresses = email_addresses or [] + + if isinstance(email_addresses, str): + email_addresses = [email_addresses] + + # Add the currently logged in user to the list of email addresses + user_info = oauth_service.userinfo().get().execute() + if user_info["email"] not in email_addresses: + email_addresses.append(user_info["email"]) + + # Get the timezone of the currently logged in user + calendar = calendar_service.calendars().get(calendarId="primary").execute() + timezone_name = calendar.get("timeZone") + + try: + tz = ZoneInfo(timezone_name) + # If the calendar timezone name is not supported by Python's zoneinfo, use UTC + except ZoneInfoNotFoundError: + timezone_name = "UTC" + tz = ZoneInfo("UTC") + + # Set default start and end dates, if not provided by the caller + start_date = start_date or datetime.now(tz=tz).date().isoformat() + end_date = end_date or (datetime.now(tz=tz).date() + timedelta(days=7)).isoformat() + + # Parse start and end dates to datetime objects + start_datetime = datetime.strptime(start_date, "%Y-%m-%d").replace( + hour=0, minute=0, second=0, microsecond=0, tzinfo=tz + ) + end_datetime = datetime.strptime(end_date, "%Y-%m-%d").replace( + hour=23, minute=59, second=59, microsecond=0, tzinfo=tz + ) + + # Get the busy slots from the calendars of the users + freebusy_response = ( + calendar_service.freebusy() + .query( + body={ + "timeMin": start_datetime.isoformat(), + "timeMax": end_datetime.isoformat(), + "timeZone": timezone_name, + "items": [{"id": email_address} for email_address in email_addresses], + } + ) + .execute() + ) + busy_slots = freebusy_response["calendars"] + + response_errors = [] + + for email in email_addresses: + if "errors" not in busy_slots[email]: + continue + errors = busy_slots[email]["errors"] + for error in errors: + response_errors.append( + f"Error retrieving free slots from calendar of '{email}': " + f"{error.get('reason', 'not determined')}" + ) + + if response_errors: + raise RetryableToolError( + "Error retrieving free slots from calendars of one or more users.", + additional_prompt_content=json.dumps(response_errors), + retry_after_ms=1000, + developer_message="Error retrieving free slots from calendars of one or more users.", + ) + + # Compute the free slots + free_slots = compute_free_time_intersection( + busy_data=busy_slots, + global_start=start_datetime, + global_end=end_datetime, + start_time_boundary=datetime.strptime(start_time_boundary, "%H:%M") + .time() + .replace(tzinfo=tz), + end_time_boundary=datetime.strptime(end_time_boundary, "%H:%M").time().replace(tzinfo=tz), + include_weekends=True, + tz=tz, + ) + + return { + "free_slots": free_slots, + "timezone": timezone_name, + } diff --git a/toolkits/google/arcade_google/utils.py b/toolkits/google/arcade_google/utils.py index 8094897c..75b6d520 100644 --- a/toolkits/google/arcade_google/utils.py +++ b/toolkits/google/arcade_google/utils.py @@ -1,7 +1,7 @@ import logging import re from base64 import urlsafe_b64decode, urlsafe_b64encode -from datetime import datetime, timedelta +from datetime import date, datetime, time, timedelta, timezone from email.message import EmailMessage from email.mime.text import MIMEText from enum import Enum @@ -795,6 +795,199 @@ def build_docs_service(auth_token: Optional[str]) -> Resource: # type: ignore[n return build("docs", "v1", credentials=Credentials(auth_token)) +def parse_rfc3339_datetime_str(dt_str: str, tz: timezone = timezone.utc) -> datetime: + """ + Parse an RFC3339 datetime string into a timezone-aware datetime. + Converts a trailing 'Z' (UTC) into +00:00. + If the parsed datetime is naive, assume it is in the provided timezone. + """ + if dt_str.endswith("Z"): + dt_str = dt_str[:-1] + "+00:00" + dt = datetime.fromisoformat(dt_str) + if dt.tzinfo is None: + dt = dt.replace(tzinfo=tz) + return dt + + +def merge_intervals(intervals: list[tuple[datetime, datetime]]) -> list[tuple[datetime, datetime]]: + """ + Given a list of (start, end) tuples, merge overlapping or adjacent intervals. + """ + merged: list[tuple[datetime, datetime]] = [] + for start, end in sorted(intervals, key=lambda x: x[0]): + if not merged: + merged.append((start, end)) + else: + last_start, last_end = merged[-1] + if start <= last_end: + merged[-1] = (last_start, max(last_end, end)) + else: + merged.append((start, end)) + return merged + + +# Calendar utils +def weekday_to_name(weekday: int) -> str: + return ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"][weekday] + + +def get_time_boundaries_for_date( + current_date: date, + global_start: datetime, + global_end: datetime, + start_time_boundary: time, + end_time_boundary: time, + tz: ZoneInfo, +) -> tuple[datetime, datetime]: + """Compute the allowed start and end times for the given day, adjusting for global bounds.""" + day_start_time = datetime.combine(current_date, start_time_boundary).replace(tzinfo=tz) + day_end_time = datetime.combine(current_date, end_time_boundary).replace(tzinfo=tz) + + if current_date == global_start.date(): + day_start_time = max(day_start_time, global_start) + + if current_date == global_end.date(): + day_end_time = min(day_end_time, global_end) + + return day_start_time, day_end_time + + +def gather_busy_intervals( + busy_data: dict[str, Any], + day_start: datetime, + day_end: datetime, + business_tz: ZoneInfo, +) -> list[tuple[datetime, datetime]]: + """ + Collect busy intervals from all calendars that intersect with the day's business hours. + Busy intervals are clipped to lie within [day_start, day_end]. + """ + busy_intervals = [] + for calendar in busy_data: + for slot in busy_data[calendar].get("busy", []): + slot_start = parse_rfc3339_datetime_str(slot["start"]).astimezone(business_tz) + slot_end = parse_rfc3339_datetime_str(slot["end"]).astimezone(business_tz) + if slot_end > day_start and slot_start < day_end: + busy_intervals.append((max(slot_start, day_start), min(slot_end, day_end))) + return busy_intervals + + +def subtract_busy_intervals( + business_start: datetime, + business_end: datetime, + busy_intervals: list[tuple[datetime, datetime]], +) -> list[dict[str, Any]]: + """ + Subtract the merged busy intervals from the business hours and return free time slots. + """ + free_slots = [] + merged_busy = merge_intervals(busy_intervals) + + # If there are no busy intervals, return the entire business window as free. + if not merged_busy: + return [ + { + "start": { + "datetime": business_start.isoformat(), + "weekday": weekday_to_name(business_start.weekday()), + }, + "end": { + "datetime": business_end.isoformat(), + "weekday": weekday_to_name(business_end.weekday()), + }, + } + ] + + current_free_start = business_start + for busy_start, busy_end in merged_busy: + if current_free_start < busy_start: + free_slots.append({ + "start": { + "datetime": current_free_start.isoformat(), + "weekday": weekday_to_name(current_free_start.weekday()), + }, + "end": { + "datetime": busy_start.isoformat(), + "weekday": weekday_to_name(busy_start.weekday()), + }, + }) + current_free_start = max(current_free_start, busy_end) + if current_free_start < business_end: + free_slots.append({ + "start": { + "datetime": current_free_start.isoformat(), + "weekday": weekday_to_name(current_free_start.weekday()), + }, + "end": { + "datetime": business_end.isoformat(), + "weekday": weekday_to_name(business_end.weekday()), + }, + }) + return free_slots + + +def compute_free_time_intersection( + busy_data: dict[str, Any], + global_start: datetime, + global_end: datetime, + start_time_boundary: time, + end_time_boundary: time, + include_weekends: bool, + tz: ZoneInfo, +) -> list[dict[str, Any]]: + """ + Returns the free time slots across all calendars within the global bounds, + ensuring that the global start is not in the past. + + Only considers business days (Monday to Friday) and business hours (08:00-19:00) + in the provided timezone. + """ + # Ensure global_start is never in the past relative to now. + now = get_now(tz) + + if now > global_start: + global_start = now + + # If after adjusting the start, there's no interval left, return empty. + if global_start >= global_end: + return [] + + free_slots = [] + current_date = global_start.date() + + while current_date <= global_end.date(): + if not include_weekends and current_date.weekday() >= 5: + current_date += timedelta(days=1) + continue + + day_start, day_end = get_time_boundaries_for_date( + current_date=current_date, + global_start=global_start, + global_end=global_end, + start_time_boundary=start_time_boundary, + end_time_boundary=end_time_boundary, + tz=tz, + ) + + # Skip if the day's allowed time window is empty. + if day_start >= day_end: + current_date += timedelta(days=1) + continue + + busy_intervals = gather_busy_intervals(busy_data, day_start, day_end, tz) + free_slots.extend(subtract_busy_intervals(day_start, day_end, busy_intervals)) + + current_date += timedelta(days=1) + + return free_slots + + +def get_now(tz: ZoneInfo | None = None) -> datetime: + if not tz: + tz = ZoneInfo("UTC") + return datetime.now(tz) + + # Contacts utils def build_people_service(auth_token: Optional[str]) -> Resource: # type: ignore[no-any-unimported] """ diff --git a/toolkits/google/evals/eval_calendar_free_slots.py b/toolkits/google/evals/eval_calendar_free_slots.py new file mode 100644 index 00000000..2b1bde15 --- /dev/null +++ b/toolkits/google/evals/eval_calendar_free_slots.py @@ -0,0 +1,470 @@ +from datetime import timedelta + +from arcade.sdk import ToolCatalog +from arcade.sdk.eval import ( + BinaryCritic, + DatetimeCritic, + EvalRubric, + EvalSuite, + ExpectedToolCall, + NoneCritic, + tool_eval, +) + +import arcade_google +from arcade_google.critics import AnyDatetimeCritic, DatetimeOrNoneCritic +from arcade_google.tools.calendar import find_time_slots_when_everyone_is_free + +rubric = EvalRubric( + fail_threshold=0.9, + warn_threshold=0.95, +) + +catalog = ToolCatalog() +catalog.add_module(arcade_google) + + +@tool_eval() +def get_free_slots_eval_suite() -> EvalSuite: + """Create an evaluation suite for free slots Calendar tool.""" + suite = EvalSuite( + name="Calendar Tools Evaluation", + system_message=( + "You are an AI assistant that can manage calendars and events using the provided tools. " + "The first day of a week is Monday and the last day is Sunday. " + "Today is Thursday, March 6, 2025 (2025-03-06). " + "This week started on Monday, March 3, 2025 (2025-03-03) and ends on Sunday, March 9, 2025 (2025-03-09). " + "Last week started on Monday, February 24, 2025 (2025-02-24) and ended on Sunday, March 2, 2025 (2025-03-02). " + "Next week starts on Monday, March 10, 2025 (2025-03-10) and ends on Sunday, March 16, 2025 (2025-03-16). " + "This month started on March 1, 2025 (2025-03-01) and ends on March 31, 2025 (2025-03-31). " + "Last month started on February 1, 2025 (2025-02-01) and ended on February 28, 2025 (2025-02-28). " + "Next month starts on April 1, 2025 (2025-04-01) and ends on April 30, 2025 (2025-04-30). " + "This quarter started on January 1, 2025 (2025-01-01) and ends on March 31, 2025 (2025-03-31). " + "Last quarter started on December 1, 2024 (2024-12-01) and ended on December 31, 2024 (2024-12-31). " + "Next quarter starts on April 1, 2025 (2025-04-01) and ends on June 30, 2025 (2025-06-30). " + ), + catalog=catalog, + rubric=rubric, + ) + + suite.add_case( + name="Get free slots for the next 5 days", + user_message=("At what times am I free in the next 5 days?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-03-06", + "end_date": "2025-03-11", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + DatetimeOrNoneCritic( + critic_field="start_date", weight=0.35, tolerance=timedelta(days=1) + ), + DatetimeCritic(critic_field="end_date", weight=0.35, tolerance=timedelta(days=1)), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots for the next 10 days", + user_message=("At what times am I free in the next 10 days?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-03-06", + "end_date": "2025-03-16", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + DatetimeOrNoneCritic( + critic_field="start_date", weight=0.35, tolerance=timedelta(days=1) + ), + DatetimeCritic(critic_field="end_date", weight=0.35, tolerance=timedelta(days=1)), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots this week", + user_message=("At what times am I free this week?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + # Models sometimes will consider today as the start range, other times it will + # consider last Monday. The question is ambiguous, so we allow both. + "start_date": ["2025-03-03", "2025-03-06"], + "end_date": "2025-03-09", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + AnyDatetimeCritic(critic_field="start_date", weight=0.35), + BinaryCritic(critic_field="end_date", weight=0.35), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots next week", + user_message=("At what times am I free next week?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-03-10", + "end_date": "2025-03-16", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + BinaryCritic(critic_field="start_date", weight=0.35), + BinaryCritic(critic_field="end_date", weight=0.35), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots today", + user_message=("At what times am I free today?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-03-06", + "end_date": "2025-03-06", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + DatetimeOrNoneCritic(critic_field="start_date", weight=0.35), + DatetimeCritic(critic_field="end_date", weight=0.35), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots today", + user_message=("At what times am I free tonight before 10 PM?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-03-06", + "end_date": "2025-03-06", + "start_time_boundary": "08:00", + "end_time_boundary": "22:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + DatetimeOrNoneCritic(critic_field="start_date", weight=0.35), + DatetimeCritic(critic_field="end_date", weight=0.35), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots this weekend", + user_message=("At what times am I free this weekend?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-03-08", + "end_date": "2025-03-09", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + DatetimeCritic(critic_field="start_date", weight=0.35), + DatetimeCritic(critic_field="end_date", weight=0.35), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots this month", + user_message=("At what times am I free this month?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": ["2025-03-06", "2025-03-01"], + "end_date": "2025-03-31", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + AnyDatetimeCritic(critic_field="start_date", weight=0.35), + DatetimeCritic(critic_field="end_date", weight=0.35), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots next month", + user_message=("At what times am I free next month?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-04-01", + "end_date": "2025-04-30", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + DatetimeCritic(critic_field="start_date", weight=0.35), + DatetimeCritic(critic_field="end_date", weight=0.35), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots last week", + user_message=("At what times was I free last week?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-02-24", + "end_date": "2025-03-02", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + DatetimeCritic(critic_field="start_date", weight=0.35), + DatetimeCritic(critic_field="end_date", weight=0.35), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots next quarter", + user_message=("At what times am I free next quarter?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-04-01", + "end_date": "2025-06-30", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + DatetimeCritic(critic_field="start_date", weight=0.35), + DatetimeCritic(critic_field="end_date", weight=0.35), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots in the next 30 days", + user_message=("At what times am I free in the next 30 days?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-03-06", + "end_date": "2025-04-05", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + DatetimeOrNoneCritic( + critic_field="start_date", weight=0.35, tolerance=timedelta(days=1) + ), + DatetimeCritic(critic_field="end_date", weight=0.35, tolerance=timedelta(days=1)), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots in April", + user_message=("At what times am I free in April?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-04-01", + "end_date": "2025-04-30", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + DatetimeCritic(critic_field="start_date", weight=0.35), + DatetimeCritic(critic_field="end_date", weight=0.35), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots for a specific email address", + user_message=("Is johndoe@example.com free some time tomorrow?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": ["johndoe@example.com"], + "start_date": "2025-03-07", + "end_date": "2025-03-07", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + BinaryCritic(critic_field="email_addresses", weight=0.30), + DatetimeCritic(critic_field="start_date", weight=0.25), + DatetimeCritic(critic_field="end_date", weight=0.25), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots for a specific email address", + user_message=( + "I need to schedule a meeting with johndoe@example.com tomorrow. When are both of us free?" + ), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": ["johndoe@example.com"], + "start_date": "2025-03-07", + "end_date": "2025-03-07", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + BinaryCritic(critic_field="email_addresses", weight=0.3), + DatetimeCritic(critic_field="start_date", weight=0.25), + DatetimeCritic(critic_field="end_date", weight=0.25), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + suite.add_case( + name="Get free slots for a specific email address", + user_message=( + "I need to schedule a meeting with johndoe@example.com tomorrow morning. When are both of us free?" + ), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": ["johndoe@example.com"], + "start_date": "2025-03-07", + "end_date": "2025-03-07", + "start_time_boundary": "08:00", + "end_time_boundary": "12:00", + }, + ) + ], + critics=[ + BinaryCritic(critic_field="email_addresses", weight=0.2), + DatetimeCritic(critic_field="start_date", weight=0.2), + DatetimeCritic(critic_field="end_date", weight=0.2), + BinaryCritic(critic_field="start_time_boundary", weight=0.2), + BinaryCritic(critic_field="end_time_boundary", weight=0.2), + ], + ) + + suite.add_case( + name="Get free slots for a specific date range", + user_message=("At what times am I free between 2025-04-27 and 2025-04-29?"), + expected_tool_calls=[ + ExpectedToolCall( + func=find_time_slots_when_everyone_is_free, + args={ + "email_addresses": None, + "start_date": "2025-04-27", + "end_date": "2025-04-29", + "start_time_boundary": "08:00", + "end_time_boundary": "18:00", + }, + ) + ], + critics=[ + NoneCritic(critic_field="email_addresses", weight=0.1), + DatetimeCritic(critic_field="start_date", weight=0.35), + DatetimeCritic(critic_field="end_date", weight=0.35), + BinaryCritic(critic_field="start_time_boundary", weight=0.1), + BinaryCritic(critic_field="end_time_boundary", weight=0.1), + ], + ) + + return suite diff --git a/toolkits/google/evals/eval_google_calendar.py b/toolkits/google/evals/eval_google_calendar.py index b7cb5b92..fed77c59 100644 --- a/toolkits/google/evals/eval_google_calendar.py +++ b/toolkits/google/evals/eval_google_calendar.py @@ -12,13 +12,12 @@ from arcade.sdk.eval import ( import arcade_google from arcade_google.tools.calendar import ( - EventVisibility, - SendUpdatesOptions, create_event, delete_event, list_events, update_event, ) +from arcade_google.tools.models import EventVisibility, SendUpdatesOptions # Evaluation rubric rubric = EvalRubric( diff --git a/toolkits/google/tests/test_calendar.py b/toolkits/google/tests/test_calendar.py index d2ab62e4..296c3cd8 100644 --- a/toolkits/google/tests/test_calendar.py +++ b/toolkits/google/tests/test_calendar.py @@ -1,12 +1,20 @@ +from datetime import datetime from unittest.mock import MagicMock, patch +from zoneinfo import ZoneInfo import pytest from arcade.sdk import ToolAuthorizationContext, ToolContext -from arcade.sdk.errors import ToolExecutionError +from arcade.sdk.errors import RetryableToolError, ToolExecutionError from googleapiclient.errors import HttpError from arcade_google.models import EventVisibility, SendUpdatesOptions -from arcade_google.tools.calendar import create_event, delete_event, list_events, update_event +from arcade_google.tools.calendar import ( + create_event, + delete_event, + find_time_slots_when_everyone_is_free, + list_events, + update_event, +) @pytest.fixture @@ -152,3 +160,325 @@ async def test_delete_event(mock_build, mock_context): event_id="nonexistent_event", send_updates=SendUpdatesOptions.ALL, ) + + +@pytest.mark.asyncio +@patch("arcade_google.utils.get_now") +@patch("arcade_google.tools.calendar.build") +async def test_find_free_slots_happiest_path_single_user(mock_build, mock_get_now, mock_context): + calendar_service = MagicMock() + oauth_service = MagicMock() + + mock_get_now.return_value = datetime( + 2025, 3, 10, 9, 25, 0, tzinfo=ZoneInfo("America/Los_Angeles") + ) + mock_build.side_effect = [oauth_service, calendar_service] + + oauth_service.userinfo().get().execute.return_value = { + "email": "example@arcade.dev", + } + + calendar_service.freebusy().query().execute.return_value = { + "calendars": { + "example@arcade.dev": {"busy": []}, + } + } + + calendar_service.calendars().get().execute.return_value = { + "timeZone": "America/Los_Angeles", + } + + response = await find_time_slots_when_everyone_is_free( + context=mock_context, + email_addresses=["example@arcade.dev"], + start_date="2025-03-10", + end_date="2025-03-11", + start_time_boundary="08:00", + end_time_boundary="18:00", + ) + + assert response == { + "free_slots": [ + { + "start": { + "datetime": "2025-03-10T09:25:00-07:00", + "weekday": "Monday", + }, + "end": { + "datetime": "2025-03-10T18:00:00-07:00", + "weekday": "Monday", + }, + }, + { + "start": { + "datetime": "2025-03-11T08:00:00-07:00", + "weekday": "Tuesday", + }, + "end": { + "datetime": "2025-03-11T18:00:00-07:00", + "weekday": "Tuesday", + }, + }, + ], + "timezone": "America/Los_Angeles", + } + + +@pytest.mark.asyncio +@patch("arcade_google.utils.get_now") +@patch("arcade_google.tools.calendar.build") +async def test_find_free_slots_happiest_path_single_user_with_busy_times( + mock_build, mock_get_now, mock_context +): + calendar_service = MagicMock() + oauth_service = MagicMock() + + mock_get_now.return_value = datetime( + 2025, 3, 10, 9, 25, 0, tzinfo=ZoneInfo("America/Los_Angeles") + ) + mock_build.side_effect = [oauth_service, calendar_service] + + oauth_service.userinfo().get().execute.return_value = { + "email": "example@arcade.dev", + } + + calendar_service.freebusy().query().execute.return_value = { + "calendars": { + "example@arcade.dev": { + "busy": [ + { + "start": "2025-03-10T11:00:00-07:00", + "end": "2025-03-10T12:00:00-07:00", + }, + { + "start": "2025-03-10T14:15:00-07:00", + "end": "2025-03-10T14:30:00-07:00", + }, + ] + }, + } + } + + calendar_service.calendars().get().execute.return_value = { + "timeZone": "America/Los_Angeles", + } + + response = await find_time_slots_when_everyone_is_free( + context=mock_context, + email_addresses=["example@arcade.dev"], + start_date="2025-03-10", + end_date="2025-03-11", + start_time_boundary="08:00", + end_time_boundary="18:00", + ) + + assert response == { + "free_slots": [ + { + "start": { + "datetime": "2025-03-10T09:25:00-07:00", + "weekday": "Monday", + }, + "end": { + "datetime": "2025-03-10T11:00:00-07:00", + "weekday": "Monday", + }, + }, + { + "start": { + "datetime": "2025-03-10T12:00:00-07:00", + "weekday": "Monday", + }, + "end": { + "datetime": "2025-03-10T14:15:00-07:00", + "weekday": "Monday", + }, + }, + { + "start": { + "datetime": "2025-03-10T14:30:00-07:00", + "weekday": "Monday", + }, + "end": { + "datetime": "2025-03-10T18:00:00-07:00", + "weekday": "Monday", + }, + }, + { + "start": { + "datetime": "2025-03-11T08:00:00-07:00", + "weekday": "Tuesday", + }, + "end": { + "datetime": "2025-03-11T18:00:00-07:00", + "weekday": "Tuesday", + }, + }, + ], + "timezone": "America/Los_Angeles", + } + + +@pytest.mark.asyncio +@patch("arcade_google.utils.get_now") +@patch("arcade_google.tools.calendar.build") +async def test_find_free_slots_happiest_path_multiple_users_with_busy_times( + mock_build, mock_get_now, mock_context +): + calendar_service = MagicMock() + oauth_service = MagicMock() + + mock_get_now.return_value = datetime( + 2025, 3, 10, 9, 25, 0, tzinfo=ZoneInfo("America/Los_Angeles") + ) + mock_build.side_effect = [oauth_service, calendar_service] + + oauth_service.userinfo().get().execute.return_value = { + "email": "example@arcade.dev", + } + + calendar_service.freebusy().query().execute.return_value = { + "calendars": { + "example@arcade.dev": { + "busy": [ + { + "start": "2025-03-10T11:00:00-07:00", + "end": "2025-03-10T12:00:00-07:00", + }, + { + "start": "2025-03-10T14:15:00-07:00", + "end": "2025-03-10T14:30:00-07:00", + }, + ] + }, + "example2@arcade.dev": { + "busy": [ + { + "start": "2025-03-10T11:30:00-07:00", + "end": "2025-03-10T12:45:00-07:00", + }, + { + "start": "2025-03-11T06:00:00-07:00", + "end": "2025-03-11T07:00:00-07:00", + }, + ] + }, + } + } + + calendar_service.calendars().get().execute.return_value = { + "timeZone": "America/Los_Angeles", + } + + response = await find_time_slots_when_everyone_is_free( + context=mock_context, + email_addresses=["example@arcade.dev", "example2@arcade.dev"], + start_date="2025-03-10", + end_date="2025-03-11", + start_time_boundary="08:00", + end_time_boundary="18:00", + ) + + assert response == { + "free_slots": [ + { + "start": { + "datetime": "2025-03-10T09:25:00-07:00", + "weekday": "Monday", + }, + "end": { + "datetime": "2025-03-10T11:00:00-07:00", + "weekday": "Monday", + }, + }, + { + "start": { + "datetime": "2025-03-10T12:45:00-07:00", + "weekday": "Monday", + }, + "end": { + "datetime": "2025-03-10T14:15:00-07:00", + "weekday": "Monday", + }, + }, + { + "start": { + "datetime": "2025-03-10T14:30:00-07:00", + "weekday": "Monday", + }, + "end": { + "datetime": "2025-03-10T18:00:00-07:00", + "weekday": "Monday", + }, + }, + { + "start": { + "datetime": "2025-03-11T08:00:00-07:00", + "weekday": "Tuesday", + }, + "end": { + "datetime": "2025-03-11T18:00:00-07:00", + "weekday": "Tuesday", + }, + }, + ], + "timezone": "America/Los_Angeles", + } + + +@pytest.mark.asyncio +@patch("arcade_google.utils.get_now") +@patch("arcade_google.tools.calendar.build") +async def test_find_free_slots_with_google_calendar_error_not_found( + mock_build, mock_get_now, mock_context +): + calendar_service = MagicMock() + oauth_service = MagicMock() + + mock_get_now.return_value = datetime( + 2025, 3, 10, 9, 25, 0, tzinfo=ZoneInfo("America/Los_Angeles") + ) + mock_build.side_effect = [oauth_service, calendar_service] + + oauth_service.userinfo().get().execute.return_value = { + "email": "example@arcade.dev", + } + + calendar_service.freebusy().query().execute.return_value = { + "calendars": { + "example@arcade.dev": { + "busy": [ + { + "start": "2025-03-10T11:00:00-07:00", + "end": "2025-03-10T12:00:00-07:00", + }, + { + "start": "2025-03-10T14:15:00-07:00", + "end": "2025-03-10T14:30:00-07:00", + }, + ] + }, + "example2@arcade.dev": { + "errors": [ + { + "reason": "notFound", + "domain": "calendar", + } + ] + }, + } + } + + calendar_service.calendars().get().execute.return_value = { + "timeZone": "America/Los_Angeles", + } + + with pytest.raises(RetryableToolError): + await find_time_slots_when_everyone_is_free( + context=mock_context, + email_addresses=["example@arcade.dev", "example2@arcade.dev"], + start_date="2025-03-10", + end_date="2025-03-11", + start_time_boundary="08:00", + end_time_boundary="18:00", + )