admin-valentin/arcade-mcp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Google Calendar tool to find time slots when one or more people (within the same org domain) are simultaneously free (#279)

Browse source
This commit is contained in:
Renato Byrro 2025-04-01 14:53:27 -03:00 committed by GitHub
parent 351814aa74
commit 79e3b03a8e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 1261 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

41
toolkits/google/arcade_google/critics.py Normal file
View file

@ -0,0 +1,41 @@
from collections.abc import Collection
from typing import Any
from arcade.sdk.eval import DatetimeCritic
class DatetimeOrNoneCritic(DatetimeCritic):
"""
A critic that evaluates the closeness of datetime values within a specified tolerance or whether
it's a None value.
Attributes:
tolerance: Acceptable timedelta between expected and actual datetimes.
max_difference: Maximum timedelta for a partial score.
"""
def evaluate(self, expected: Any, actual: Any) -> dict[str, Any]:
if actual is None:
return {"match": True, "score": self.weight}
return super().evaluate(expected, actual)
class AnyDatetimeCritic(DatetimeCritic):
"""
A critic that evaluates the closeness of datetime values within a list of expected values.
"""
def evaluate(self, expected: Any, actual: Any) -> dict[str, Any]:
if not isinstance(expected, Collection):
expected = [expected]
for expected_value in expected:
critic = DatetimeCritic(
critic_field=self.critic_field,
weight=self.weight,
tolerance=self.tolerance,
max_difference=self.max_difference,
)
result = critic.evaluate(expected_value, actual)
if result["match"]:
return result
return {"match": False, "score": 0}

27
toolkits/google/arcade_google/exceptions.py
View file

@ -1,3 +1,8 @@
from zoneinfo import available_timezones
from arcade.sdk.errors import RetryableToolError
class GoogleToolError(Exception):
"""Base exception for Google tool errors."""
@ -13,6 +18,12 @@ class GoogleToolError(Exception):
return base_message
class RetryableGoogleToolError(RetryableToolError):
"""Raised when there's an error in a Google tool that can be retried."""
pass
class GoogleServiceError(GoogleToolError):
"""Raised when there's an error building or using the Google service."""
@ -31,6 +42,22 @@ class GoogleCalendarToolError(GoogleToolError):
pass
class InvalidTimezoneError(RetryableGoogleToolError):
"""Raised when a timezone is provided that is not supported by Python's zoneinfo."""
def __init__(self, timezone_str: str):
self.timezone_str = timezone_str
available_timezones_msg = (
"Here is a list of valid timezones (from Python's zoneinfo.available_timezones()): "
f"{available_timezones()}"
)
super().__init__(
f"Invalid timezone: '{timezone_str}'",
developer_message=available_timezones_msg,
additional_prompt_content=available_timezones_msg,
)
class GoogleDriveToolError(GoogleToolError):
"""Raised when there's an error in the Google Drive tools."""

81
toolkits/google/arcade_google/models.py
View file

@ -18,36 +18,59 @@ class DateRange(Enum):
THIS_MONTH = "this_month"
NEXT_MONTH = "next_month"
def to_date_range(self) -> tuple[date, date]:
today = datetime.now().date()
if self == DateRange.TODAY:
return today, today + timedelta(days=1)
elif self == DateRange.TOMORROW:
return today + timedelta(days=1), today + timedelta(days=2)
elif self == DateRange.THIS_WEEK:
start = today - timedelta(days=today.weekday())
return start, start + timedelta(days=7)
elif self == DateRange.NEXT_WEEK:
start = today + timedelta(days=7 - today.weekday())
return start, start + timedelta(days=7)
elif self == DateRange.THIS_MONTH:
start = today.replace(day=1)
next_month = start + timedelta(days=32)
end = next_month.replace(day=1)
return start, end
elif self == DateRange.NEXT_MONTH:
start = (today.replace(day=1) + timedelta(days=32)).replace(day=1)
next_month = start + timedelta(days=32)
end = next_month.replace(day=1)
return start, end
def to_datetime_range(
self,
start_time: Optional[time] = None,
end_time: Optional[time] = None,
time_zone: Optional[ZoneInfo] = None,
today: Optional[date] = None,
) -> tuple[datetime, datetime]:
"""
Convert a DateRange enum value to a tuple with two datetime objects representing the start
and end of the date range.
:param start_time: The start time of the date range. Defaults to the current time.
:param end_time: The end time of the date range. Defaults to 23:59:59.
:param time_zone: The time zone to use for the date range. Defaults to UTC.
:param today: Today's date. Defaults to the current date provided by `datetime.now().date()`
"""
start_time = start_time or datetime.now().time()
end_time = end_time or time(23, 59, 59)
today = today or datetime.now().date()
if self == DateRange.TODAY:
start_date, end_date = today, today
elif self == DateRange.TOMORROW:
start_date, end_date = today + timedelta(days=1), today + timedelta(days=1)
elif self == DateRange.THIS_WEEK:
start_date = today - timedelta(days=today.weekday())
end_date = start_date + timedelta(days=6)
elif self == DateRange.NEXT_WEEK:
start_date = today + timedelta(days=7 - today.weekday())
end_date = start_date + timedelta(days=6)
elif self == DateRange.THIS_MONTH:
start_date = today.replace(day=1)
next_month = start_date + timedelta(days=31)
end_date = next_month.replace(day=1) - timedelta(days=1)
elif self == DateRange.NEXT_MONTH:
start_date = (today.replace(day=1) + timedelta(days=31)).replace(day=1)
next_month = start_date + timedelta(days=31)
end_date = next_month.replace(day=1) - timedelta(days=1)
else:
raise ValueError(
f"DateRange enum value: {self} is not supported for date range conversion"
)
start_time = start_time or time(0, 0, 0)
end_time = end_time or time(23, 59, 59)
start_datetime = datetime.combine(start_date, start_time)
end_datetime = datetime.combine(end_date, end_time)
if time_zone:
start_datetime = start_datetime.replace(tzinfo=time_zone)
end_datetime = end_datetime.replace(tzinfo=time_zone)
def to_datetime_range(self, time_zone_name: str | None = None) -> tuple[datetime, datetime]:
start_date, end_date = self.to_date_range()
# time_zone = ZoneInfo(time_zone_name)
start_datetime = datetime.combine(
start_date, datetime.min.time()
) # .replace(tzinfo=time_zone)
end_datetime = datetime.combine(end_date, datetime.min.time()) # .replace(tzinfo=time_zone)
return start_datetime, end_datetime

146
toolkits/google/arcade_google/tools/calendar.py
View file

@ -1,5 +1,7 @@
import json
from datetime import datetime, timedelta
from typing import Annotated, Any
from typing import Annotated, Any, Optional
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
from arcade.sdk import ToolContext, tool
from arcade.sdk.auth import Google
@ -9,7 +11,10 @@ from googleapiclient.discovery import build
from googleapiclient.errors import HttpError
from arcade_google.models import EventVisibility, SendUpdatesOptions
from arcade_google.utils import parse_datetime
from arcade_google.utils import (
compute_free_time_intersection,
parse_datetime,
)
@tool(
@ -356,3 +361,140 @@ async def delete_event(
f"Event with ID '{event_id}' successfully deleted from calendar '{calendar_id}'. "
f"{notification_message}"
)
# TODO: would be nice to have a "min_slot_duration" parameter
# TODO: find a way to have "include_weekends" parameter without confusing LLMs
@tool(
requires_auth=Google(
scopes=["https://www.googleapis.com/auth/calendar.readonly"],
),
)
async def find_time_slots_when_everyone_is_free(
context: ToolContext,
email_addresses: Annotated[
Optional[list[str]],
"The list of email addresses from people in the same organization domain (apart from the "
"currently logged in user) to search for free time slots. Defaults to None, which will "
"return free time slots for the current user only.",
] = None,
start_date: Annotated[
Optional[str],
"The start date to search for time slots in the format 'YYYY-MM-DD'. Defaults to today's "
"date. It will search starting from this date at the time 00:00:00.",
] = None,
end_date: Annotated[
Optional[str],
"The end date to search for time slots in the format 'YYYY-MM-DD'. Defaults to seven days "
"from the start date. It will search until this date at the time 23:59:59.",
] = None,
start_time_boundary: Annotated[
str,
"Will return free slots in any given day starting from this time in the format 'HH:MM'. "
"Defaults to '08:00', which is a usual business hour start time.",
] = "08:00",
end_time_boundary: Annotated[
str,
"Will return free slots in any given day until this time in the format 'HH:MM'. "
"Defaults to '18:00', which is a usual business hour end time.",
] = "18:00",
) -> Annotated[
dict,
"A dictionary with the free slots and the timezone in which time slots are represented.",
]:
"""
Provides time slots when everyone is free within a given date range and time boundaries.
"""
credentials = Credentials(
context.authorization.token if context.authorization and context.authorization.token else ""
)
# Build google api services
oauth_service = build("oauth2", "v2", credentials=credentials)
calendar_service = build("calendar", "v3", credentials=credentials)
email_addresses = email_addresses or []
if isinstance(email_addresses, str):
email_addresses = [email_addresses]
# Add the currently logged in user to the list of email addresses
user_info = oauth_service.userinfo().get().execute()
if user_info["email"] not in email_addresses:
email_addresses.append(user_info["email"])
# Get the timezone of the currently logged in user
calendar = calendar_service.calendars().get(calendarId="primary").execute()
timezone_name = calendar.get("timeZone")
try:
tz = ZoneInfo(timezone_name)
# If the calendar timezone name is not supported by Python's zoneinfo, use UTC
except ZoneInfoNotFoundError:
timezone_name = "UTC"
tz = ZoneInfo("UTC")
# Set default start and end dates, if not provided by the caller
start_date = start_date or datetime.now(tz=tz).date().isoformat()
end_date = end_date or (datetime.now(tz=tz).date() + timedelta(days=7)).isoformat()
# Parse start and end dates to datetime objects
start_datetime = datetime.strptime(start_date, "%Y-%m-%d").replace(
hour=0, minute=0, second=0, microsecond=0, tzinfo=tz
)
end_datetime = datetime.strptime(end_date, "%Y-%m-%d").replace(
hour=23, minute=59, second=59, microsecond=0, tzinfo=tz
)
# Get the busy slots from the calendars of the users
freebusy_response = (
calendar_service.freebusy()
.query(
body={
"timeMin": start_datetime.isoformat(),
"timeMax": end_datetime.isoformat(),
"timeZone": timezone_name,
"items": [{"id": email_address} for email_address in email_addresses],
}
)
.execute()
)
busy_slots = freebusy_response["calendars"]
response_errors = []
for email in email_addresses:
if "errors" not in busy_slots[email]:
continue
errors = busy_slots[email]["errors"]
for error in errors:
response_errors.append(
f"Error retrieving free slots from calendar of '{email}': "
f"{error.get('reason', 'not determined')}"
)
if response_errors:
raise RetryableToolError(
"Error retrieving free slots from calendars of one or more users.",
additional_prompt_content=json.dumps(response_errors),
retry_after_ms=1000,
developer_message="Error retrieving free slots from calendars of one or more users.",
)
# Compute the free slots
free_slots = compute_free_time_intersection(
busy_data=busy_slots,
global_start=start_datetime,
global_end=end_datetime,
start_time_boundary=datetime.strptime(start_time_boundary, "%H:%M")
.time()
.replace(tzinfo=tz),
end_time_boundary=datetime.strptime(end_time_boundary, "%H:%M").time().replace(tzinfo=tz),
include_weekends=True,
tz=tz,
)
return {
"free_slots": free_slots,
"timezone": timezone_name,
}

195
toolkits/google/arcade_google/utils.py
View file

@ -1,7 +1,7 @@
import logging
import re
from base64 import urlsafe_b64decode, urlsafe_b64encode
from datetime import datetime, timedelta
from datetime import date, datetime, time, timedelta, timezone
from email.message import EmailMessage
from email.mime.text import MIMEText
from enum import Enum
@ -795,6 +795,199 @@ def build_docs_service(auth_token: Optional[str]) -> Resource: # type: ignore[n
return build("docs", "v1", credentials=Credentials(auth_token))
def parse_rfc3339_datetime_str(dt_str: str, tz: timezone = timezone.utc) -> datetime:
"""
Parse an RFC3339 datetime string into a timezone-aware datetime.
Converts a trailing 'Z' (UTC) into +00:00.
If the parsed datetime is naive, assume it is in the provided timezone.
"""
if dt_str.endswith("Z"):
dt_str = dt_str[:-1] + "+00:00"
dt = datetime.fromisoformat(dt_str)
if dt.tzinfo is None:
dt = dt.replace(tzinfo=tz)
return dt
def merge_intervals(intervals: list[tuple[datetime, datetime]]) -> list[tuple[datetime, datetime]]:
"""
Given a list of (start, end) tuples, merge overlapping or adjacent intervals.
"""
merged: list[tuple[datetime, datetime]] = []
for start, end in sorted(intervals, key=lambda x: x[0]):
if not merged:
merged.append((start, end))
else:
last_start, last_end = merged[-1]
if start <= last_end:
merged[-1] = (last_start, max(last_end, end))
else:
merged.append((start, end))
return merged
# Calendar utils
def weekday_to_name(weekday: int) -> str:
return ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"][weekday]
def get_time_boundaries_for_date(
current_date: date,
global_start: datetime,
global_end: datetime,
start_time_boundary: time,
end_time_boundary: time,
tz: ZoneInfo,
) -> tuple[datetime, datetime]:
"""Compute the allowed start and end times for the given day, adjusting for global bounds."""
day_start_time = datetime.combine(current_date, start_time_boundary).replace(tzinfo=tz)
day_end_time = datetime.combine(current_date, end_time_boundary).replace(tzinfo=tz)
if current_date == global_start.date():
day_start_time = max(day_start_time, global_start)
if current_date == global_end.date():
day_end_time = min(day_end_time, global_end)
return day_start_time, day_end_time
def gather_busy_intervals(
busy_data: dict[str, Any],
day_start: datetime,
day_end: datetime,
business_tz: ZoneInfo,
) -> list[tuple[datetime, datetime]]:
"""
Collect busy intervals from all calendars that intersect with the day's business hours.
Busy intervals are clipped to lie within [day_start, day_end].
"""
busy_intervals = []
for calendar in busy_data:
for slot in busy_data[calendar].get("busy", []):
slot_start = parse_rfc3339_datetime_str(slot["start"]).astimezone(business_tz)
slot_end = parse_rfc3339_datetime_str(slot["end"]).astimezone(business_tz)
if slot_end > day_start and slot_start < day_end:
busy_intervals.append((max(slot_start, day_start), min(slot_end, day_end)))
return busy_intervals
def subtract_busy_intervals(
business_start: datetime,
business_end: datetime,
busy_intervals: list[tuple[datetime, datetime]],
) -> list[dict[str, Any]]:
"""
Subtract the merged busy intervals from the business hours and return free time slots.
"""
free_slots = []
merged_busy = merge_intervals(busy_intervals)
# If there are no busy intervals, return the entire business window as free.
if not merged_busy:
return [
{
"start": {
"datetime": business_start.isoformat(),
"weekday": weekday_to_name(business_start.weekday()),
},
"end": {
"datetime": business_end.isoformat(),
"weekday": weekday_to_name(business_end.weekday()),
},
}
]
current_free_start = business_start
for busy_start, busy_end in merged_busy:
if current_free_start < busy_start:
free_slots.append({
"start": {
"datetime": current_free_start.isoformat(),
"weekday": weekday_to_name(current_free_start.weekday()),
},
"end": {
"datetime": busy_start.isoformat(),
"weekday": weekday_to_name(busy_start.weekday()),
},
})
current_free_start = max(current_free_start, busy_end)
if current_free_start < business_end:
free_slots.append({
"start": {
"datetime": current_free_start.isoformat(),
"weekday": weekday_to_name(current_free_start.weekday()),
},
"end": {
"datetime": business_end.isoformat(),
"weekday": weekday_to_name(business_end.weekday()),
},
})
return free_slots
def compute_free_time_intersection(
busy_data: dict[str, Any],
global_start: datetime,
global_end: datetime,
start_time_boundary: time,
end_time_boundary: time,
include_weekends: bool,
tz: ZoneInfo,
) -> list[dict[str, Any]]:
"""
Returns the free time slots across all calendars within the global bounds,
ensuring that the global start is not in the past.
Only considers business days (Monday to Friday) and business hours (08:00-19:00)
in the provided timezone.
"""
# Ensure global_start is never in the past relative to now.
now = get_now(tz)
if now > global_start:
global_start = now
# If after adjusting the start, there's no interval left, return empty.
if global_start >= global_end:
return []
free_slots = []
current_date = global_start.date()
while current_date <= global_end.date():
if not include_weekends and current_date.weekday() >= 5:
current_date += timedelta(days=1)
continue
day_start, day_end = get_time_boundaries_for_date(
current_date=current_date,
global_start=global_start,
global_end=global_end,
start_time_boundary=start_time_boundary,
end_time_boundary=end_time_boundary,
tz=tz,
)
# Skip if the day's allowed time window is empty.
if day_start >= day_end:
current_date += timedelta(days=1)
continue
busy_intervals = gather_busy_intervals(busy_data, day_start, day_end, tz)
free_slots.extend(subtract_busy_intervals(day_start, day_end, busy_intervals))
current_date += timedelta(days=1)
return free_slots
def get_now(tz: ZoneInfo | None = None) -> datetime:
if not tz:
tz = ZoneInfo("UTC")
return datetime.now(tz)
# Contacts utils
def build_people_service(auth_token: Optional[str]) -> Resource: # type: ignore[no-any-unimported]
"""

470
toolkits/google/evals/eval_calendar_free_slots.py Normal file
View file

@ -0,0 +1,470 @@
from datetime import timedelta
from arcade.sdk import ToolCatalog
from arcade.sdk.eval import (
BinaryCritic,
DatetimeCritic,
EvalRubric,
EvalSuite,
ExpectedToolCall,
NoneCritic,
tool_eval,
)
import arcade_google
from arcade_google.critics import AnyDatetimeCritic, DatetimeOrNoneCritic
from arcade_google.tools.calendar import find_time_slots_when_everyone_is_free
rubric = EvalRubric(
fail_threshold=0.9,
warn_threshold=0.95,
)
catalog = ToolCatalog()
catalog.add_module(arcade_google)
@tool_eval()
def get_free_slots_eval_suite() -> EvalSuite:
"""Create an evaluation suite for free slots Calendar tool."""
suite = EvalSuite(
name="Calendar Tools Evaluation",
system_message=(
"You are an AI assistant that can manage calendars and events using the provided tools. "
"The first day of a week is Monday and the last day is Sunday. "
"Today is Thursday, March 6, 2025 (2025-03-06). "
"This week started on Monday, March 3, 2025 (2025-03-03) and ends on Sunday, March 9, 2025 (2025-03-09). "
"Last week started on Monday, February 24, 2025 (2025-02-24) and ended on Sunday, March 2, 2025 (2025-03-02). "
"Next week starts on Monday, March 10, 2025 (2025-03-10) and ends on Sunday, March 16, 2025 (2025-03-16). "
"This month started on March 1, 2025 (2025-03-01) and ends on March 31, 2025 (2025-03-31). "
"Last month started on February 1, 2025 (2025-02-01) and ended on February 28, 2025 (2025-02-28). "
"Next month starts on April 1, 2025 (2025-04-01) and ends on April 30, 2025 (2025-04-30). "
"This quarter started on January 1, 2025 (2025-01-01) and ends on March 31, 2025 (2025-03-31). "
"Last quarter started on December 1, 2024 (2024-12-01) and ended on December 31, 2024 (2024-12-31). "
"Next quarter starts on April 1, 2025 (2025-04-01) and ends on June 30, 2025 (2025-06-30). "
),
catalog=catalog,
rubric=rubric,
)
suite.add_case(
name="Get free slots for the next 5 days",
user_message=("At what times am I free in the next 5 days?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-03-06",
"end_date": "2025-03-11",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
DatetimeOrNoneCritic(
critic_field="start_date", weight=0.35, tolerance=timedelta(days=1)
),
DatetimeCritic(critic_field="end_date", weight=0.35, tolerance=timedelta(days=1)),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots for the next 10 days",
user_message=("At what times am I free in the next 10 days?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-03-06",
"end_date": "2025-03-16",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
DatetimeOrNoneCritic(
critic_field="start_date", weight=0.35, tolerance=timedelta(days=1)
),
DatetimeCritic(critic_field="end_date", weight=0.35, tolerance=timedelta(days=1)),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots this week",
user_message=("At what times am I free this week?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
# Models sometimes will consider today as the start range, other times it will
# consider last Monday. The question is ambiguous, so we allow both.
"start_date": ["2025-03-03", "2025-03-06"],
"end_date": "2025-03-09",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
AnyDatetimeCritic(critic_field="start_date", weight=0.35),
BinaryCritic(critic_field="end_date", weight=0.35),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots next week",
user_message=("At what times am I free next week?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-03-10",
"end_date": "2025-03-16",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
BinaryCritic(critic_field="start_date", weight=0.35),
BinaryCritic(critic_field="end_date", weight=0.35),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots today",
user_message=("At what times am I free today?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-03-06",
"end_date": "2025-03-06",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
DatetimeOrNoneCritic(critic_field="start_date", weight=0.35),
DatetimeCritic(critic_field="end_date", weight=0.35),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots today",
user_message=("At what times am I free tonight before 10 PM?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-03-06",
"end_date": "2025-03-06",
"start_time_boundary": "08:00",
"end_time_boundary": "22:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
DatetimeOrNoneCritic(critic_field="start_date", weight=0.35),
DatetimeCritic(critic_field="end_date", weight=0.35),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots this weekend",
user_message=("At what times am I free this weekend?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-03-08",
"end_date": "2025-03-09",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
DatetimeCritic(critic_field="start_date", weight=0.35),
DatetimeCritic(critic_field="end_date", weight=0.35),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots this month",
user_message=("At what times am I free this month?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": ["2025-03-06", "2025-03-01"],
"end_date": "2025-03-31",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
AnyDatetimeCritic(critic_field="start_date", weight=0.35),
DatetimeCritic(critic_field="end_date", weight=0.35),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots next month",
user_message=("At what times am I free next month?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-04-01",
"end_date": "2025-04-30",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
DatetimeCritic(critic_field="start_date", weight=0.35),
DatetimeCritic(critic_field="end_date", weight=0.35),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots last week",
user_message=("At what times was I free last week?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-02-24",
"end_date": "2025-03-02",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
DatetimeCritic(critic_field="start_date", weight=0.35),
DatetimeCritic(critic_field="end_date", weight=0.35),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots next quarter",
user_message=("At what times am I free next quarter?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-04-01",
"end_date": "2025-06-30",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
DatetimeCritic(critic_field="start_date", weight=0.35),
DatetimeCritic(critic_field="end_date", weight=0.35),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots in the next 30 days",
user_message=("At what times am I free in the next 30 days?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-03-06",
"end_date": "2025-04-05",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
DatetimeOrNoneCritic(
critic_field="start_date", weight=0.35, tolerance=timedelta(days=1)
),
DatetimeCritic(critic_field="end_date", weight=0.35, tolerance=timedelta(days=1)),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots in April",
user_message=("At what times am I free in April?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-04-01",
"end_date": "2025-04-30",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
DatetimeCritic(critic_field="start_date", weight=0.35),
DatetimeCritic(critic_field="end_date", weight=0.35),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots for a specific email address",
user_message=("Is johndoe@example.com free some time tomorrow?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": ["johndoe@example.com"],
"start_date": "2025-03-07",
"end_date": "2025-03-07",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
BinaryCritic(critic_field="email_addresses", weight=0.30),
DatetimeCritic(critic_field="start_date", weight=0.25),
DatetimeCritic(critic_field="end_date", weight=0.25),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots for a specific email address",
user_message=(
"I need to schedule a meeting with johndoe@example.com tomorrow. When are both of us free?"
),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": ["johndoe@example.com"],
"start_date": "2025-03-07",
"end_date": "2025-03-07",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
BinaryCritic(critic_field="email_addresses", weight=0.3),
DatetimeCritic(critic_field="start_date", weight=0.25),
DatetimeCritic(critic_field="end_date", weight=0.25),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
suite.add_case(
name="Get free slots for a specific email address",
user_message=(
"I need to schedule a meeting with johndoe@example.com tomorrow morning. When are both of us free?"
),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": ["johndoe@example.com"],
"start_date": "2025-03-07",
"end_date": "2025-03-07",
"start_time_boundary": "08:00",
"end_time_boundary": "12:00",
},
)
],
critics=[
BinaryCritic(critic_field="email_addresses", weight=0.2),
DatetimeCritic(critic_field="start_date", weight=0.2),
DatetimeCritic(critic_field="end_date", weight=0.2),
BinaryCritic(critic_field="start_time_boundary", weight=0.2),
BinaryCritic(critic_field="end_time_boundary", weight=0.2),
],
)
suite.add_case(
name="Get free slots for a specific date range",
user_message=("At what times am I free between 2025-04-27 and 2025-04-29?"),
expected_tool_calls=[
ExpectedToolCall(
func=find_time_slots_when_everyone_is_free,
args={
"email_addresses": None,
"start_date": "2025-04-27",
"end_date": "2025-04-29",
"start_time_boundary": "08:00",
"end_time_boundary": "18:00",
},
)
],
critics=[
NoneCritic(critic_field="email_addresses", weight=0.1),
DatetimeCritic(critic_field="start_date", weight=0.35),
DatetimeCritic(critic_field="end_date", weight=0.35),
BinaryCritic(critic_field="start_time_boundary", weight=0.1),
BinaryCritic(critic_field="end_time_boundary", weight=0.1),
],
)
return suite

3
toolkits/google/evals/eval_google_calendar.py
View file

@ -12,13 +12,12 @@ from arcade.sdk.eval import (
import arcade_google
from arcade_google.tools.calendar import (
EventVisibility,
SendUpdatesOptions,
create_event,
delete_event,
list_events,
update_event,
)
from arcade_google.tools.models import EventVisibility, SendUpdatesOptions
# Evaluation rubric
rubric = EvalRubric(

334
toolkits/google/tests/test_calendar.py
View file

@ -1,12 +1,20 @@
from datetime import datetime
from unittest.mock import MagicMock, patch
from zoneinfo import ZoneInfo
import pytest
from arcade.sdk import ToolAuthorizationContext, ToolContext
from arcade.sdk.errors import ToolExecutionError
from arcade.sdk.errors import RetryableToolError, ToolExecutionError
from googleapiclient.errors import HttpError
from arcade_google.models import EventVisibility, SendUpdatesOptions
from arcade_google.tools.calendar import create_event, delete_event, list_events, update_event
from arcade_google.tools.calendar import (
create_event,
delete_event,
find_time_slots_when_everyone_is_free,
list_events,
update_event,
)
@pytest.fixture
@ -152,3 +160,325 @@ async def test_delete_event(mock_build, mock_context):
event_id="nonexistent_event",
send_updates=SendUpdatesOptions.ALL,
)
@pytest.mark.asyncio
@patch("arcade_google.utils.get_now")
@patch("arcade_google.tools.calendar.build")
async def test_find_free_slots_happiest_path_single_user(mock_build, mock_get_now, mock_context):
calendar_service = MagicMock()
oauth_service = MagicMock()
mock_get_now.return_value = datetime(
2025, 3, 10, 9, 25, 0, tzinfo=ZoneInfo("America/Los_Angeles")
)
mock_build.side_effect = [oauth_service, calendar_service]
oauth_service.userinfo().get().execute.return_value = {
"email": "example@arcade.dev",
}
calendar_service.freebusy().query().execute.return_value = {
"calendars": {
"example@arcade.dev": {"busy": []},
}
}
calendar_service.calendars().get().execute.return_value = {
"timeZone": "America/Los_Angeles",
}
response = await find_time_slots_when_everyone_is_free(
context=mock_context,
email_addresses=["example@arcade.dev"],
start_date="2025-03-10",
end_date="2025-03-11",
start_time_boundary="08:00",
end_time_boundary="18:00",
)
assert response == {
"free_slots": [
{
"start": {
"datetime": "2025-03-10T09:25:00-07:00",
"weekday": "Monday",
},
"end": {
"datetime": "2025-03-10T18:00:00-07:00",
"weekday": "Monday",
},
},
{
"start": {
"datetime": "2025-03-11T08:00:00-07:00",
"weekday": "Tuesday",
},
"end": {
"datetime": "2025-03-11T18:00:00-07:00",
"weekday": "Tuesday",
},
},
],
"timezone": "America/Los_Angeles",
}
@pytest.mark.asyncio
@patch("arcade_google.utils.get_now")
@patch("arcade_google.tools.calendar.build")
async def test_find_free_slots_happiest_path_single_user_with_busy_times(
mock_build, mock_get_now, mock_context
):
calendar_service = MagicMock()
oauth_service = MagicMock()
mock_get_now.return_value = datetime(
2025, 3, 10, 9, 25, 0, tzinfo=ZoneInfo("America/Los_Angeles")
)
mock_build.side_effect = [oauth_service, calendar_service]
oauth_service.userinfo().get().execute.return_value = {
"email": "example@arcade.dev",
}
calendar_service.freebusy().query().execute.return_value = {
"calendars": {
"example@arcade.dev": {
"busy": [
{
"start": "2025-03-10T11:00:00-07:00",
"end": "2025-03-10T12:00:00-07:00",
},
{
"start": "2025-03-10T14:15:00-07:00",
"end": "2025-03-10T14:30:00-07:00",
},
]
},
}
}
calendar_service.calendars().get().execute.return_value = {
"timeZone": "America/Los_Angeles",
}
response = await find_time_slots_when_everyone_is_free(
context=mock_context,
email_addresses=["example@arcade.dev"],
start_date="2025-03-10",
end_date="2025-03-11",
start_time_boundary="08:00",
end_time_boundary="18:00",
)
assert response == {
"free_slots": [
{
"start": {
"datetime": "2025-03-10T09:25:00-07:00",
"weekday": "Monday",
},
"end": {
"datetime": "2025-03-10T11:00:00-07:00",
"weekday": "Monday",
},
},
{
"start": {
"datetime": "2025-03-10T12:00:00-07:00",
"weekday": "Monday",
},
"end": {
"datetime": "2025-03-10T14:15:00-07:00",
"weekday": "Monday",
},
},
{
"start": {
"datetime": "2025-03-10T14:30:00-07:00",
"weekday": "Monday",
},
"end": {
"datetime": "2025-03-10T18:00:00-07:00",
"weekday": "Monday",
},
},
{
"start": {
"datetime": "2025-03-11T08:00:00-07:00",
"weekday": "Tuesday",
},
"end": {
"datetime": "2025-03-11T18:00:00-07:00",
"weekday": "Tuesday",
},
},
],
"timezone": "America/Los_Angeles",
}
@pytest.mark.asyncio
@patch("arcade_google.utils.get_now")
@patch("arcade_google.tools.calendar.build")
async def test_find_free_slots_happiest_path_multiple_users_with_busy_times(
mock_build, mock_get_now, mock_context
):
calendar_service = MagicMock()
oauth_service = MagicMock()
mock_get_now.return_value = datetime(
2025, 3, 10, 9, 25, 0, tzinfo=ZoneInfo("America/Los_Angeles")
)
mock_build.side_effect = [oauth_service, calendar_service]
oauth_service.userinfo().get().execute.return_value = {
"email": "example@arcade.dev",
}
calendar_service.freebusy().query().execute.return_value = {
"calendars": {
"example@arcade.dev": {
"busy": [
{
"start": "2025-03-10T11:00:00-07:00",
"end": "2025-03-10T12:00:00-07:00",
},
{
"start": "2025-03-10T14:15:00-07:00",
"end": "2025-03-10T14:30:00-07:00",
},
]
},
"example2@arcade.dev": {
"busy": [
{
"start": "2025-03-10T11:30:00-07:00",
"end": "2025-03-10T12:45:00-07:00",
},
{
"start": "2025-03-11T06:00:00-07:00",
"end": "2025-03-11T07:00:00-07:00",
},
]
},
}
}
calendar_service.calendars().get().execute.return_value = {
"timeZone": "America/Los_Angeles",
}
response = await find_time_slots_when_everyone_is_free(
context=mock_context,
email_addresses=["example@arcade.dev", "example2@arcade.dev"],
start_date="2025-03-10",
end_date="2025-03-11",
start_time_boundary="08:00",
end_time_boundary="18:00",
)
assert response == {
"free_slots": [
{
"start": {
"datetime": "2025-03-10T09:25:00-07:00",
"weekday": "Monday",
},
"end": {
"datetime": "2025-03-10T11:00:00-07:00",
"weekday": "Monday",
},
},
{
"start": {
"datetime": "2025-03-10T12:45:00-07:00",
"weekday": "Monday",
},
"end": {
"datetime": "2025-03-10T14:15:00-07:00",
"weekday": "Monday",
},
},
{
"start": {
"datetime": "2025-03-10T14:30:00-07:00",
"weekday": "Monday",
},
"end": {
"datetime": "2025-03-10T18:00:00-07:00",
"weekday": "Monday",
},
},
{
"start": {
"datetime": "2025-03-11T08:00:00-07:00",
"weekday": "Tuesday",
},
"end": {
"datetime": "2025-03-11T18:00:00-07:00",
"weekday": "Tuesday",
},
},
],
"timezone": "America/Los_Angeles",
}
@pytest.mark.asyncio
@patch("arcade_google.utils.get_now")
@patch("arcade_google.tools.calendar.build")
async def test_find_free_slots_with_google_calendar_error_not_found(
mock_build, mock_get_now, mock_context
):
calendar_service = MagicMock()
oauth_service = MagicMock()
mock_get_now.return_value = datetime(
2025, 3, 10, 9, 25, 0, tzinfo=ZoneInfo("America/Los_Angeles")
)
mock_build.side_effect = [oauth_service, calendar_service]
oauth_service.userinfo().get().execute.return_value = {
"email": "example@arcade.dev",
}
calendar_service.freebusy().query().execute.return_value = {
"calendars": {
"example@arcade.dev": {
"busy": [
{
"start": "2025-03-10T11:00:00-07:00",
"end": "2025-03-10T12:00:00-07:00",
},
{
"start": "2025-03-10T14:15:00-07:00",
"end": "2025-03-10T14:30:00-07:00",
},
]
},
"example2@arcade.dev": {
"errors": [
{
"reason": "notFound",
"domain": "calendar",
}
]
},
}
}
calendar_service.calendars().get().execute.return_value = {
"timeZone": "America/Los_Angeles",
}
with pytest.raises(RetryableToolError):
await find_time_slots_when_everyone_is_free(
context=mock_context,
email_addresses=["example@arcade.dev", "example2@arcade.dev"],
start_date="2025-03-10",
end_date="2025-03-11",
start_time_boundary="08:00",
end_time_boundary="18:00",
)