bf53439b55
This PR adds four new tools to the Google ToolKit * `create_event` * `list_events` * `update_event` * `delete_event` I also improved an error log when tools are being registered by the actor. This PR also sneaks in an eval for gmail Here is a sample conversation that shows the tools and their capabilities and limitiations: 
176 lines
8.5 KiB
Python
176 lines
8.5 KiB
Python
import arcade_google
|
|
from arcade_google.tools.calendar import create_event, delete_event, list_events, update_event
|
|
from arcade_google.tools.models import Day, EventVisibility, TimeSlot
|
|
|
|
from arcade.core.catalog import ToolCatalog
|
|
from arcade.sdk.eval import (
|
|
BinaryCritic,
|
|
EvalRubric,
|
|
EvalSuite,
|
|
tool_eval,
|
|
)
|
|
|
|
# Evaluation rubric
|
|
rubric = EvalRubric(
|
|
fail_threshold=0.9,
|
|
warn_threshold=0.95,
|
|
)
|
|
|
|
|
|
catalog = ToolCatalog()
|
|
catalog.add_module(arcade_google)
|
|
|
|
|
|
history_after_list_events = [
|
|
{"role": "user", "content": "do i have any events on my calendar for today?"},
|
|
{
|
|
"role": "assistant",
|
|
"content": "Please go to this URL and authorize the action: \n[Link](https://accounts.google.com/o/oauth2/v2/auth?)",
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": "",
|
|
"tool_calls": [
|
|
{
|
|
"id": "call_uHdRlr4z7sFm39ZrPsE5wcdT",
|
|
"type": "function",
|
|
"function": {
|
|
"name": "Google_ListEvents",
|
|
"arguments": '{"max_day":"today","max_time_slot":"23:45","min_day":"today","min_time_slot":"00:00"}',
|
|
},
|
|
}
|
|
],
|
|
},
|
|
{
|
|
"role": "tool",
|
|
"content": '{"events_count": 3, "events": [{"creator": {"email": "john@example.com", "self": true}, "description": "1:1 meeting with Joe", "end": {"dateTime": "2024-09-26T00:15:00-07:00", "timeZone": "America/Los_Angeles"}, "eventType": "default", "htmlLink": "https://www.google.com/calendar/event?eid=01234", "id": "10009199283838467", "location": "622 Rainbow Ave, South San Francisco, CA 94080, USA", "organizer": {"email": "john@example.com", "self": true}, "start": {"dateTime": "2024-09-25T23:15:00-07:00", "timeZone": "America/Los_Angeles"}, "summary": "1:1 meeting"}, {"attendees": [{"email": "joe@example.com", "responseStatus": "accepted"}], "creator": {"email": "john@example.com", "self": true}, "description": "This is just a test", "end": {"dateTime": "2024-09-26T14:00:00-07:00", "timeZone": "America/Los_Angeles"}, "eventType": "default", "htmlLink": "https://www.google.com/calendar/event?eid=OXB2OGFwcmZraWk1N234324", "id": "00099992228181818181", "organizer": {"email": "john@example.com", "self": true}, "start": {"dateTime": "2024-09-26T12:00:00-07:00", "timeZone": "America/Los_Angeles"}, "summary": "API test"}, {"attendees": [{"email": "henry@example.com", "responseStatus": "needsAction"}], "creator": {"email": "john@example.com", "self": true}, "end": {"dateTime": "2024-09-26T17:00:00-07:00", "timeZone": "America/Los_Angeles"}, "eventType": "default", "htmlLink": "https://www.google.com/calendar/event?eid=Z3I1ZzE4b324534556", "id": "gr5g18lf88tfpp3vkareukkc7g", "location": "611 Rainbow Road", "organizer": {"email": "john@example.com", "self": true}, "start": {"dateTime": "2024-09-26T15:00:00-07:00", "timeZone": "America/Los_Angeles"}, "summary": "Focus Time", "visibility": "public"}]}',
|
|
"tool_call_id": "call_uHdRlr4z7sFm39ZrPsE5wcdT",
|
|
"name": "Google_ListEvents",
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": "Yes, you have three events on your calendar for today:\n\n1. **Event:** Test2\n - **Time:** 23:15 - 00:15 (PST)\n - **Location:** 611 Gateway Blvd, South San Francisco, CA 94080, USA\n - **Description:** 1:1 meeting with Joe\n 2. **Event:** API Test\n - **Time:** 12:00 - 14:00 (PST)\n **Description:** This is just a test\n - [View Event](https://www.google.com/calendar/event?eid=OXB2OGFwcmZraWk1NGUwa24xaTNya2lvZDggZXJpY0BhcmNhZGUtYWkuY29t)\n\n3. **Event:** Focus Time\n - **Time:** 15:00 - 17:00 (PST)\n - **Location:** 611 Rainbow Road\n - **Visibility:** Public\n - [View Event](https://www.google.com/calendar/event?eid=Z3I1ZzE4bGY4OHRmcHAzdmthcmV1a2tjN2cgZXJpY0BhcmNhZGUtYWkuY29t)\n\nIf you need more details or help with anything else, feel free to ask!",
|
|
},
|
|
]
|
|
|
|
|
|
@tool_eval()
|
|
def calendar_eval_suite() -> EvalSuite:
|
|
"""Create an evaluation suite for Calendar tools."""
|
|
suite = EvalSuite(
|
|
name="Calendar Tools Evaluation",
|
|
system_message="You are an AI assistant that can create and list events using the provided tools.",
|
|
catalog=catalog,
|
|
rubric=rubric,
|
|
)
|
|
|
|
# Cases for create_event
|
|
suite.add_case(
|
|
name="Create calendar event",
|
|
user_message="Create a meeting for 'Team Meeting' starting next thursday from 11:45pm to 12:15am. Invite johndoe@example.com",
|
|
expected_tool_calls=[
|
|
(
|
|
create_event,
|
|
{
|
|
"summary": "Team Meeting",
|
|
"start_date": Day.NEXT_THURSDAY.value,
|
|
"start_time": TimeSlot._2345.value,
|
|
"end_date": Day.NEXT_FRIDAY.value,
|
|
"end_time": TimeSlot._0015.value,
|
|
"calendar_id": "primary",
|
|
"attendee_emails": ["johndoe@example.com"],
|
|
"description": None,
|
|
"location": None,
|
|
"visibility": EventVisibility.DEFAULT,
|
|
},
|
|
)
|
|
],
|
|
critics=[
|
|
BinaryCritic(critic_field="summary", weight=0.15),
|
|
BinaryCritic(critic_field="start_date", weight=0.15),
|
|
BinaryCritic(critic_field="start_time", weight=0.15),
|
|
BinaryCritic(critic_field="end_date", weight=0.15),
|
|
BinaryCritic(critic_field="end_time", weight=0.15),
|
|
BinaryCritic(critic_field="attendee_emails", weight=0.15),
|
|
BinaryCritic(critic_field="description", weight=0.1),
|
|
],
|
|
)
|
|
|
|
# Cases for list_events
|
|
suite.add_case(
|
|
name="List calendar events",
|
|
user_message="Do I have any events on my calendar today?",
|
|
expected_tool_calls=[
|
|
(
|
|
list_events,
|
|
{
|
|
"min_day": Day.TODAY.value,
|
|
"min_time_slot": TimeSlot._0000.value,
|
|
"max_day": Day.TOMORROW.value,
|
|
"max_time_slot": TimeSlot._0000.value,
|
|
"calendar_id": "primary",
|
|
"event_types": None,
|
|
"max_results": 10,
|
|
},
|
|
)
|
|
],
|
|
critics=[
|
|
BinaryCritic(critic_field="min_day", weight=0.1),
|
|
BinaryCritic(critic_field="min_time_slot", weight=0.1),
|
|
BinaryCritic(critic_field="max_day", weight=0.1),
|
|
BinaryCritic(critic_field="max_time_slot", weight=0.1),
|
|
BinaryCritic(critic_field="calendar_id", weight=0.1),
|
|
BinaryCritic(critic_field="event_types", weight=0.1),
|
|
BinaryCritic(critic_field="max_results", weight=0.1),
|
|
],
|
|
)
|
|
|
|
# Cases for update_event
|
|
suite.add_case(
|
|
name="Update a calendar event",
|
|
user_message="Oh no! I cant make it to the API Test since i have lunch with an old friend at that time. Change the meeting to 3pm to 4pm please.",
|
|
expected_tool_calls=[
|
|
(
|
|
update_event,
|
|
{
|
|
"event_id": "00099992228181818181",
|
|
"updated_start_day": Day.TODAY.value,
|
|
"updated_start_time": TimeSlot._1500.value,
|
|
"updated_end_day": Day.TODAY.value,
|
|
"updated_end_time": TimeSlot._1600.value,
|
|
},
|
|
)
|
|
],
|
|
critics=[
|
|
BinaryCritic(critic_field="event_id", weight=0.2),
|
|
BinaryCritic(critic_field="updated_start_day", weight=0.1),
|
|
BinaryCritic(critic_field="updated_start_time", weight=0.1),
|
|
BinaryCritic(critic_field="updated_end_day", weight=0.1),
|
|
BinaryCritic(critic_field="updated_end_time", weight=0.1),
|
|
],
|
|
additional_messages=history_after_list_events,
|
|
)
|
|
|
|
# Cases for delete_event
|
|
suite.add_case(
|
|
name="Delete a calendar event",
|
|
user_message="I don't need to have focus time today. Please delete it from my calendar. Don't send any notifications.",
|
|
expected_tool_calls=[
|
|
(
|
|
delete_event,
|
|
{
|
|
"event_id": "gr5g18lf88tfpp3vkareukkc7g",
|
|
"calendar_id": "primary",
|
|
"send_updates": "none",
|
|
},
|
|
)
|
|
],
|
|
critics=[
|
|
BinaryCritic(critic_field="event_id", weight=0.6),
|
|
BinaryCritic(critic_field="calendar_id", weight=0.2),
|
|
BinaryCritic(critic_field="send_updates", weight=0.2),
|
|
],
|
|
additional_messages=history_after_list_events,
|
|
)
|
|
|
|
return suite
|