07c52100f3
# PR Description ## Split toolkits This PR splits the `Microsoft`, `Google`, and `Search` toolkits into multiple toolkits each. * `Microsoft` --> `OutlookCalendar`, `OutlookMail`. * `Google` -----> `GoogleCalendar`, `GoogleContacts`, `GoogleDocs`, `GoogleDrive`, `Gmail`, `GoogleSheets` * `Search` -----> `GoogleFinance`, `GoogleFlights`, `GoogleHotels`, `GoogleJobs`, `GoogleMaps`, `GoogleNews`, `GoogleSearch`, `GoogleShopping`, `Walmart`, `Youtube` > The original monolithic toolkits (`Microsoft`, `Google`, `Search`) are not removed in this PR. The plan is to keep those toolkits around while we > 1. Stop documenting the toolkits, > 2. Stop displaying the toolkits in the dashboard, and > 3. Help customers migrate over to the new split toolkits. ## Rename toolkits This PR renames the following toolkits * `Web` ------------> `Firecrawl` * `CodeSandbox` ---> `E2B` > The `Web` and `CodeSandbox` toolkits are not removed in this PR. The plan is to keep them around while we > 1. Stop documenting the toolkits, > 2. Stop displaying the toolkits in the dashboard, and > 3. Help customers migrate over to the new renamed toolkits. ## Rename tools Since toolkit names were changed, this called for some tools to be renamed as well. * `GoogleSearch.SearchGoogle` ----------------> `GoogleSearch.Search` * `GoogleShopping.SearchShoppingProducts` ---> `GoogleShopping.SearchProducts` * `Walmart.SearchWalmartProducts` ------------> `Walmart.SearchProducts` * `Walmart.GetWalmartProductDetails` ---------> `Walmart.GetProductDetails` * `Youtube.SearchYoutubeVideos` --------------> `Youtube.SearchForVideos` ## Google File Picker Improvements to the Google File Picker experience were also added in this PR. The following tools will ALWAYS provide llm_instructions in their response to "let the end-user know that they have the option to select more files via the file picker url if they want to": * `GoogleDocs.SearchDocuments` * `GoogleDocs.SearchAndRetrieveDocuments` * `GoogleDrive.GetFileTreeStructure` The following tools will only provide the file picker URL if a 404 or 403 from the Google API: * `GoogleDocs.InsertTextAtEndOfDocument` * `GoogleDocs.GetDocumentById` * `GoogleSheets.GetSpreadsheet` * `GoogleSheets.WriteToCell` Also, a standalone `GoogleDrive.GenerateGoogleFilePickerUrl` tool exists. ## Other * The `SearchDocuments` and `SearchAndRetrieveDocuments` tools used to be organized within the Drive portion of the Google toolkit, but I moved these into the new GoogleDocs toolkit because they are specific to Docs. # Progress - [x] `OutlookCalendar` - [x] `OutlookMail` - [x] `GoogleFinance` - [x] `GoogleFlights` - [x] `GoogleHotels` - [x] `GoogleJobs` - [x] `GoogleMaps` - [x] `GoogleNews` - [x] `GoogleSearch` - [x] `GoogleShopping` - [x] `Walmart` - [x] `Youtube` - [x] `GoogleCalendar` - [x] `GoogleContacts` - [x] `GoogleDocs` - [x] `GoogleDrive` - [x] `Gmail` - [x] `GoogleSheets` - [x] `Firecrawl` - [x] `E2B` - [x] File picker # Discussion * Repeated code is a consequence of splitting toolkits that use the same provider. I am open to any ideas that would allow multiple toolkits to reference common code. Comment your ideas in this PR.
226 lines
10 KiB
Python
226 lines
10 KiB
Python
from arcade_evals import (
|
|
BinaryCritic,
|
|
EvalRubric,
|
|
EvalSuite,
|
|
ExpectedToolCall,
|
|
SimilarityCritic,
|
|
tool_eval,
|
|
)
|
|
from arcade_tdk import ToolCatalog
|
|
|
|
import arcade_google_maps
|
|
from arcade_google_maps.constants import (
|
|
DEFAULT_GOOGLE_MAPS_COUNTRY,
|
|
DEFAULT_GOOGLE_MAPS_DISTANCE_UNIT,
|
|
DEFAULT_GOOGLE_MAPS_LANGUAGE,
|
|
DEFAULT_GOOGLE_MAPS_TRAVEL_MODE,
|
|
)
|
|
from arcade_google_maps.enums import GoogleMapsDistanceUnit, GoogleMapsTravelMode
|
|
from arcade_google_maps.tools.google_maps import (
|
|
get_directions_between_addresses,
|
|
get_directions_between_coordinates,
|
|
)
|
|
|
|
# Evaluation rubric
|
|
rubric = EvalRubric(
|
|
fail_threshold=0.8,
|
|
warn_threshold=0.9,
|
|
)
|
|
|
|
catalog = ToolCatalog()
|
|
# Register the Google Search tool
|
|
catalog.add_module(arcade_google_maps)
|
|
|
|
|
|
@tool_eval()
|
|
def google_maps_directions_by_addresses_eval_suite() -> EvalSuite:
|
|
"""Create an evaluation suite for the Google Maps Directions tools."""
|
|
suite = EvalSuite(
|
|
name="Google Maps Directions Tool Evaluation",
|
|
system_message="You are an AI assistant that can get directions from Google Maps using the provided tools.",
|
|
catalog=catalog,
|
|
rubric=rubric,
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Get directions between two addresses",
|
|
user_message="Get directions from Google Maps between the following addresses: 1600 Amphitheatre Parkway, Mountain View, CA 94043 and 1 Infinite Loop, Cupertino, CA 95014.",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=get_directions_between_addresses,
|
|
args={
|
|
"origin_address": "1600 Amphitheatre Parkway, Mountain View, CA 94043",
|
|
"destination_address": "1 Infinite Loop, Cupertino, CA 95014",
|
|
"language": DEFAULT_GOOGLE_MAPS_LANGUAGE,
|
|
"country": DEFAULT_GOOGLE_MAPS_COUNTRY,
|
|
"distance_unit": DEFAULT_GOOGLE_MAPS_DISTANCE_UNIT,
|
|
"travel_mode": DEFAULT_GOOGLE_MAPS_TRAVEL_MODE,
|
|
},
|
|
)
|
|
],
|
|
critics=[
|
|
SimilarityCritic(critic_field="origin_address", weight=0.3),
|
|
SimilarityCritic(critic_field="destination_address", weight=0.3),
|
|
BinaryCritic(critic_field="language", weight=0.1),
|
|
BinaryCritic(critic_field="country", weight=0.1),
|
|
BinaryCritic(critic_field="distance_unit", weight=0.1),
|
|
BinaryCritic(critic_field="travel_mode", weight=0.1),
|
|
],
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Get directions between two addresses with custom distance unit and travel mode",
|
|
user_message="Get walking directions from Google Maps between the following addresses in miles: 1600 Amphitheatre Parkway, Mountain View, CA 94043 and 1 Infinite Loop, Cupertino, CA 95014.",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=get_directions_between_addresses,
|
|
args={
|
|
"origin_address": "1600 Amphitheatre Parkway, Mountain View, CA 94043",
|
|
"destination_address": "1 Infinite Loop, Cupertino, CA 95014",
|
|
"language": DEFAULT_GOOGLE_MAPS_LANGUAGE,
|
|
"country": DEFAULT_GOOGLE_MAPS_COUNTRY,
|
|
"distance_unit": GoogleMapsDistanceUnit.MILES.value,
|
|
"travel_mode": GoogleMapsTravelMode.WALKING.value,
|
|
},
|
|
)
|
|
],
|
|
critics=[
|
|
SimilarityCritic(critic_field="origin_address", weight=0.3),
|
|
SimilarityCritic(critic_field="destination_address", weight=0.3),
|
|
BinaryCritic(critic_field="language", weight=0.1),
|
|
BinaryCritic(critic_field="country", weight=0.1),
|
|
BinaryCritic(critic_field="distance_unit", weight=0.1),
|
|
BinaryCritic(critic_field="travel_mode", weight=0.1),
|
|
],
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Get directions between two addresses in a given country and language",
|
|
user_message="Get directions from Google Maps in Portuguese between the following addresses in Brazil: Rua do Amendoim, 1, Belo Horizonte, MG and Av. do Descobrimento, 515, Porto Seguro, BA.",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=get_directions_between_addresses,
|
|
args={
|
|
"origin_address": "Rua do Amendoim, 1, Belo Horizonte, MG",
|
|
"destination_address": "Av. do Descobrimento, 515, Porto Seguro, BA",
|
|
"language": "pt",
|
|
"country": "br",
|
|
"distance_unit": DEFAULT_GOOGLE_MAPS_DISTANCE_UNIT,
|
|
"travel_mode": DEFAULT_GOOGLE_MAPS_TRAVEL_MODE,
|
|
},
|
|
)
|
|
],
|
|
critics=[
|
|
SimilarityCritic(critic_field="origin_address", weight=0.3),
|
|
SimilarityCritic(critic_field="destination_address", weight=0.3),
|
|
BinaryCritic(critic_field="language", weight=0.1),
|
|
BinaryCritic(critic_field="country", weight=0.1),
|
|
BinaryCritic(critic_field="distance_unit", weight=0.1),
|
|
BinaryCritic(critic_field="travel_mode", weight=0.1),
|
|
],
|
|
)
|
|
|
|
return suite
|
|
|
|
|
|
@tool_eval()
|
|
def google_maps_directions_by_coordinates_eval_suite() -> EvalSuite:
|
|
"""Create an evaluation suite for the Google Maps Directions tools."""
|
|
suite = EvalSuite(
|
|
name="Google Maps Directions Tool Evaluation",
|
|
system_message="You are an AI assistant that can get directions from Google Maps using the provided tools.",
|
|
catalog=catalog,
|
|
rubric=rubric,
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Get directions between two coordinates",
|
|
user_message="Get directions from Google Maps between the following coordinates: 37.422740,-122.084961 and 37.331820,-122.031180.",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=get_directions_between_coordinates,
|
|
args={
|
|
"origin_latitude": "37.422740",
|
|
"origin_longitude": "-122.084961",
|
|
"destination_latitude": "37.331820",
|
|
"destination_longitude": "-122.031180",
|
|
"language": DEFAULT_GOOGLE_MAPS_LANGUAGE,
|
|
"country": DEFAULT_GOOGLE_MAPS_COUNTRY,
|
|
"distance_unit": DEFAULT_GOOGLE_MAPS_DISTANCE_UNIT,
|
|
"travel_mode": DEFAULT_GOOGLE_MAPS_TRAVEL_MODE,
|
|
},
|
|
)
|
|
],
|
|
critics=[
|
|
SimilarityCritic(critic_field="origin_latitude", weight=0.15),
|
|
SimilarityCritic(critic_field="origin_longitude", weight=0.15),
|
|
SimilarityCritic(critic_field="destination_latitude", weight=0.15),
|
|
SimilarityCritic(critic_field="destination_longitude", weight=0.15),
|
|
BinaryCritic(critic_field="language", weight=0.1),
|
|
BinaryCritic(critic_field="country", weight=0.1),
|
|
BinaryCritic(critic_field="distance_unit", weight=0.1),
|
|
BinaryCritic(critic_field="travel_mode", weight=0.1),
|
|
],
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Get directions between two coordinates with custom distance unit and travel mode",
|
|
user_message="Get walking directions from Google Maps between the following coordinates in miles: 37.422740,-122.084961 and 37.331820,-122.031180.",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=get_directions_between_coordinates,
|
|
args={
|
|
"origin_latitude": "37.422740",
|
|
"origin_longitude": "-122.084961",
|
|
"destination_latitude": "37.331820",
|
|
"destination_longitude": "-122.031180",
|
|
"language": DEFAULT_GOOGLE_MAPS_LANGUAGE,
|
|
"country": DEFAULT_GOOGLE_MAPS_COUNTRY,
|
|
"distance_unit": GoogleMapsDistanceUnit.MILES.value,
|
|
"travel_mode": GoogleMapsTravelMode.WALKING.value,
|
|
},
|
|
)
|
|
],
|
|
critics=[
|
|
SimilarityCritic(critic_field="origin_latitude", weight=0.15),
|
|
SimilarityCritic(critic_field="origin_longitude", weight=0.15),
|
|
SimilarityCritic(critic_field="destination_latitude", weight=0.15),
|
|
SimilarityCritic(critic_field="destination_longitude", weight=0.15),
|
|
BinaryCritic(critic_field="language", weight=0.1),
|
|
BinaryCritic(critic_field="country", weight=0.1),
|
|
BinaryCritic(critic_field="distance_unit", weight=0.1),
|
|
BinaryCritic(critic_field="travel_mode", weight=0.1),
|
|
],
|
|
)
|
|
|
|
suite.add_case(
|
|
name="Get directions between two coordinates in a given country and language",
|
|
user_message="Get directions from Google Maps in Portuguese between the following coordinates in Brazil: 37.422740,-122.084961 and 37.331820,-122.031180.",
|
|
expected_tool_calls=[
|
|
ExpectedToolCall(
|
|
func=get_directions_between_coordinates,
|
|
args={
|
|
"origin_latitude": "37.422740",
|
|
"origin_longitude": "-122.084961",
|
|
"destination_latitude": "37.331820",
|
|
"destination_longitude": "-122.031180",
|
|
"language": "pt",
|
|
"country": "br",
|
|
"distance_unit": DEFAULT_GOOGLE_MAPS_DISTANCE_UNIT,
|
|
"travel_mode": DEFAULT_GOOGLE_MAPS_TRAVEL_MODE,
|
|
},
|
|
)
|
|
],
|
|
critics=[
|
|
SimilarityCritic(critic_field="origin_latitude", weight=0.15),
|
|
SimilarityCritic(critic_field="origin_longitude", weight=0.15),
|
|
SimilarityCritic(critic_field="destination_latitude", weight=0.15),
|
|
SimilarityCritic(critic_field="destination_longitude", weight=0.15),
|
|
BinaryCritic(critic_field="language", weight=0.1),
|
|
BinaryCritic(critic_field="country", weight=0.1),
|
|
BinaryCritic(critic_field="distance_unit", weight=0.1),
|
|
BinaryCritic(critic_field="travel_mode", weight=0.1),
|
|
],
|
|
)
|
|
|
|
return suite
|