admin-valentin/arcade-mcp
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Polish up Google.SearchContacts and CreateContact (#259)

Browse source 
Adding the missing polish (evals, tests) for #249
This commit is contained in:
Nate Barbettini 2025-02-20 08:53:12 -08:00 committed by GitHub
parent 1e0def78df
commit 274e63c9e5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 298 additions and 18 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
toolkits/google/arcade_google/tools/contacts.py
View file

@ -7,6 +7,16 @@ from google.oauth2.credentials import Credentials
from googleapiclient.discovery import build
async def _warmup_cache(service) -> None: # type: ignore[no-untyped-def]
"""
Warm-up the search cache for contacts by sending a request with an empty query.
This ensures that the lazy cache is updated for both primary contacts and other contacts.
This is unfortunately a real thing: https://developers.google.com/people/v1/contacts#search_the_users_contacts
"""
service.people().searchContacts(query="", pageSize=1, readMask="names,emailAddresses").execute()
await asyncio.sleep(3) # TODO experiment with this value
@tool(requires_auth=Google(scopes=["https://www.googleapis.com/auth/contacts.readonly"]))
async def search_contacts(
context: ToolContext,
@ -20,10 +30,10 @@ async def search_contacts(
] = 10,
) -> Annotated[dict, "A dictionary containing the list of matching contacts"]:
"""
Search the user's contacts using the People API.
Search the user's contacts in Google Contacts.
This tool queries the contacts with the provided query string.
The API returns contacts that match based on names, email addresses, and more.
Up to 30 contacts with a name or email address containing the query will be returned.
If the query matches more than 30 contacts, only the first 30 will be returned.
"""
# Build the People API service
service = build(
@ -51,33 +61,32 @@ async def search_contacts(
return {"contacts": primary_results}
async def _warmup_cache(service) -> None: # type: ignore[no-untyped-def]
"""
Warm-up the search cache for contacts by sending a request with an empty query.
This ensures that the lazy cache is updated for both primary contacts and other contacts.
This is a real thing: https://developers.google.com/people/v1/contacts#search_the_users_contacts
"""
service.people().searchContacts(query="", pageSize=1, readMask="names,emailAddresses").execute()
await asyncio.sleep(3) # TODO experiment with this value
@tool(requires_auth=Google(scopes=["https://www.googleapis.com/auth/contacts"]))
async def create_contact(
context: ToolContext,
given_name: Annotated[str, "The given name of the contact"],
family_name: Annotated[Optional[str], "The family name of the contact"],
email: Annotated[Optional[str], "The email address of the contact"],
family_name: Annotated[Optional[str], "The optional family name of the contact"],
email: Annotated[Optional[str], "The optional email address of the contact"],
) -> Annotated[dict, "A dictionary containing the details of the created contact"]:
"""
Create a new contact in the user's Google Contacts using the People API.
Create a new contact record in Google Contacts.
This tool creates a contact with the basic name fields.
Examples:
```
create_contact(given_name="Alice")
create_contact(given_name="Alice", family_name="Smith")
create_contact(given_name="Alice", email="alice@example.com")
```
"""
# Build the People API service
service = build(
"people",
"v1",
credentials=Credentials(context.get_auth_token_or_empty()),
credentials=Credentials(
context.authorization.token
if context.authorization and context.authorization.token
else ""
),
)
# Construct the person payload with the specified names

116
toolkits/google/evals/eval_google_contacts.py Normal file
View file

@ -0,0 +1,116 @@
from arcade.sdk import ToolCatalog
from arcade.sdk.eval import (
EvalRubric,
EvalSuite,
ExpectedToolCall,
tool_eval,
)
from arcade.sdk.eval.critic import BinaryCritic
import arcade_google
from arcade_google.tools.contacts import create_contact, search_contacts
# Evaluation rubric
rubric = EvalRubric(
fail_threshold=0.9,
warn_threshold=0.95,
)
catalog = ToolCatalog()
catalog.add_module(arcade_google)
@tool_eval()
def contacts_eval_suite() -> EvalSuite:
"""Create an evaluation suite for Google Contacts tools."""
suite = EvalSuite(
name="Google Contacts Tools Evaluation",
system_message="You are an AI assistant that can manage Google Contacts using the provided tools.",
catalog=catalog,
rubric=rubric,
)
suite.add_case(
name="Find a contact by name",
user_message="Find my contact Bob",
expected_tool_calls=[
ExpectedToolCall(
func=search_contacts,
args={"query": "Bob"},
)
],
)
suite.add_case(
name="Search contacts with query and limit",
user_message="Find 5 contacts whose names include 'Alice'",
expected_tool_calls=[
ExpectedToolCall(
func=search_contacts,
args={
"query": "Alice",
"limit": 5,
},
)
],
critics=[
BinaryCritic(critic_field="query", weight=0.5),
BinaryCritic(critic_field="limit", weight=0.5),
],
)
suite.add_case(
name="Create new contact with only given name",
user_message="Create a new contact for Alice",
expected_tool_calls=[
ExpectedToolCall(
func=create_contact,
args={
"given_name": "Alice",
},
)
],
critics=[
BinaryCritic(critic_field="given_name", weight=1.0),
],
)
suite.add_case(
name="Create new contact with only email (infer name from email)",
user_message="Create a new contact for alice@example.com",
expected_tool_calls=[
ExpectedToolCall(
func=create_contact,
args={
"given_name": "Alice",
"email": "alice@example.com",
},
)
],
critics=[
BinaryCritic(critic_field="email", weight=0.5),
BinaryCritic(critic_field="given_name", weight=0.5),
],
)
suite.add_case(
name="Create new contact with full name and email",
user_message="Create a contact for Bob Smith (bob.smith@example.com)",
expected_tool_calls=[
ExpectedToolCall(
func=create_contact,
args={
"given_name": "Bob",
"family_name": "Smith",
"email": "bob.smith@example.com",
},
)
],
critics=[
BinaryCritic(critic_field="given_name", weight=0.33),
BinaryCritic(critic_field="family_name", weight=0.33),
BinaryCritic(critic_field="email", weight=0.34),
],
)
return suite

155
toolkits/google/tests/test_contacts.py Normal file
View file

@ -0,0 +1,155 @@
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from arcade.sdk import ToolContext
from arcade_google.tools.contacts import create_contact, search_contacts
@pytest.fixture
def mock_context():
context = AsyncMock(spec=ToolContext)
context.authorization = MagicMock()
context.authorization.token = "mock_token" # noqa: S105
return context
@pytest.mark.asyncio
async def test_search_contacts_success(mock_context):
search_response_data = {
"results": [
{
"resourceName": "people/1",
"names": [{"displayName": "John Doe"}],
"emailAddresses": [{"value": "john@example.com"}],
},
{
"resourceName": "people/2",
"names": [{"displayName": "Jane Doe"}],
"emailAddresses": [{"value": "jane@example.com"}],
},
]
}
search_call = MagicMock()
search_call.execute.return_value = search_response_data
people_mock = MagicMock()
people_mock.searchContacts.return_value = search_call
service_mock = MagicMock()
service_mock.people.return_value = people_mock
with (
patch("arcade_google.tools.contacts.build", return_value=service_mock) as mock_build,
patch(
"arcade_google.tools.contacts._warmup_cache", new=AsyncMock(return_value=None)
) as mock_warmup,
):
result = await search_contacts(mock_context, query="Doe", limit=2)
assert "contacts" in result
assert result["contacts"] == search_response_data["results"]
assert mock_warmup.call_count == 1
assert people_mock.searchContacts.call_count == 1
# Check that the People API service was built with the expected parameters.
mock_build.assert_called_once()
@pytest.mark.asyncio
async def test_search_contacts_error(mock_context):
error_call = MagicMock()
error_call.execute.side_effect = Exception("Search error")
people_mock = MagicMock()
people_mock.searchContacts.return_value = error_call
service_mock = MagicMock()
service_mock.people.return_value = people_mock
with (
patch("arcade_google.tools.contacts.build", return_value=service_mock),
patch("arcade_google.tools.contacts._warmup_cache", new=AsyncMock(return_value=None)),
pytest.raises(Exception, match="Error in execution of SearchContacts"),
):
await search_contacts(mock_context, query="Doe")
@pytest.mark.asyncio
async def test_create_contact_success(mock_context):
# Test create_contact with all parameters (given, family names and email)
created_contact_data = {"resourceName": "people/123", "etag": "abc"}
create_contact_call = MagicMock()
create_contact_call.execute.return_value = created_contact_data
people_mock = MagicMock()
people_mock.createContact.return_value = create_contact_call
service_mock = MagicMock()
service_mock.people.return_value = people_mock
with patch("arcade_google.tools.contacts.build", return_value=service_mock) as mock_build:
result = await create_contact(
mock_context,
given_name="Alice",
family_name="Smith",
email="alice@example.com",
)
assert "contact" in result
assert result["contact"] == created_contact_data
# Verify that the createContact API was called with the correct body contents.
expected_body = {
"names": [{"givenName": "Alice", "familyName": "Smith"}],
"emailAddresses": [{"value": "alice@example.com", "type": "work"}],
}
people_mock.createContact.assert_called_once_with(
body=expected_body, personFields="names,emailAddresses"
)
mock_build.assert_called_once()
@pytest.mark.asyncio
async def test_create_contact_success_without_optional(mock_context):
# Test create_contact without optional parameters family_name and email.
created_contact_data = {"resourceName": "people/456", "etag": "def"}
create_contact_call = MagicMock()
create_contact_call.execute.return_value = created_contact_data
people_mock = MagicMock()
people_mock.createContact.return_value = create_contact_call
service_mock = MagicMock()
service_mock.people.return_value = people_mock
with patch("arcade_google.tools.contacts.build", return_value=service_mock):
result = await create_contact(mock_context, given_name="Bob", family_name=None, email=None)
assert "contact" in result
assert result["contact"] == created_contact_data
# Expected body should only include the givenName when family_name and email are omitted.
expected_body = {"names": [{"givenName": "Bob"}]}
people_mock.createContact.assert_called_once_with(
body=expected_body, personFields="names,emailAddresses"
)
@pytest.mark.asyncio
async def test_create_contact_error(mock_context):
# Simulate an error thrown by createContact
error_call = MagicMock()
error_call.execute.side_effect = Exception("Create error")
people_mock = MagicMock()
people_mock.createContact.return_value = error_call
service_mock = MagicMock()
service_mock.people.return_value = people_mock
with (
patch("arcade_google.tools.contacts.build", return_value=service_mock),
pytest.raises(Exception, match="Error in execution of CreateContact"),
):
await create_contact(mock_context, given_name="Alice", family_name="Doe", email=None)