diff --git a/toolkits/google/arcade_google/tools/contacts.py b/toolkits/google/arcade_google/tools/contacts.py index d0c36b03..9ff5ab98 100644 --- a/toolkits/google/arcade_google/tools/contacts.py +++ b/toolkits/google/arcade_google/tools/contacts.py @@ -7,6 +7,16 @@ from google.oauth2.credentials import Credentials from googleapiclient.discovery import build +async def _warmup_cache(service) -> None: # type: ignore[no-untyped-def] + """ + Warm-up the search cache for contacts by sending a request with an empty query. + This ensures that the lazy cache is updated for both primary contacts and other contacts. + This is unfortunately a real thing: https://developers.google.com/people/v1/contacts#search_the_users_contacts + """ + service.people().searchContacts(query="", pageSize=1, readMask="names,emailAddresses").execute() + await asyncio.sleep(3) # TODO experiment with this value + + @tool(requires_auth=Google(scopes=["https://www.googleapis.com/auth/contacts.readonly"])) async def search_contacts( context: ToolContext, @@ -20,10 +30,10 @@ async def search_contacts( ] = 10, ) -> Annotated[dict, "A dictionary containing the list of matching contacts"]: """ - Search the user's contacts using the People API. + Search the user's contacts in Google Contacts. - This tool queries the contacts with the provided query string. - The API returns contacts that match based on names, email addresses, and more. + Up to 30 contacts with a name or email address containing the query will be returned. + If the query matches more than 30 contacts, only the first 30 will be returned. """ # Build the People API service service = build( @@ -51,33 +61,32 @@ async def search_contacts( return {"contacts": primary_results} -async def _warmup_cache(service) -> None: # type: ignore[no-untyped-def] - """ - Warm-up the search cache for contacts by sending a request with an empty query. - This ensures that the lazy cache is updated for both primary contacts and other contacts. - This is a real thing: https://developers.google.com/people/v1/contacts#search_the_users_contacts - """ - service.people().searchContacts(query="", pageSize=1, readMask="names,emailAddresses").execute() - await asyncio.sleep(3) # TODO experiment with this value - - @tool(requires_auth=Google(scopes=["https://www.googleapis.com/auth/contacts"])) async def create_contact( context: ToolContext, given_name: Annotated[str, "The given name of the contact"], - family_name: Annotated[Optional[str], "The family name of the contact"], - email: Annotated[Optional[str], "The email address of the contact"], + family_name: Annotated[Optional[str], "The optional family name of the contact"], + email: Annotated[Optional[str], "The optional email address of the contact"], ) -> Annotated[dict, "A dictionary containing the details of the created contact"]: """ - Create a new contact in the user's Google Contacts using the People API. + Create a new contact record in Google Contacts. - This tool creates a contact with the basic name fields. + Examples: + ``` + create_contact(given_name="Alice") + create_contact(given_name="Alice", family_name="Smith") + create_contact(given_name="Alice", email="alice@example.com") + ``` """ # Build the People API service service = build( "people", "v1", - credentials=Credentials(context.get_auth_token_or_empty()), + credentials=Credentials( + context.authorization.token + if context.authorization and context.authorization.token + else "" + ), ) # Construct the person payload with the specified names diff --git a/toolkits/google/evals/eval_google_contacts.py b/toolkits/google/evals/eval_google_contacts.py new file mode 100644 index 00000000..653fbbe4 --- /dev/null +++ b/toolkits/google/evals/eval_google_contacts.py @@ -0,0 +1,116 @@ +from arcade.sdk import ToolCatalog +from arcade.sdk.eval import ( + EvalRubric, + EvalSuite, + ExpectedToolCall, + tool_eval, +) +from arcade.sdk.eval.critic import BinaryCritic + +import arcade_google +from arcade_google.tools.contacts import create_contact, search_contacts + +# Evaluation rubric +rubric = EvalRubric( + fail_threshold=0.9, + warn_threshold=0.95, +) + +catalog = ToolCatalog() +catalog.add_module(arcade_google) + + +@tool_eval() +def contacts_eval_suite() -> EvalSuite: + """Create an evaluation suite for Google Contacts tools.""" + suite = EvalSuite( + name="Google Contacts Tools Evaluation", + system_message="You are an AI assistant that can manage Google Contacts using the provided tools.", + catalog=catalog, + rubric=rubric, + ) + + suite.add_case( + name="Find a contact by name", + user_message="Find my contact Bob", + expected_tool_calls=[ + ExpectedToolCall( + func=search_contacts, + args={"query": "Bob"}, + ) + ], + ) + + suite.add_case( + name="Search contacts with query and limit", + user_message="Find 5 contacts whose names include 'Alice'", + expected_tool_calls=[ + ExpectedToolCall( + func=search_contacts, + args={ + "query": "Alice", + "limit": 5, + }, + ) + ], + critics=[ + BinaryCritic(critic_field="query", weight=0.5), + BinaryCritic(critic_field="limit", weight=0.5), + ], + ) + + suite.add_case( + name="Create new contact with only given name", + user_message="Create a new contact for Alice", + expected_tool_calls=[ + ExpectedToolCall( + func=create_contact, + args={ + "given_name": "Alice", + }, + ) + ], + critics=[ + BinaryCritic(critic_field="given_name", weight=1.0), + ], + ) + + suite.add_case( + name="Create new contact with only email (infer name from email)", + user_message="Create a new contact for alice@example.com", + expected_tool_calls=[ + ExpectedToolCall( + func=create_contact, + args={ + "given_name": "Alice", + "email": "alice@example.com", + }, + ) + ], + critics=[ + BinaryCritic(critic_field="email", weight=0.5), + BinaryCritic(critic_field="given_name", weight=0.5), + ], + ) + + suite.add_case( + name="Create new contact with full name and email", + user_message="Create a contact for Bob Smith (bob.smith@example.com)", + expected_tool_calls=[ + ExpectedToolCall( + func=create_contact, + args={ + "given_name": "Bob", + "family_name": "Smith", + "email": "bob.smith@example.com", + }, + ) + ], + critics=[ + BinaryCritic(critic_field="given_name", weight=0.33), + BinaryCritic(critic_field="family_name", weight=0.33), + BinaryCritic(critic_field="email", weight=0.34), + ], + ) + + return suite diff --git a/toolkits/google/tests/test_contacts.py b/toolkits/google/tests/test_contacts.py new file mode 100644 index 00000000..c6bb57c6 --- /dev/null +++ b/toolkits/google/tests/test_contacts.py @@ -0,0 +1,155 @@ +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from arcade.sdk import ToolContext + +from arcade_google.tools.contacts import create_contact, search_contacts + + +@pytest.fixture +def mock_context(): + context = AsyncMock(spec=ToolContext) + context.authorization = MagicMock() + context.authorization.token = "mock_token" # noqa: S105 + return context + + +@pytest.mark.asyncio +async def test_search_contacts_success(mock_context): + search_response_data = { + "results": [ + { + "resourceName": "people/1", + "names": [{"displayName": "John Doe"}], + "emailAddresses": [{"value": "john@example.com"}], + }, + { + "resourceName": "people/2", + "names": [{"displayName": "Jane Doe"}], + "emailAddresses": [{"value": "jane@example.com"}], + }, + ] + } + search_call = MagicMock() + search_call.execute.return_value = search_response_data + + people_mock = MagicMock() + people_mock.searchContacts.return_value = search_call + + service_mock = MagicMock() + service_mock.people.return_value = people_mock + + with ( + patch("arcade_google.tools.contacts.build", return_value=service_mock) as mock_build, + patch( + "arcade_google.tools.contacts._warmup_cache", new=AsyncMock(return_value=None) + ) as mock_warmup, + ): + result = await search_contacts(mock_context, query="Doe", limit=2) + assert "contacts" in result + assert result["contacts"] == search_response_data["results"] + + assert mock_warmup.call_count == 1 + assert people_mock.searchContacts.call_count == 1 + + # Check that the People API service was built with the expected parameters. + mock_build.assert_called_once() + + +@pytest.mark.asyncio +async def test_search_contacts_error(mock_context): + error_call = MagicMock() + error_call.execute.side_effect = Exception("Search error") + + people_mock = MagicMock() + people_mock.searchContacts.return_value = error_call + + service_mock = MagicMock() + service_mock.people.return_value = people_mock + + with ( + patch("arcade_google.tools.contacts.build", return_value=service_mock), + patch("arcade_google.tools.contacts._warmup_cache", new=AsyncMock(return_value=None)), + pytest.raises(Exception, match="Error in execution of SearchContacts"), + ): + await search_contacts(mock_context, query="Doe") + + +@pytest.mark.asyncio +async def test_create_contact_success(mock_context): + # Test create_contact with all parameters (given, family names and email) + created_contact_data = {"resourceName": "people/123", "etag": "abc"} + + create_contact_call = MagicMock() + create_contact_call.execute.return_value = created_contact_data + + people_mock = MagicMock() + people_mock.createContact.return_value = create_contact_call + + service_mock = MagicMock() + service_mock.people.return_value = people_mock + + with patch("arcade_google.tools.contacts.build", return_value=service_mock) as mock_build: + result = await create_contact( + mock_context, + given_name="Alice", + family_name="Smith", + email="alice@example.com", + ) + assert "contact" in result + assert result["contact"] == created_contact_data + + # Verify that the createContact API was called with the correct body contents. + expected_body = { + "names": [{"givenName": "Alice", "familyName": "Smith"}], + "emailAddresses": [{"value": "alice@example.com", "type": "work"}], + } + people_mock.createContact.assert_called_once_with( + body=expected_body, personFields="names,emailAddresses" + ) + mock_build.assert_called_once() + + +@pytest.mark.asyncio +async def test_create_contact_success_without_optional(mock_context): + # Test create_contact without optional parameters family_name and email. + created_contact_data = {"resourceName": "people/456", "etag": "def"} + + create_contact_call = MagicMock() + create_contact_call.execute.return_value = created_contact_data + + people_mock = MagicMock() + people_mock.createContact.return_value = create_contact_call + + service_mock = MagicMock() + service_mock.people.return_value = people_mock + + with patch("arcade_google.tools.contacts.build", return_value=service_mock): + result = await create_contact(mock_context, given_name="Bob", family_name=None, email=None) + assert "contact" in result + assert result["contact"] == created_contact_data + + # Expected body should only include the givenName when family_name and email are omitted. + expected_body = {"names": [{"givenName": "Bob"}]} + people_mock.createContact.assert_called_once_with( + body=expected_body, personFields="names,emailAddresses" + ) + + +@pytest.mark.asyncio +async def test_create_contact_error(mock_context): + # Simulate an error thrown by createContact + error_call = MagicMock() + error_call.execute.side_effect = Exception("Create error") + + people_mock = MagicMock() + people_mock.createContact.return_value = error_call + + service_mock = MagicMock() + service_mock.people.return_value = people_mock + + with ( + patch("arcade_google.tools.contacts.build", return_value=service_mock), + pytest.raises(Exception, match="Error in execution of CreateContact"), + ): + await create_contact(mock_context, given_name="Alice", family_name="Doe", email=None)