Polish up Google.SearchContacts and CreateContact (#259)

Adding the missing polish (evals, tests) for #249

toolkits/google/arcade_google/tools/contacts.py

@@ -7,6 +7,16 @@ from google.oauth2.credentials import Credentials
 from googleapiclient.discovery import build
 
 
+async def _warmup_cache(service) -> None:  # type: ignore[no-untyped-def]
+    """
+    Warm-up the search cache for contacts by sending a request with an empty query.
+    This ensures that the lazy cache is updated for both primary contacts and other contacts.
+    This is unfortunately a real thing: https://developers.google.com/people/v1/contacts#search_the_users_contacts
+    """
+    service.people().searchContacts(query="", pageSize=1, readMask="names,emailAddresses").execute()
+    await asyncio.sleep(3)  # TODO experiment with this value
+
+
 @tool(requires_auth=Google(scopes=["https://www.googleapis.com/auth/contacts.readonly"]))
 async def search_contacts(
     context: ToolContext,
@@ -20,10 +30,10 @@ async def search_contacts(
     ] = 10,
 ) -> Annotated[dict, "A dictionary containing the list of matching contacts"]:
     """
-    Search the user's contacts using the People API.
+    Search the user's contacts in Google Contacts.
 
-    This tool queries the contacts with the provided query string.
+    Up to 30 contacts with a name or email address containing the query will be returned.
-    The API returns contacts that match based on names, email addresses, and more.
+    If the query matches more than 30 contacts, only the first 30 will be returned.
     """
     # Build the People API service
     service = build(
@@ -51,33 +61,32 @@ async def search_contacts(
     return {"contacts": primary_results}
 
 
-async def _warmup_cache(service) -> None:  # type: ignore[no-untyped-def]
-    """
-    Warm-up the search cache for contacts by sending a request with an empty query.
-    This ensures that the lazy cache is updated for both primary contacts and other contacts.
-    This is a real thing: https://developers.google.com/people/v1/contacts#search_the_users_contacts
-    """
-    service.people().searchContacts(query="", pageSize=1, readMask="names,emailAddresses").execute()
-    await asyncio.sleep(3)  # TODO experiment with this value
-
-
 @tool(requires_auth=Google(scopes=["https://www.googleapis.com/auth/contacts"]))
 async def create_contact(
     context: ToolContext,
     given_name: Annotated[str, "The given name of the contact"],
-    family_name: Annotated[Optional[str], "The family name of the contact"],
+    family_name: Annotated[Optional[str], "The optional family name of the contact"],
-    email: Annotated[Optional[str], "The email address of the contact"],
+    email: Annotated[Optional[str], "The optional email address of the contact"],
 ) -> Annotated[dict, "A dictionary containing the details of the created contact"]:
     """
-    Create a new contact in the user's Google Contacts using the People API.
+    Create a new contact record in Google Contacts.
 
-    This tool creates a contact with the basic name fields.
+    Examples:
+    ```
+    create_contact(given_name="Alice")
+    create_contact(given_name="Alice", family_name="Smith")
+    create_contact(given_name="Alice", email="alice@example.com")
+    ```
     """
     # Build the People API service
     service = build(
         "people",
         "v1",
-        credentials=Credentials(context.get_auth_token_or_empty()),
+        credentials=Credentials(
+            context.authorization.token
+            if context.authorization and context.authorization.token
+            else ""
+        ),
     )
 
     # Construct the person payload with the specified names

toolkits/google/evals/eval_google_contacts.py

@@ -0,0 +1,116 @@
+from arcade.sdk import ToolCatalog
+from arcade.sdk.eval import (
+    EvalRubric,
+    EvalSuite,
+    ExpectedToolCall,
+    tool_eval,
+)
+from arcade.sdk.eval.critic import BinaryCritic
+
+import arcade_google
+from arcade_google.tools.contacts import create_contact, search_contacts
+
+# Evaluation rubric
+rubric = EvalRubric(
+    fail_threshold=0.9,
+    warn_threshold=0.95,
+)
+
+catalog = ToolCatalog()
+catalog.add_module(arcade_google)
+
+
+@tool_eval()
+def contacts_eval_suite() -> EvalSuite:
+    """Create an evaluation suite for Google Contacts tools."""
+    suite = EvalSuite(
+        name="Google Contacts Tools Evaluation",
+        system_message="You are an AI assistant that can manage Google Contacts using the provided tools.",
+        catalog=catalog,
+        rubric=rubric,
+    )
+
+    suite.add_case(
+        name="Find a contact by name",
+        user_message="Find my contact Bob",
+        expected_tool_calls=[
+            ExpectedToolCall(
+                func=search_contacts,
+                args={"query": "Bob"},
+            )
+        ],
+    )
+
+    suite.add_case(
+        name="Search contacts with query and limit",
+        user_message="Find 5 contacts whose names include 'Alice'",
+        expected_tool_calls=[
+            ExpectedToolCall(
+                func=search_contacts,
+                args={
+                    "query": "Alice",
+                    "limit": 5,
+                },
+            )
+        ],
+        critics=[
+            BinaryCritic(critic_field="query", weight=0.5),
+            BinaryCritic(critic_field="limit", weight=0.5),
+        ],
+    )
+
+    suite.add_case(
+        name="Create new contact with only given name",
+        user_message="Create a new contact for Alice",
+        expected_tool_calls=[
+            ExpectedToolCall(
+                func=create_contact,
+                args={
+                    "given_name": "Alice",
+                },
+            )
+        ],
+        critics=[
+            BinaryCritic(critic_field="given_name", weight=1.0),
+        ],
+    )
+
+    suite.add_case(
+        name="Create new contact with only email (infer name from email)",
+        user_message="Create a new contact for alice@example.com",
+        expected_tool_calls=[
+            ExpectedToolCall(
+                func=create_contact,
+                args={
+                    "given_name": "Alice",
+                    "email": "alice@example.com",
+                },
+            )
+        ],
+        critics=[
+            BinaryCritic(critic_field="email", weight=0.5),
+            BinaryCritic(critic_field="given_name", weight=0.5),
+        ],
+    )
+
+    suite.add_case(
+        name="Create new contact with full name and email",
+        user_message="Create a contact for Bob Smith (bob.smith@example.com)",
+        expected_tool_calls=[
+            ExpectedToolCall(
+                func=create_contact,
+                args={
+                    "given_name": "Bob",
+                    "family_name": "Smith",
+                    "email": "bob.smith@example.com",
+                },
+            )
+        ],
+        critics=[
+            BinaryCritic(critic_field="given_name", weight=0.33),
+            BinaryCritic(critic_field="family_name", weight=0.33),
+            BinaryCritic(critic_field="email", weight=0.34),
+        ],
+    )
+
+    return suite

toolkits/google/tests/test_contacts.py

@@ -0,0 +1,155 @@
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+from arcade.sdk import ToolContext
+
+from arcade_google.tools.contacts import create_contact, search_contacts
+
+
+@pytest.fixture
+def mock_context():
+    context = AsyncMock(spec=ToolContext)
+    context.authorization = MagicMock()
+    context.authorization.token = "mock_token"  # noqa: S105
+    return context
+
+
+@pytest.mark.asyncio
+async def test_search_contacts_success(mock_context):
+    search_response_data = {
+        "results": [
+            {
+                "resourceName": "people/1",
+                "names": [{"displayName": "John Doe"}],
+                "emailAddresses": [{"value": "john@example.com"}],
+            },
+            {
+                "resourceName": "people/2",
+                "names": [{"displayName": "Jane Doe"}],
+                "emailAddresses": [{"value": "jane@example.com"}],
+            },
+        ]
+    }
+    search_call = MagicMock()
+    search_call.execute.return_value = search_response_data
+
+    people_mock = MagicMock()
+    people_mock.searchContacts.return_value = search_call
+
+    service_mock = MagicMock()
+    service_mock.people.return_value = people_mock
+
+    with (
+        patch("arcade_google.tools.contacts.build", return_value=service_mock) as mock_build,
+        patch(
+            "arcade_google.tools.contacts._warmup_cache", new=AsyncMock(return_value=None)
+        ) as mock_warmup,
+    ):
+        result = await search_contacts(mock_context, query="Doe", limit=2)
+        assert "contacts" in result
+        assert result["contacts"] == search_response_data["results"]
+
+        assert mock_warmup.call_count == 1
+        assert people_mock.searchContacts.call_count == 1
+
+        # Check that the People API service was built with the expected parameters.
+        mock_build.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_search_contacts_error(mock_context):
+    error_call = MagicMock()
+    error_call.execute.side_effect = Exception("Search error")
+
+    people_mock = MagicMock()
+    people_mock.searchContacts.return_value = error_call
+
+    service_mock = MagicMock()
+    service_mock.people.return_value = people_mock
+
+    with (
+        patch("arcade_google.tools.contacts.build", return_value=service_mock),
+        patch("arcade_google.tools.contacts._warmup_cache", new=AsyncMock(return_value=None)),
+        pytest.raises(Exception, match="Error in execution of SearchContacts"),
+    ):
+        await search_contacts(mock_context, query="Doe")
+
+
+@pytest.mark.asyncio
+async def test_create_contact_success(mock_context):
+    # Test create_contact with all parameters (given, family names and email)
+    created_contact_data = {"resourceName": "people/123", "etag": "abc"}
+
+    create_contact_call = MagicMock()
+    create_contact_call.execute.return_value = created_contact_data
+
+    people_mock = MagicMock()
+    people_mock.createContact.return_value = create_contact_call
+
+    service_mock = MagicMock()
+    service_mock.people.return_value = people_mock
+
+    with patch("arcade_google.tools.contacts.build", return_value=service_mock) as mock_build:
+        result = await create_contact(
+            mock_context,
+            given_name="Alice",
+            family_name="Smith",
+            email="alice@example.com",
+        )
+        assert "contact" in result
+        assert result["contact"] == created_contact_data
+
+        # Verify that the createContact API was called with the correct body contents.
+        expected_body = {
+            "names": [{"givenName": "Alice", "familyName": "Smith"}],
+            "emailAddresses": [{"value": "alice@example.com", "type": "work"}],
+        }
+        people_mock.createContact.assert_called_once_with(
+            body=expected_body, personFields="names,emailAddresses"
+        )
+        mock_build.assert_called_once()
+
+
+@pytest.mark.asyncio
+async def test_create_contact_success_without_optional(mock_context):
+    # Test create_contact without optional parameters family_name and email.
+    created_contact_data = {"resourceName": "people/456", "etag": "def"}
+
+    create_contact_call = MagicMock()
+    create_contact_call.execute.return_value = created_contact_data
+
+    people_mock = MagicMock()
+    people_mock.createContact.return_value = create_contact_call
+
+    service_mock = MagicMock()
+    service_mock.people.return_value = people_mock
+
+    with patch("arcade_google.tools.contacts.build", return_value=service_mock):
+        result = await create_contact(mock_context, given_name="Bob", family_name=None, email=None)
+        assert "contact" in result
+        assert result["contact"] == created_contact_data
+
+        # Expected body should only include the givenName when family_name and email are omitted.
+        expected_body = {"names": [{"givenName": "Bob"}]}
+        people_mock.createContact.assert_called_once_with(
+            body=expected_body, personFields="names,emailAddresses"
+        )
+
+
+@pytest.mark.asyncio
+async def test_create_contact_error(mock_context):
+    # Simulate an error thrown by createContact
+    error_call = MagicMock()
+    error_call.execute.side_effect = Exception("Create error")
+
+    people_mock = MagicMock()
+    people_mock.createContact.return_value = error_call
+
+    service_mock = MagicMock()
+    service_mock.people.return_value = people_mock
+
+    with (
+        patch("arcade_google.tools.contacts.build", return_value=service_mock),
+        pytest.raises(Exception, match="Error in execution of CreateContact"),
+    ):
+        await create_contact(mock_context, given_name="Alice", family_name="Doe", email=None)