From e764205e6c4b3e38a2863216a8aa1e1cb53d1afa Mon Sep 17 00:00:00 2001 From: Eric Gustin <34000337+EricGustin@users.noreply.github.com> Date: Fri, 16 May 2025 13:08:29 -0800 Subject: [PATCH] Add `Microsoft.ListEmailsByProperty` tool (#386) --- .../arcade_microsoft/outlook_mail/__init__.py | 2 + .../arcade_microsoft/outlook_mail/_utils.py | 45 +++++++- .../arcade_microsoft/outlook_mail/enums.py | 42 +++++++ .../outlook_mail/tools/__init__.py | 2 + .../outlook_mail/tools/read.py | 35 +++++- .../microsoft/evals/outlook_mail/eval_read.py | 105 +++++++++++++++++- .../tests/outlook_mail/test_utils.py | 55 +++++++++ 7 files changed, 280 insertions(+), 6 deletions(-) create mode 100644 toolkits/microsoft/tests/outlook_mail/test_utils.py diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/__init__.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/__init__.py index 3e04aa37..f0d7ab26 100644 --- a/toolkits/microsoft/arcade_microsoft/outlook_mail/__init__.py +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/__init__.py @@ -2,6 +2,7 @@ from arcade_microsoft.outlook_mail.tools import ( create_and_send_email, create_draft_email, list_emails, + list_emails_by_property, list_emails_in_folder, reply_to_email, send_draft_email, @@ -11,6 +12,7 @@ from arcade_microsoft.outlook_mail.tools import ( __all__ = [ # Read "list_emails", + "list_emails_by_property", "list_emails_in_folder", # Send "create_and_send_email", diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/_utils.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/_utils.py index 9d281ab2..ec9f2b8c 100644 --- a/toolkits/microsoft/arcade_microsoft/outlook_mail/_utils.py +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/_utils.py @@ -15,7 +15,7 @@ from msgraph.generated.users.item.messages.messages_request_builder import ( from arcade_microsoft.client import get_client from arcade_microsoft.outlook_mail.constants import DEFAULT_MESSAGE_FIELDS -from arcade_microsoft.outlook_mail.enums import ReplyType +from arcade_microsoft.outlook_mail.enums import EmailFilterProperty, FilterOperator, ReplyType def remove_none_values(data: dict) -> dict: @@ -23,15 +23,54 @@ def remove_none_values(data: dict) -> dict: return {k: v for k, v in data.items() if v is not None} +def _create_filter_expression( + property_: EmailFilterProperty | None = None, + operator: FilterOperator | None = None, + value: str | None = None, +) -> str | None: + if property_ and operator and value: + property_value = property_.value + operator_value = operator.value + + # Never use quotes around 'value' for booleans and numerics + value_quote = "'" + if value.lower() in ["true", "false"] or value.isdigit(): + value_quote = "" + + # Handle function operators (e.g., contains, startsWith, endsWith) + if operator.is_function(): + filter_expr = f"{operator_value}({property_value}, {value_quote}{value}{value_quote})" + else: + # Handle comparison operators (e.g., eq, ne, gt, ge, lt, le) + filter_expr = f"{property_value} {operator_value} {value_quote}{value}{value_quote}" + + if property_value == EmailFilterProperty.RECEIVED_DATE_TIME: + filter_expr = filter_expr + else: # Since receivedDateTime is in orderby, it must be in filter: https://learn.microsoft.com/en-us/graph/api/user-list-messages?view=graph-rest-1.0&tabs=http#optional-query-parameters + filter_expr = f"receivedDateTime ge 1900-01-01T00:00:00Z and {filter_expr}" + + return filter_expr + + return None + + def prepare_list_emails_request_config( limit: int, + property_: EmailFilterProperty | None = None, + operator: FilterOperator | None = None, + value: str | None = None, ) -> MailFolderMessagesRequestBuilder.MessagesRequestBuilderGetRequestConfiguration: - limit = max(1, min(limit, 100)) # limit must be between 1 and 100 """Prepare a request configuration for listing emails.""" + limit = max(1, min(limit, 100)) # limit must be between 1 and 100 + + orderby = ["receivedDateTime DESC"] + filter_expr = _create_filter_expression(property_, operator, value) + query_params = MailFolderMessagesRequestBuilder.MessagesRequestBuilderGetQueryParameters( count=True, select=DEFAULT_MESSAGE_FIELDS, - orderby=["receivedDateTime DESC"], + orderby=orderby, + filter=filter_expr, top=limit, ) return MailFolderMessagesRequestBuilder.MessagesRequestBuilderGetRequestConfiguration( diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/enums.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/enums.py index 05f55546..ce68162f 100644 --- a/toolkits/microsoft/arcade_microsoft/outlook_mail/enums.py +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/enums.py @@ -21,3 +21,45 @@ class ReplyType(str, Enum): REPLY = "reply" REPLY_ALL = "reply_all" + + +class EmailFilterProperty(str, Enum): + """The property to filter the emails by.""" + + # Basic properties + SUBJECT = "subject" + CONVERSATION_ID = "conversationId" + RECEIVED_DATE_TIME = "receivedDateTime" + SENDER = "sender/emailAddress/address" + + +class FilterOperator(str, Enum): + """The operator to use for the filter. + + For a full list of possible operators, see: https://learn.microsoft.com/en-us/graph/filter-query-parameter?tabs=http#operators-and-functions-supported-in-filter-expressions + """ + + # Equality operators + EQUAL = "eq" # example: $filter=conversationId eq 'hello' + NOT_EQUAL = "ne" # example: $filter=subject ne 'hello' + + # Relational operators + GREATER_THAN = "gt" # example: $filter=receivedDateTime gt 2024-01-01 + GREATER_THAN_OR_EQUAL_TO = "ge" # example: $filter=receivedDateTime ge 2024-01-01 + LESS_THAN = "lt" # example: $filter=receivedDateTime lt 2024-01-01 + LESS_THAN_OR_EQUAL_TO = "le" # example: $filter=receivedDateTime le 2024-01-01 + + # Functions + STARTS_WITH = "startsWith" # example: $filter=startsWith(subject, 'hello') + ENDS_WITH = "endsWith" # example: $filter=endsWith(subject, 'hello') + CONTAINS = "contains" # example: $filter=contains(subject, 'hello') + + def is_operator(self) -> bool: + """Check if the operator is a comparison operator.""" + operators = [self.EQUAL, self.NOT_EQUAL] + return self in operators + + def is_function(self) -> bool: + """Check if the operator is a function.""" + functions = [self.STARTS_WITH, self.ENDS_WITH, self.CONTAINS] + return self in functions diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/__init__.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/__init__.py index b08b9dec..5e4278a1 100644 --- a/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/__init__.py +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/__init__.py @@ -1,5 +1,6 @@ from arcade_microsoft.outlook_mail.tools.read import ( list_emails, + list_emails_by_property, list_emails_in_folder, ) from arcade_microsoft.outlook_mail.tools.send import ( @@ -15,6 +16,7 @@ from arcade_microsoft.outlook_mail.tools.write import ( __all__ = [ # Read "list_emails", + "list_emails_by_property", "list_emails_in_folder", # Send "create_and_send_email", diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/read.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/read.py index be6ff1b4..2210af7a 100644 --- a/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/read.py +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/read.py @@ -10,7 +10,11 @@ from arcade_microsoft.outlook_mail._utils import ( prepare_list_emails_request_config, remove_none_values, ) -from arcade_microsoft.outlook_mail.enums import WellKnownFolderNames +from arcade_microsoft.outlook_mail.enums import ( + EmailFilterProperty, + FilterOperator, + WellKnownFolderNames, +) from arcade_microsoft.outlook_mail.message import Message @@ -87,3 +91,32 @@ async def list_emails_in_folder( } result = remove_none_values(result) return result + + +@tool(requires_auth=Microsoft(scopes=["Mail.Read"])) +async def list_emails_by_property( + context: ToolContext, + property: Annotated[EmailFilterProperty, "The property to filter the emails by."], # noqa: A002 + operator: Annotated[FilterOperator, "The operator to use for the filter."], + value: Annotated[str, "The value to filter the emails by"], + limit: Annotated[int, "The number of messages to return. Max is 100. Defaults to 5."] = 5, + pagination_token: Annotated[ + str | None, "The pagination token to continue a previous request" + ] = None, +) -> Annotated[dict, "A dictionary containing a list of emails"]: + """List emails in the user's mailbox across all folders filtering by a property.""" + client = get_client(context.get_auth_token_or_empty()) + request_config = prepare_list_emails_request_config(limit, property, operator, value) + message_builder = client.me.messages + + response = await fetch_emails(message_builder, pagination_token, request_config) + messages = [Message.from_sdk(msg).to_dict() for msg in response.value or []] + pagination_token = response.odata_next_link + + result = { + "messages": messages, + "num_messages": len(messages), + "pagination_token": pagination_token, + } + result = remove_none_values(result) + return result diff --git a/toolkits/microsoft/evals/outlook_mail/eval_read.py b/toolkits/microsoft/evals/outlook_mail/eval_read.py index 25740d3e..4c5c284c 100644 --- a/toolkits/microsoft/evals/outlook_mail/eval_read.py +++ b/toolkits/microsoft/evals/outlook_mail/eval_read.py @@ -1,13 +1,20 @@ +from datetime import timedelta + from arcade.sdk import ToolCatalog from arcade.sdk.eval import ( BinaryCritic, + DatetimeCritic, EvalRubric, EvalSuite, ExpectedToolCall, tool_eval, ) -from arcade_microsoft.outlook_mail import list_emails, list_emails_in_folder +from arcade_microsoft.outlook_mail import ( + list_emails, + list_emails_by_property, + list_emails_in_folder, +) from arcade_microsoft.outlook_mail.enums import WellKnownFolderNames from evals.outlook_mail.additional_messages import ( list_emails_with_pagination_token_additional_messages, @@ -23,10 +30,11 @@ rubric = EvalRubric( catalog = ToolCatalog() catalog.add_tool(list_emails, "Microsoft") catalog.add_tool(list_emails_in_folder, "Microsoft") +catalog.add_tool(list_emails_by_property, "Microsoft") @tool_eval() -def outlook_mail_eval_suite() -> EvalSuite: +def outlook_mail_read_eval_suite() -> EvalSuite: """Create an evaluation suite for Outlook Mail tools.""" suite = EvalSuite( name="Outlook Mail Tools Evaluation", @@ -107,3 +115,96 @@ def outlook_mail_eval_suite() -> EvalSuite: ) return suite + + +@tool_eval() +def outlook_mail_list_emails_by_property_eval_suite() -> EvalSuite: + """Create an evaluation suite for Outlook Mail tools.""" + suite = EvalSuite( + name="Outlook Mail Tools Evaluation", + system_message=("You are an AI that has access to tools to send, read, and write emails."), + catalog=catalog, + rubric=rubric, + ) + + suite.add_case( + name="List emails by subject", + user_message="get all emails that talk about The Green Bottle", + expected_tool_calls=[ + ExpectedToolCall( + func=list_emails_by_property, + args={ + "property": "subject", + "operator": "contains", + "value": "The Green Bottle", + }, + ), + ], + critics=[ + BinaryCritic(critic_field="property", weight=1 / 3), + BinaryCritic(critic_field="operator", weight=1 / 3), + BinaryCritic(critic_field="value", weight=1 / 3), + ], + ) + + suite.extend_case( + name="List emails by thread", + user_message="get all emails in my thread 1k2jh324h92f24krjb34mtb43kj4bk3tmn34b3k4nnm3tb34mntb34mntb3m4bt3mn4bt3mn4btmnb34tmnb3t4mnb==34tkjh", # noqa: E501 + expected_tool_calls=[ + ExpectedToolCall( + func=list_emails_by_property, + args={ + "property": "conversationId", + "operator": "eq", + "value": "1k2jh324h92f24krjb34mtb43kj4bk3tmn34b3k4nnm3tb34mntb34mntb3m4bt3mn4bt3mn4btmnb34tmnb3t4mnb==34tkjh", # noqa: E501 + }, + ), + ], + critics=[ + BinaryCritic(critic_field="property", weight=1 / 3), + BinaryCritic(critic_field="operator", weight=1 / 3), + BinaryCritic(critic_field="value", weight=1 / 3), + ], + ) + + suite.extend_case( + name="List emails by date", + user_message="Today is May 1st 2025. Get all emails that are a year old or older", + expected_tool_calls=[ + ExpectedToolCall( + func=list_emails_by_property, + args={ + "property": "receivedDateTime", + "operator": "le", + "value": "2024-05-01T00:00:00Z", + }, + ), + ], + critics=[ + BinaryCritic(critic_field="property", weight=1 / 3), + BinaryCritic(critic_field="operator", weight=1 / 3), + DatetimeCritic(critic_field="value", weight=1 / 3, tolerance=timedelta(days=1)), + ], + ) + + suite.extend_case( + name="List emails by sender", + user_message="get all of my correspondence with the folks over at arcade.dev", + expected_tool_calls=[ + ExpectedToolCall( + func=list_emails_by_property, + args={ + "property": "sender/emailAddress/address", + "operator": "contains", + "value": "arcade.dev", + }, + ), + ], + critics=[ + BinaryCritic(critic_field="property", weight=1 / 3), + BinaryCritic(critic_field="operator", weight=1 / 3), + BinaryCritic(critic_field="value", weight=1 / 3), + ], + ) + + return suite diff --git a/toolkits/microsoft/tests/outlook_mail/test_utils.py b/toolkits/microsoft/tests/outlook_mail/test_utils.py new file mode 100644 index 00000000..1d5bb0b2 --- /dev/null +++ b/toolkits/microsoft/tests/outlook_mail/test_utils.py @@ -0,0 +1,55 @@ +import pytest + +from arcade_microsoft.outlook_mail._utils import _create_filter_expression +from arcade_microsoft.outlook_mail.enums import EmailFilterProperty, FilterOperator + + +@pytest.mark.parametrize( + "property_, operator, value, expected_filter_expr", + [ + ( + EmailFilterProperty.SUBJECT, + FilterOperator.EQUAL, + "Hello", + "receivedDateTime ge 1900-01-01T00:00:00Z and subject eq 'Hello'", + ), + ( + EmailFilterProperty.SUBJECT, + FilterOperator.STARTS_WITH, + "He", + "receivedDateTime ge 1900-01-01T00:00:00Z and startsWith(subject, 'He')", + ), + ( + EmailFilterProperty.CONVERSATION_ID, + FilterOperator.EQUAL, + "12345askdfjh=wef67890", + "receivedDateTime ge 1900-01-01T00:00:00Z and conversationId eq '12345askdfjh=wef67890'", # noqa: E501 + ), + ( + EmailFilterProperty.CONVERSATION_ID, + FilterOperator.NOT_EQUAL, + "67890", + "receivedDateTime ge 1900-01-01T00:00:00Z and conversationId ne 67890", + ), + ( + EmailFilterProperty.RECEIVED_DATE_TIME, + FilterOperator.GREATER_THAN, + "2024-01-01", + "receivedDateTime gt '2024-01-01'", + ), + ( + EmailFilterProperty.SENDER, + FilterOperator.EQUAL, + "a@ex.com", + "receivedDateTime ge 1900-01-01T00:00:00Z and sender/emailAddress/address eq 'a@ex.com'", # noqa: E501 + ), + ( + EmailFilterProperty.SENDER, + FilterOperator.CONTAINS, + "joe", + "receivedDateTime ge 1900-01-01T00:00:00Z and contains(sender/emailAddress/address, 'joe')", # noqa: E501 + ), + ], +) +def test_create_filter_expression(property_, operator, value, expected_filter_expr): + assert _create_filter_expression(property_, operator, value) == expected_filter_expr