diff --git a/arcade/arcade/cli/utils.py b/arcade/arcade/cli/utils.py index 422715ed..524d220a 100644 --- a/arcade/arcade/cli/utils.py +++ b/arcade/arcade/cli/utils.py @@ -493,11 +493,7 @@ def get_eval_files(directory: str) -> list[Path]: directory_path = Path(directory).resolve() if directory_path.is_dir(): - eval_files = [ - f - for f in directory_path.iterdir() - if f.is_file() and f.name.startswith("eval_") and f.name.endswith(".py") - ] + eval_files = [f for f in directory_path.rglob("eval_*.py") if f.is_file()] elif directory_path.is_file(): eval_files = ( [directory_path] diff --git a/toolkits/microsoft/.pre-commit-config.yaml b/toolkits/microsoft/.pre-commit-config.yaml new file mode 100644 index 00000000..3953e996 --- /dev/null +++ b/toolkits/microsoft/.pre-commit-config.yaml @@ -0,0 +1,18 @@ +files: ^./ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v4.4.0" + hooks: + - id: check-case-conflict + - id: check-merge-conflict + - id: check-toml + - id: check-yaml + - id: end-of-file-fixer + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.7 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format diff --git a/toolkits/microsoft/.ruff.toml b/toolkits/microsoft/.ruff.toml new file mode 100644 index 00000000..9519fe6c --- /dev/null +++ b/toolkits/microsoft/.ruff.toml @@ -0,0 +1,44 @@ +target-version = "py310" +line-length = 100 +fix = true + +[lint] +select = [ + # flake8-2020 + "YTT", + # flake8-bandit + "S", + # flake8-bugbear + "B", + # flake8-builtins + "A", + # flake8-comprehensions + "C4", + # flake8-debugger + "T10", + # flake8-simplify + "SIM", + # isort + "I", + # mccabe + "C90", + # pycodestyle + "E", "W", + # pyflakes + "F", + # pygrep-hooks + "PGH", + # pyupgrade + "UP", + # ruff + "RUF", + # tryceratops + "TRY", +] + +[lint.per-file-ignores] +"**/tests/*" = ["S101"] + +[format] +preview = true +skip-magic-trailing-comma = false diff --git a/toolkits/microsoft/LICENSE b/toolkits/microsoft/LICENSE new file mode 100644 index 00000000..8c2d4f37 --- /dev/null +++ b/toolkits/microsoft/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025, Arcade + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/toolkits/microsoft/Makefile b/toolkits/microsoft/Makefile new file mode 100644 index 00000000..5c4e3a3c --- /dev/null +++ b/toolkits/microsoft/Makefile @@ -0,0 +1,58 @@ +.PHONY: help + +help: + @echo "🛠️ outlook_mail Commands:\n" + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.PHONY: install +install: ## Install the poetry environment and install the pre-commit hooks + @echo "📦 Checking if Poetry is installed" + @if ! command -v poetry >/dev/null 2>&1; then \ + echo "📦 Poetry not found. Checking if pip is available"; \ + if ! command -v pip >/dev/null 2>&1; then \ + echo "❌ pip is not installed. Please install pip first."; \ + exit 1; \ + fi; \ + echo "📦 Installing Poetry with pip"; \ + pip install poetry==1.8.5; \ + else \ + echo "📦 Poetry is already installed"; \ + fi + @echo "🚀 Installing package in development mode with all extras" + poetry install --all-extras + +.PHONY: build +build: clean-build ## Build wheel file using poetry + @echo "🚀 Creating wheel file" + poetry build + +.PHONY: clean-build +clean-build: ## clean build artifacts + @echo "🗑️ Cleaning dist directory" + rm -rf dist + +.PHONY: test +test: ## Test the code with pytest + @echo "🚀 Testing code: Running pytest" + @poetry run pytest -W ignore -v --cov --cov-config=pyproject.toml --cov-report=xml + +.PHONY: coverage +coverage: ## Generate coverage report + @echo "coverage report" + coverage report + @echo "Generating coverage report" + coverage html + +.PHONY: bump-version +bump-version: ## Bump the version in the pyproject.toml file + @echo "🚀 Bumping version in pyproject.toml" + poetry version patch + +.PHONY: check +check: ## Run code quality tools. + @echo "🚀 Checking Poetry lock file consistency with 'pyproject.toml': Running poetry check" + @poetry check + @echo "🚀 Linting code: Running pre-commit" + @poetry run pre-commit run -a + @echo "🚀 Static type checking: Running mypy" + @poetry run mypy --config-file=pyproject.toml diff --git a/toolkits/microsoft/arcade_microsoft/__init__.py b/toolkits/microsoft/arcade_microsoft/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/toolkits/microsoft/arcade_microsoft/client.py b/toolkits/microsoft/arcade_microsoft/client.py new file mode 100644 index 00000000..7bdcc205 --- /dev/null +++ b/toolkits/microsoft/arcade_microsoft/client.py @@ -0,0 +1,26 @@ +import datetime +from typing import Any + +from azure.core.credentials import AccessToken, TokenCredential +from msgraph import GraphServiceClient + +from arcade_microsoft.outlook_mail.constants import DEFAULT_SCOPE + + +class StaticTokenCredential(TokenCredential): + """Implementation of TokenCredential protocol to be provided to the MSGraph SDK client""" + + def __init__(self, token: str): + self._token = token + + def get_token(self, *scopes: str, **kwargs: Any) -> AccessToken: + # An expiration is required by MSGraph SDK. Set to 1 hour from now. + expires_on = int(datetime.datetime.now(datetime.timezone.utc).timestamp()) + 3600 + return AccessToken(self._token, expires_on) + + +def get_client(token: str) -> GraphServiceClient: + """Create and return a MSGraph SDK client, given the provided token.""" + token_credential = StaticTokenCredential(token) + + return GraphServiceClient(token_credential, scopes=[DEFAULT_SCOPE]) diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/__init__.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/__init__.py new file mode 100644 index 00000000..3e04aa37 --- /dev/null +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/__init__.py @@ -0,0 +1,22 @@ +from arcade_microsoft.outlook_mail.tools import ( + create_and_send_email, + create_draft_email, + list_emails, + list_emails_in_folder, + reply_to_email, + send_draft_email, + update_draft_email, +) + +__all__ = [ + # Read + "list_emails", + "list_emails_in_folder", + # Send + "create_and_send_email", + "send_draft_email", + "reply_to_email", + # Write + "create_draft_email", + "update_draft_email", +] diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/_utils.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/_utils.py new file mode 100644 index 00000000..9d281ab2 --- /dev/null +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/_utils.py @@ -0,0 +1,77 @@ +from arcade.sdk import ToolContext +from msgraph.generated.models.message_collection_response import MessageCollectionResponse +from msgraph.generated.users.item.mail_folders.item.messages.messages_request_builder import ( + MessagesRequestBuilder as MailFolderMessagesRequestBuilder, +) +from msgraph.generated.users.item.messages.item.reply.reply_post_request_body import ( + ReplyPostRequestBody, +) +from msgraph.generated.users.item.messages.item.reply_all.reply_all_post_request_body import ( + ReplyAllPostRequestBody, +) +from msgraph.generated.users.item.messages.messages_request_builder import ( + MessagesRequestBuilder as UserMessagesRequestBuilder, +) + +from arcade_microsoft.client import get_client +from arcade_microsoft.outlook_mail.constants import DEFAULT_MESSAGE_FIELDS +from arcade_microsoft.outlook_mail.enums import ReplyType + + +def remove_none_values(data: dict) -> dict: + """Remove all keys with None values from the dictionary.""" + return {k: v for k, v in data.items() if v is not None} + + +def prepare_list_emails_request_config( + limit: int, +) -> MailFolderMessagesRequestBuilder.MessagesRequestBuilderGetRequestConfiguration: + limit = max(1, min(limit, 100)) # limit must be between 1 and 100 + """Prepare a request configuration for listing emails.""" + query_params = MailFolderMessagesRequestBuilder.MessagesRequestBuilderGetQueryParameters( + count=True, + select=DEFAULT_MESSAGE_FIELDS, + orderby=["receivedDateTime DESC"], + top=limit, + ) + return MailFolderMessagesRequestBuilder.MessagesRequestBuilderGetRequestConfiguration( + query_parameters=query_params, + ) + + +async def fetch_emails( + message_builder: MailFolderMessagesRequestBuilder | UserMessagesRequestBuilder, + pagination_token: str | None = None, + request_config: MailFolderMessagesRequestBuilder.MessagesRequestBuilderGetRequestConfiguration + | None = None, +) -> MessageCollectionResponse: + """Fetch emails from the user's mailbox. + + Microsoft Graph Python SDK does not support pagination (as of 2025-04-17), + so we use raw URL for pagination if a pagination token is provided. + """ + if pagination_token: + return await message_builder.with_url(pagination_token).get() # type: ignore[return-value] + return await message_builder.get(request_configuration=request_config) # type: ignore[return-value, arg-type] + + +async def send_reply_email( + context: ToolContext, + message_id: str, + body: str, + reply_type: ReplyType, +) -> dict: + """Send a reply email to the sender or all recipients of an existing email.""" + client = get_client(context.get_auth_token_or_empty()) + + if reply_type == ReplyType.REPLY: + reply_request_body = ReplyPostRequestBody(comment=body) + await client.me.messages.by_message_id(message_id).reply.post(reply_request_body) + elif reply_type == ReplyType.REPLY_ALL: + reply_all_request_body = ReplyAllPostRequestBody(comment=body) + await client.me.messages.by_message_id(message_id).reply_all.post(reply_all_request_body) + + return { + "success": True, + "message": "Email sent successfully", + } diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/constants.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/constants.py new file mode 100644 index 00000000..628b513e --- /dev/null +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/constants.py @@ -0,0 +1,20 @@ +DEFAULT_SCOPE = "https://graph.microsoft.com/.default" + +DEFAULT_MESSAGE_FIELDS = [ + "bccRecipients", + "body", + "ccRecipients", + "conversationId", + "conversationIndex", + "flag", + "from", + "hasAttachments", + "importance", + "isDraft", + "isRead", + "receivedDateTime", + "replyTo", + "subject", + "toRecipients", + "webLink", +] diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/enums.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/enums.py new file mode 100644 index 00000000..05f55546 --- /dev/null +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/enums.py @@ -0,0 +1,23 @@ +from enum import Enum + + +class WellKnownFolderNames(str, Enum): + """Well-known folder names that are created for users by default. + Instead of using the ID of these folders, you can use the well-known folder names. + For a list of all well-known folder names, see: https://learn.microsoft.com/en-us/graph/api/resources/mailfolder?view=graph-rest-1.0 + """ + + DELETED_ITEMS = "deleteditems" + DRAFTS = "drafts" + INBOX = "inbox" + JUNK_EMAIL = "junkemail" + SENT_ITEMS = "sentitems" + STARRED = "starred" + TODO = "tasks" + + +class ReplyType(str, Enum): + """The type of reply to send to an email.""" + + REPLY = "reply" + REPLY_ALL = "reply_all" diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/message.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/message.py new file mode 100644 index 00000000..b7e2ec7f --- /dev/null +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/message.py @@ -0,0 +1,218 @@ +import re +from dataclasses import dataclass, field +from typing import Any + +from bs4 import BeautifulSoup +from msgraph.generated.models.body_type import BodyType +from msgraph.generated.models.email_address import EmailAddress +from msgraph.generated.models.item_body import ItemBody +from msgraph.generated.models.message import Message as GraphMessage +from msgraph.generated.models.recipient import Recipient as GraphRecipient + + +@dataclass +class Recipient: + """A recipient of an email message.""" + + email_address: str = "" + name: str = "" + + @classmethod + def from_sdk(cls, recipient: GraphRecipient) -> "Recipient": + """Convert a Microsoft Graph SDK Recipient object to a Recipient dataclass.""" + address = ( + recipient.email_address.address + if recipient and recipient.email_address and recipient.email_address.address + else "" + ) + name = ( + recipient.email_address.name + if recipient and recipient.email_address and recipient.email_address.name + else "" + ) + return cls(email_address=address, name=name) + + def to_dict(self) -> dict[str, str]: + return {"email_address": self.email_address, "name": self.name} + + def to_sdk(self) -> GraphRecipient: + """Converts the Recipient dataclass to a Microsoft Graph SDK Recipient object.""" + recipient = GraphRecipient() + email_address = EmailAddress() + email_address.address = self.email_address + email_address.name = self.name + recipient.email_address = email_address + return recipient + + +@dataclass +class Message: + """An email message in Outlook.""" + + bcc_recipients: list[Recipient] = field(default_factory=list) + cc_recipients: list[Recipient] = field(default_factory=list) + reply_to: list[Recipient] = field(default_factory=list) + to_recipients: list[Recipient] = field(default_factory=list) + from_: Recipient = field(default_factory=Recipient) + subject: str = "" + body: str = "" + conversation_id: str = "" + conversation_index: str = "" + flag: dict[str, str] = field(default_factory=dict) + has_attachments: bool = False + importance: str = "" + is_read: bool = False + received_date_time: str = "" + web_link: str = "" + is_draft: bool = True + message_id: str = "" # The unique identifier of the email message. Read-only. + + @staticmethod + def _safe_str(value: Any) -> str: + if not value: + return "" + if isinstance(value, bytes | bytearray): + return value.decode("utf-8", errors="ignore") + return str(value) + + @staticmethod + def _safe_bool(value: Any) -> bool: + return bool(value) + + @staticmethod + def _parse_body(mime: str) -> str: + if not mime: + return "" + soup = BeautifulSoup(mime, "html.parser") + text = soup.get_text(separator=" ") + # Replace multiple newlines with a single newline + text = re.sub(r"\n+", "\n", text) + # Replace multiple spaces with a single space + text = re.sub(r"\s+", " ", text) + # Remove leading/trailing whitespace from each line + text = "\n".join(line.strip() for line in text.split("\n")) + + return text + + @staticmethod + def _parse_importance(value: Any) -> str: + return value.value if getattr(value, "value", None) else "" + + @staticmethod + def _parse_flag(flag: Any) -> dict[str, str]: + if not flag: + return {"flag_status": "", "due_date_time": ""} + status = flag.flag_status.value if getattr(flag, "flag_status", None) else "" + due = "" + if getattr(flag, "due_date_time", None) and getattr(flag.due_date_time, "date_time", None): + due = Message._safe_str(flag.due_date_time.date_time) + return {"flag_status": status, "due_date_time": due} + + @classmethod + def from_sdk(cls, msg: GraphMessage) -> "Message": + """Convert a Microsoft Graph SDK Message object to a Message dataclass.""" + text = cls._parse_body(msg.body.content if msg.body and msg.body.content else "") + return cls( + bcc_recipients=[ + Recipient.from_sdk(recipient) for recipient in msg.bcc_recipients or [] + ], + cc_recipients=[Recipient.from_sdk(recipient) for recipient in msg.cc_recipients or []], + reply_to=[Recipient.from_sdk(recipient) for recipient in msg.reply_to or []], + to_recipients=[Recipient.from_sdk(recipient) for recipient in msg.to_recipients or []], + from_=Recipient.from_sdk(msg.from_) if msg.from_ else Recipient(), + subject=cls._safe_str(msg.subject), + body=text, + conversation_id=cls._safe_str(msg.conversation_id), + conversation_index=( + msg.conversation_index.decode("utf-8", errors="ignore") + if isinstance(msg.conversation_index, bytes | bytearray) + else cls._safe_str(msg.conversation_index) + ), + flag=cls._parse_flag(msg.flag), + has_attachments=cls._safe_bool(msg.has_attachments), + importance=cls._parse_importance(msg.importance), + is_read=cls._safe_bool(msg.is_read), + received_date_time=( + msg.received_date_time.isoformat() if msg.received_date_time else "" + ), + web_link=cls._safe_str(msg.web_link), + is_draft=cls._safe_bool(msg.is_draft), + message_id=cls._safe_str(msg.id), + ) + + def to_sdk(self) -> GraphMessage: + """Converts the Message dataclass to a Microsoft Graph SDK Message object.""" + sdk_msg = GraphMessage() + sdk_msg.subject = self.subject + body_obj = ItemBody() + body_obj.content = self.body + body_obj.content_type = BodyType.Text + sdk_msg.body = body_obj + sdk_msg.is_draft = self.is_draft + sdk_msg.to_recipients = [r.to_sdk() for r in self.to_recipients] + sdk_msg.cc_recipients = [r.to_sdk() for r in self.cc_recipients] + sdk_msg.bcc_recipients = [r.to_sdk() for r in self.bcc_recipients] + sdk_msg.reply_to = [r.to_sdk() for r in self.reply_to] + + return sdk_msg + + def to_dict(self) -> dict[str, Any]: + """Converts the Message dataclass to a dictionary.""" + return { + "bcc_recipients": [recipient.to_dict() for recipient in self.bcc_recipients], + "cc_recipients": [recipient.to_dict() for recipient in self.cc_recipients], + "reply_to": [recipient.to_dict() for recipient in self.reply_to], + "to_recipients": [recipient.to_dict() for recipient in self.to_recipients], + "from": self.from_.to_dict(), + "subject": self.subject, + "body": self.body, + "conversation_id": self.conversation_id, + "conversation_index": self.conversation_index, + "flag": self.flag, + "has_attachments": self.has_attachments, + "importance": self.importance, + "is_read": self.is_read, + "received_date_time": self.received_date_time, + "web_link": self.web_link, + "is_draft": self.is_draft, + "message_id": self.message_id, + } + + def update_recipient_lists( + self, + to_add: list[str] | None = None, + to_remove: list[str] | None = None, + cc_add: list[str] | None = None, + cc_remove: list[str] | None = None, + bcc_add: list[str] | None = None, + bcc_remove: list[str] | None = None, + ) -> None: + """Update each recipient list of the message. + + This function updates the recipient lists of the message by first adding new recipients + and then removing existing recipients. Therefore, if an email address is both + added and removed, then it will not be included in the returned list. + """ + for attr, add_emails_input, remove_emails_input in ( + ("to_recipients", to_add, to_remove), + ("cc_recipients", cc_add, cc_remove), + ("bcc_recipients", bcc_add, bcc_remove), + ): + current_recipients = getattr(self, attr) or [] + # Add recipients + existing_emails = {r.email_address.lower() for r in current_recipients} + new_additions = [ + Recipient(email_address=email) + for email in (add_emails_input or []) + if email.lower() not in existing_emails + ] + # Remove recipients + updated_list = current_recipients + new_additions + remove_emails = {email.lower() for email in (remove_emails_input or [])} + updated_list = [ + recipient + for recipient in updated_list + if recipient.email_address.lower() not in remove_emails + ] + # Update the message's attribute with the new list + setattr(self, attr, updated_list) diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/__init__.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/__init__.py new file mode 100644 index 00000000..b08b9dec --- /dev/null +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/__init__.py @@ -0,0 +1,26 @@ +from arcade_microsoft.outlook_mail.tools.read import ( + list_emails, + list_emails_in_folder, +) +from arcade_microsoft.outlook_mail.tools.send import ( + create_and_send_email, + reply_to_email, + send_draft_email, +) +from arcade_microsoft.outlook_mail.tools.write import ( + create_draft_email, + update_draft_email, +) + +__all__ = [ + # Read + "list_emails", + "list_emails_in_folder", + # Send + "create_and_send_email", + "reply_to_email", + "send_draft_email", + # Write + "create_draft_email", + "update_draft_email", +] diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/read.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/read.py new file mode 100644 index 00000000..be6ff1b4 --- /dev/null +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/read.py @@ -0,0 +1,89 @@ +from typing import Annotated + +from arcade.sdk import ToolContext, tool +from arcade.sdk.auth import Microsoft +from arcade.sdk.errors import ToolExecutionError + +from arcade_microsoft.client import get_client +from arcade_microsoft.outlook_mail._utils import ( + fetch_emails, + prepare_list_emails_request_config, + remove_none_values, +) +from arcade_microsoft.outlook_mail.enums import WellKnownFolderNames +from arcade_microsoft.outlook_mail.message import Message + + +@tool(requires_auth=Microsoft(scopes=["Mail.Read"])) +async def list_emails( + context: ToolContext, + limit: Annotated[int, "The number of messages to return. Max is 100. Defaults to 5."] = 5, + pagination_token: Annotated[ + str | None, "The pagination token to continue a previous request" + ] = None, +) -> Annotated[dict, "A dictionary containing a list of emails"]: + """List emails in the user's mailbox across all folders. + + Since this tool lists email across all folders, it may return sent items, drafts, + and other items that are not in the inbox. + """ + client = get_client(context.get_auth_token_or_empty()) + request_config = prepare_list_emails_request_config(limit) + message_builder = client.me.messages + + response = await fetch_emails(message_builder, pagination_token, request_config) + messages = [Message.from_sdk(msg).to_dict() for msg in response.value or []] + pagination_token = response.odata_next_link + + result = { + "messages": messages, + "num_messages": len(messages), + "pagination_token": pagination_token, + } + result = remove_none_values(result) + return result + + +@tool(requires_auth=Microsoft(scopes=["Mail.Read"])) +async def list_emails_in_folder( + context: ToolContext, + well_known_folder_name: Annotated[ + WellKnownFolderNames | None, + "The name of the folder to list emails from. Defaults to None.", + ] = None, + folder_id: Annotated[ + str | None, + "The ID of the folder to list emails from if the folder is not a well-known folder. " + "Defaults to None.", + ] = None, + limit: Annotated[int, "The number of messages to return. Max is 100. Defaults to 5."] = 5, + pagination_token: Annotated[ + str | None, "The pagination token to continue a previous request" + ] = None, +) -> Annotated[ + dict, "A dictionary containing a list of emails and a pagination token, if applicable" +]: + """List the user's emails in the specified folder. + + Exactly one of `well_known_folder_name` or `folder_id` MUST be provided. + """ + if not (bool(well_known_folder_name) ^ bool(folder_id)): + raise ToolExecutionError( + message="Exactly one of `well_known_folder_name` or `folder_id` must be provided." + ) + folder_name = well_known_folder_name.value if well_known_folder_name else folder_id + client = get_client(context.get_auth_token_or_empty()) + request_config = prepare_list_emails_request_config(limit) + message_builder = client.me.mail_folders.by_mail_folder_id(folder_name).messages # type: ignore[arg-type] + + response = await fetch_emails(message_builder, pagination_token, request_config) + messages = [Message.from_sdk(msg).to_dict() for msg in response.value or []] + pagination_token = response.odata_next_link + + result = { + "messages": messages, + "num_messages": len(messages), + "pagination_token": pagination_token, + } + result = remove_none_values(result) + return result diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/send.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/send.py new file mode 100644 index 00000000..1c68ccc6 --- /dev/null +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/send.py @@ -0,0 +1,94 @@ +from typing import Annotated + +from arcade.sdk import ToolContext, tool +from arcade.sdk.auth import Microsoft +from msgraph.generated.users.item.send_mail.send_mail_post_request_body import ( + SendMailPostRequestBody, +) + +from arcade_microsoft.client import get_client +from arcade_microsoft.outlook_mail._utils import send_reply_email +from arcade_microsoft.outlook_mail.enums import ReplyType +from arcade_microsoft.outlook_mail.message import Message, Recipient + + +@tool(requires_auth=Microsoft(scopes=["Mail.Send"])) +async def create_and_send_email( + context: ToolContext, + subject: Annotated[str, "The subject of the email to create"], + body: Annotated[str, "The body of the email to create"], + to_recipients: Annotated[ + list[str], "The email addresses that will be the recipients of the email" + ], + cc_recipients: Annotated[ + list[str] | None, "The email addresses that will be the CC recipients of the email." + ] = None, + bcc_recipients: Annotated[ + list[str] | None, + "The email addresses that will be the BCC recipients of the email.", + ] = None, +) -> Annotated[dict, "A dictionary containing the created email details"]: + """Create and immediately send a new email in Outlook to the specified recipients""" + client = get_client(context.get_auth_token_or_empty()) + message = Message( + subject=subject, + body=body, + to_recipients=[Recipient(email_address=email) for email in to_recipients], + cc_recipients=[Recipient(email_address=email) for email in cc_recipients or []], + bcc_recipients=[Recipient(email_address=email) for email in bcc_recipients or []], + ).to_sdk() + + send_mail_request_body = SendMailPostRequestBody( + message=message, + save_to_sent_items=True, + ) + + await client.me.send_mail.post(send_mail_request_body) + + return { + "success": True, + "message": "Email sent successfully", + } + + +@tool(requires_auth=Microsoft(scopes=["Mail.Send"])) +async def send_draft_email( + context: ToolContext, + message_id: Annotated[str, "The ID of the draft email to send"], +) -> Annotated[dict, "A dictionary containing the sent email details"]: + """Send an existing draft email in Outlook + + This tool can send any un-sent email: + - draft + - reply-draft + - reply-all draft + - forward draft + """ + client = get_client(context.get_auth_token_or_empty()) + + await client.me.messages.by_message_id(message_id).send.post() + + return { + "success": True, + "message": "Email sent successfully", + } + + +@tool(requires_auth=Microsoft(scopes=["Mail.Send"])) +async def reply_to_email( + context: ToolContext, + message_id: Annotated[str, "The ID of the email to reply to"], + body: Annotated[str, "The body of the reply to the email"], + reply_type: Annotated[ + ReplyType, + f"Specify {ReplyType.REPLY} to reply only to the sender or " + f"{ReplyType.REPLY_ALL} to reply to all recipients. " + f"Defaults to {ReplyType.REPLY}.", + ] = ReplyType.REPLY, +) -> Annotated[dict, "A dictionary containing the sent email details"]: + """Reply to an existing email in Outlook. + + Use this tool to reply to the sender or all recipients of the email. + Specify the reply_type to determine the scope of the reply. + """ + return await send_reply_email(context, message_id, body, reply_type) diff --git a/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/write.py b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/write.py new file mode 100644 index 00000000..0923ba1c --- /dev/null +++ b/toolkits/microsoft/arcade_microsoft/outlook_mail/tools/write.py @@ -0,0 +1,115 @@ +from typing import Annotated + +from arcade.sdk import ToolContext, tool +from arcade.sdk.auth import Microsoft +from arcade.sdk.errors import ToolExecutionError + +from arcade_microsoft.client import get_client +from arcade_microsoft.outlook_mail.message import Message, Recipient + + +@tool(requires_auth=Microsoft(scopes=["Mail.ReadWrite"])) +async def create_draft_email( + context: ToolContext, + subject: Annotated[str, "The subject of the draft email to create"], + body: Annotated[str, "The body of the draft email to create"], + to_recipients: Annotated[ + list[str], "The email addresses that will be the recipients of the draft email" + ], + cc_recipients: Annotated[ + list[str] | None, + "The email addresses that will be the CC recipients of the draft email.", + ] = None, + bcc_recipients: Annotated[ + list[str] | None, + "The email addresses that will be the BCC recipients of the draft email.", + ] = None, +) -> Annotated[dict, "A dictionary containing the created email details"]: + """Compose a new draft email in Outlook""" + client = get_client(context.get_auth_token_or_empty()) + + message = Message( + subject=subject, + body=body, + to_recipients=[Recipient(email_address=email) for email in to_recipients], + cc_recipients=[Recipient(email_address=email) for email in cc_recipients or []], + bcc_recipients=[Recipient(email_address=email) for email in bcc_recipients or []], + is_draft=True, + ).to_sdk() + + response = await client.me.messages.post(message) + draft_message = Message.from_sdk(response).to_dict() # type: ignore[arg-type] + + return draft_message + + +@tool(requires_auth=Microsoft(scopes=["Mail.ReadWrite"])) +async def update_draft_email( + context: ToolContext, + message_id: Annotated[str, "The ID of the draft email to update"], + subject: Annotated[ + str | None, + "The new subject of the draft email. If provided, the existing subject will be overwritten", + ] = None, + body: Annotated[ + str | None, + "The new body of the draft email. If provided, the existing body will be overwritten", + ] = None, + to_add: Annotated[list[str] | None, "Email addresses to add as 'To' recipients."] = None, + to_remove: Annotated[ + list[str] | None, + "Email addresses to remove from the current 'To' recipients.", + ] = None, + cc_add: Annotated[ + list[str] | None, + "Email addresses to add as 'CC' recipients.", + ] = None, + cc_remove: Annotated[ + list[str] | None, + "Email addresses to remove from the current 'CC' recipients.", + ] = None, + bcc_add: Annotated[ + list[str] | None, + "Email addresses to add as 'BCC' recipients.", + ] = None, + bcc_remove: Annotated[ + list[str] | None, + "Email addresses to remove from the current 'BCC' recipients.", + ] = None, +) -> Annotated[dict, "A dictionary containing the updated email details"]: + """Update an existing draft email in Outlook. + + This tool overwrites the subject and body of a draft email (if provided), + and modifies its recipient lists by selectively adding or removing email addresses. + + This tool can update any un-sent email: + - draft + - reply-draft + - reply-all draft + - forward draft + """ + client = get_client(context.get_auth_token_or_empty()) + + # Get the draft email + draft_email_sdk = await client.me.messages.by_message_id(message_id).get() + + if draft_email_sdk is None: + raise ToolExecutionError(message=f"The draft email with ID {message_id} was not found.") + + # Update the draft email + draft_email = Message.from_sdk(draft_email_sdk) + draft_email.subject = subject if subject else draft_email.subject + draft_email.body = body if body else draft_email.body or "" + draft_email.update_recipient_lists( + to_add=to_add, + to_remove=to_remove, + cc_add=cc_add, + cc_remove=cc_remove, + bcc_add=bcc_add, + bcc_remove=bcc_remove, + ) + updated_draft_email = await client.me.messages.by_message_id(message_id).patch( + draft_email.to_sdk() + ) + + return Message.from_sdk(updated_draft_email).to_dict() # type: ignore[arg-type] diff --git a/toolkits/microsoft/evals/outlook_mail/additional_messages.py b/toolkits/microsoft/evals/outlook_mail/additional_messages.py new file mode 100644 index 00000000..547b3109 --- /dev/null +++ b/toolkits/microsoft/evals/outlook_mail/additional_messages.py @@ -0,0 +1,83 @@ +update_draft_email_additional_messages = [ + {"role": "system", "content": "Today is 2025-04-22, Tuesday."}, + { + "role": "user", + "content": '"create a new draft email with subject \'Hello friends\' and body "\n"\'I\'ve gathered you all here to celebrate the launch of the new Arcade platform."\n "address it to e@arcade.dev and z@arcade.dev. also carbon copy to j@arcade.dev, "\n"f@arcade.dev, k@arcade.dev and finally to m@arcade.dev. also bcc to r@arcade.dev"', # noqa: E501 + }, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_lKw4S01FGe03oZeuW25roepy", + "type": "function", + "function": { + "name": "Microsoft_CreateDraftEmail", + "arguments": '{"subject":"Hello friends","body":"I\'ve gathered you all here to celebrate the launch of the new Arcade platform.","to_recipients":["e@arcade.dev","z@arcade.dev"],"cc_recipients":["j@arcade.dev","f@arcade.dev","k@arcade.dev","m@arcade.dev"],"bcc_recipients":["r@arcade.dev"]}', # noqa: E501 + }, + } + ], + }, + { + "role": "tool", + "content": '{"bcc_recipients":[{"email_address":"r@arcade.dev","name":"r@arcade.dev"}],"body":"I\'ve gathered you all here to celebrate the launch of the new Arcade platform.","cc_recipients":[{"email_address":"j@arcade.dev","name":"j@arcade.dev"},{"email_address":"f@arcade.dev","name":"f@arcade.dev"},{"email_address":"k@arcade.dev","name":"k@arcade.dev"},{"email_address":"m@arcade.dev","name":"m@arcade.dev"}],"conversation_id":"AQQkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoAEAAskq2oM-moTbt3gDT_yK0e","conversation_index":"AQHbs6c0LJKtqDP5qE27d4A0/sitHg==","flag":{"due_date_time":"","flag_status":"notFlagged"},"from":{"email_address":"","name":""},"has_attachments":false,"importance":"normal","is_draft":true,"is_read":true,"message_id":"AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4_x_WeUwAAAIBDwAAAFuxokOLZRtDncM4_x_WeUwAAAAC-dpvAAAA","received_date_time":"2025-04-22T16:54:25+00:00","reply_to":[],"subject":"Hello friends","to_recipients":[{"email_address":"e@arcade.dev","name":"e@arcade.dev"},{"email_address":"z@arcade.dev","name":"z@arcade.dev"}],"web_link":"https://outlook.live.com/owa/?ItemID=AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4%2Bx%2BWeUwAAAIBDwAAAFuxokOLZRtDncM4%2Bx%2BWeUwAAAAC%2FdpvAAAA\\u0026exvsurl=1\\u0026viewmodel=ReadMessageItem"}', # noqa: E501 + "tool_call_id": "call_lKw4S01FGe03oZeuW25roepy", + "name": "Microsoft_CreateDraftEmail", + }, + { + "role": "assistant", + "content": 'I have created a draft email with the subject "Hello friends" addressed to the specified recipients. You can view and edit the draft through [this link](https://outlook.live.com/owa/?ItemID=AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4%2Bx%2BWeUwAAAIBDwAAAFuxokOLZRtDncM4%2Bx%2BWeUwAAAAC%2FdpvAAAA&exvsurl=1&viewmodel=ReadMessageItem).', # noqa: E501 + }, +] + +list_emails_with_pagination_token_additional_messages = [ + {"role": "system", "content": "Today is 2025-04-21, Monday."}, + {"role": "user", "content": "get one email"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_jACvc3Gl1WHkqWgI8gdsIt0G", + "type": "function", + "function": {"name": "Microsoft_ListEmails", "arguments": '{"limit":1}'}, + } + ], + }, + { + "role": "tool", + "content": '{"messages":[{"bcc_recipients":[],"body":"Microsoft account New app(s) have access to your data Arcade","cc_recipients":["e@arcade.dev],"conversation_id":"AQQkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoAEABOD15A17tWSaVHkmjhko1R","conversation_index":"AQHbsuDd Tg9eQNe7VkmlR5Jo4ZKNUQ==","flag":{"due_date_time":"","flag_status":"notFlagged"},"from":{"email_address":"account-security@accountprotect ion.microsoft.com","name":"Microsoft account team"},"has_attachments":false,"importance":"normal","is_draft":false,"is_read":true,"message_id":"AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4_x_WeUwAAAIBDAAAAFuxokOLZRtDncM4_x_WeUwAAAABc_ezAAAA","received_date_time":"2025-04-21T17:14:39+00:00", "reply_to":[],"subject":"New app(s) connected to your Microsoft account","to_recipients":[{"email_address":"ericarcade@outlook.com","name":"ericarcade@outlook.com"}],"web_link":"https://outlook.live.com/owa/?ItemID=AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4%2Bx%2BWeUwAAAIBDAAAAFuxokOLZRtDncM4%2Bx%2BWeUwAAAABc%2BezAAAA\\u0026exvsurl=1\\u0026viewmodel=ReadMessageItem"}],"num_messages":1,"pagination_token":"https://graph.microsoft.com/v1.0/me/messages?%24count=true&%24orderby=receivedDateTime+DESC&%24select=bccRecipients%2cbody%2cccRecipients%2cconversationId%2cconversationIndex%2cflag%2cfrom%2chasAttachments%2cimportance%2cisDraft%2cisRead%2creceivedDateTime%2creplyTo%2csubject%2ctoRecipients%2cwebLink&%24top=1&%24skip=1"}', # noqa: E501 + "tool_call_id": "call_jACvc3Gl1WHkqWgI8gdsIt0G", + "name": "Microsoft_ListEmails", + }, + { + "role": "assistant", + "content": "Here is the most recent email you received:\n\n- **From:** Microsoft account team (account-security@accountprotection.microsoft.com)\n- **To:** e@outlook.com\n- **Subject:** New app(s) connected to your Microsoft account\n- **Received Date:** April 21, 2025\n- **Body:**\n ```\n Microsoft account\n\n New app(s) have access to your data Arcade connected to the Microsoft account *@outlook.com.```\n- **Link to email:** [Read in Outlook](https://outlook.live.com/owa/?ItemID=AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4%2Bx%2BWeUwAAAIBDAAAAFuxokOLZRtDncM4%2Bx%2BWeUwAAAABc%2BezAAAA&exvsurl=1&viewmodel=ReadMessageItem)", # noqa: E501 + }, +] + +list_emails_with_pagination_token_additional_messages = [ + {"role": "system", "content": "Today is 2025-04-21, Monday."}, + {"role": "user", "content": "get one email"}, + { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_jACvc3Gl1WHkqWgI8gdsIt0G", + "type": "function", + "function": {"name": "Microsoft_ListEmails", "arguments": '{"limit":1}'}, + } + ], + }, + { + "role": "tool", + "content": '{"messages":[{"bcc_recipients":[],"body":"Microsoft account New app(s) have access to your data Arcade","cc_recipients":[],"conversation_id":"AQQkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoAEABOD15A17tWSaVHkmjhko1R","conversation_index":"AQHbsuDdTg9eQNe7VkmlR5Jo4ZKNUQ==","flag":{"due_date_time":"","flag_status":"notFlagged"},"from":{"email_address":"account-security-noreply@accountprotection.microsoft.com","name":"Microsoft account team"},"has_attachments":false,"importance":"normal","is_draft":false,"is_read":true,"message_id":"AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4_x_WeUwAAAIBDAAAAFuxokOLZRtDncM4_x_WeUwAAAABc_ezAAAA","received_date_time":"2025-04-21T17:14:39+00:00", "reply_to":[],"subject":"New app(s) connected to your Microsoft account","to_recipients":[{"email_address":"ericarcade@outlook.com","name":"ericarcade@outlook.com"}],"web_link":"https://outlook.live.com/owa/?I temID=AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4%2Bx%2BWeUwAAAIBDAAAAFuxokOLZRtDncM4%2Bx%2BWeUwAAAA Bc%2BezAAAA\\u0026exvsurl=1\\u0026viewmodel=ReadMessageItem"}],"num_messages":1,"pagination_token":"https://graph.microsoft.com/v1.0/me/messages?%24count=true&%24orderby=receivedDateTime+DESC&%24select=bccRecipients%2cbody%2cccRecipients%2cconversationId%2cconversationIndex%2cflag%2cfrom%2chasAttachments%2cimportance%2cisDraft%2cisRead%2creceivedDateTime%2creplyTo%2csubject%2ctoRecipients%2cwebLink&%24top=1&%24skip=1"}', # noqa: E501 + "tool_call_id": "call_jACvc3Gl1WHkqWgI8gdsIt0G", + "name": "Microsoft_ListEmails", + }, + { + "role": "assistant", + "content": "Here is the most recent email you received:\n\n- **From:** Microsoft account team (account-security-noreply@accountprotection.microsoft.com)\n- **To:** e@outlook.com\n- **Subject:** New app(s) connected to your Microsoft account\n- **Received Date:** April 21, 2025\n- **Body:**\n ```\n Microsoft account\n\n New app(s) have access to your data Arcade connected to the Microsoft account *@outlook.com.```\n- **Link to email:** [Read in Outlook](https://outlook.live.com/owa/?ItemID=AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4%2Bx%2BWeUwAAAIBDAAAAFuxokOLZRtDncM4%2Bx%2BWeUwAAAABc%2BezAAAA&exvsurl=1&viewmodel=ReadMessageItem)", # noqa: E501 + }, +] diff --git a/toolkits/microsoft/evals/outlook_mail/eval_read.py b/toolkits/microsoft/evals/outlook_mail/eval_read.py new file mode 100644 index 00000000..25740d3e --- /dev/null +++ b/toolkits/microsoft/evals/outlook_mail/eval_read.py @@ -0,0 +1,109 @@ +from arcade.sdk import ToolCatalog +from arcade.sdk.eval import ( + BinaryCritic, + EvalRubric, + EvalSuite, + ExpectedToolCall, + tool_eval, +) + +from arcade_microsoft.outlook_mail import list_emails, list_emails_in_folder +from arcade_microsoft.outlook_mail.enums import WellKnownFolderNames +from evals.outlook_mail.additional_messages import ( + list_emails_with_pagination_token_additional_messages, +) + +# Evaluation rubric +rubric = EvalRubric( + fail_threshold=0.9, + warn_threshold=0.95, +) + + +catalog = ToolCatalog() +catalog.add_tool(list_emails, "Microsoft") +catalog.add_tool(list_emails_in_folder, "Microsoft") + + +@tool_eval() +def outlook_mail_eval_suite() -> EvalSuite: + """Create an evaluation suite for Outlook Mail tools.""" + suite = EvalSuite( + name="Outlook Mail Tools Evaluation", + system_message=("You are an AI that has access to tools to send, read, and write emails."), + catalog=catalog, + rubric=rubric, + ) + + suite.add_case( + name="List emails in mailbox", + user_message="get my five most recent emails", + expected_tool_calls=[ + ExpectedToolCall( + func=list_emails, + args={"limit": 5}, + ) + ], + critics=[ + BinaryCritic(critic_field="limit", weight=1.0), + ], + ) + + suite.add_case( + name="List emails in mailbox with pagination token", + user_message="get the next 3", + expected_tool_calls=[ + ExpectedToolCall( + func=list_emails, + args={ + "limit": 3, + "pagination_token": "https://graph.microsoft.com/v1.0/me/messages?%24count=true&%24orderby=receivedDateTime+DESC&%24select=bccRecipients%2cbody%2cccRecipients%2cconversationId%2cconversationIndex%2cflag%2cfrom%2chasAttachments%2cimportance%2cisDraft%2cisRead%2creceivedDateTime%2creplyTo%2csubject%2ctoRecipients%2cwebLink&%24top=1&%24skip=1", + }, + ) + ], + critics=[ + BinaryCritic(critic_field="limit", weight=0.2), + BinaryCritic(critic_field="pagination_token", weight=0.8), + ], + additional_messages=list_emails_with_pagination_token_additional_messages, + ) + + suite.add_case( + name="List emails in well-known folder", + user_message="summarize my inbox", + expected_tool_calls=[ + ExpectedToolCall( + func=list_emails_in_folder, + args={ + "well_known_folder_name": WellKnownFolderNames.INBOX, + "folder_id": None, + }, + ) + ], + critics=[ + BinaryCritic(critic_field="well_known_folder_name", weight=0.5), + BinaryCritic(critic_field="folder_id", weight=0.5), + ], + ) + + suite.add_case( + name="List emails in folder by id", + user_message="get 5 from folder AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoALgAAAyXxSd3UxTpCkDpGouEg0JMBAFuxokOLZRtDncM4", # noqa: E501 + expected_tool_calls=[ + ExpectedToolCall( + func=list_emails_in_folder, + args={ + "well_known_folder_name": None, + "folder_id": "AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoALgAAAyXxSd3UxTpCkDpGouEg0JMBAFuxokOLZRtDncM4", # noqa: E501 + "limit": 5, + }, + ) + ], + critics=[ + BinaryCritic(critic_field="well_known_folder_name", weight=0.4), + BinaryCritic(critic_field="folder_id", weight=0.4), + BinaryCritic(critic_field="limit", weight=0.2), + ], + ) + + return suite diff --git a/toolkits/microsoft/evals/outlook_mail/eval_send.py b/toolkits/microsoft/evals/outlook_mail/eval_send.py new file mode 100644 index 00000000..9fc046c6 --- /dev/null +++ b/toolkits/microsoft/evals/outlook_mail/eval_send.py @@ -0,0 +1,127 @@ +from arcade.sdk import ToolCatalog +from arcade.sdk.eval import ( + BinaryCritic, + EvalRubric, + EvalSuite, + ExpectedToolCall, + tool_eval, +) +from arcade.sdk.eval.critic import SimilarityCritic + +from arcade_microsoft.outlook_mail import ( + create_and_send_email, + reply_to_email, + send_draft_email, +) +from arcade_microsoft.outlook_mail.enums import ReplyType +from evals.outlook_mail.additional_messages import ( + list_emails_with_pagination_token_additional_messages, +) + +# Evaluation rubric +rubric = EvalRubric( + fail_threshold=0.9, + warn_threshold=0.95, +) + + +catalog = ToolCatalog() +catalog.add_tool(create_and_send_email, "Microsoft") +catalog.add_tool(send_draft_email, "Microsoft") +catalog.add_tool(reply_to_email, "Microsoft") + + +@tool_eval() +def outlook_mail_send_eval_suite() -> EvalSuite: + """Create an evaluation suite for Outlook Mail tools.""" + suite = EvalSuite( + name="Outlook Mail Send Evaluation", + system_message=("You are an AI that has access to tools to send, read, and write emails."), + catalog=catalog, + rubric=rubric, + ) + + suite.add_case( + name="Create draft email", + user_message=( + "send an email to j@arcade.dev and e@arcade.dev. Title it 'Hello friends' and have it " + "say 'I've gathered you all here to celebrate the launch of the new Arcade platform.'" + ), + expected_tool_calls=[ + ExpectedToolCall( + func=create_and_send_email, + args={ + "subject": "Hello friends", + "body": "I've gathered you all here to celebrate the launch of the new Arcade platform.", # noqa: E501 + "to_recipients": ["j@arcade.dev", "e@arcade.dev"], + }, + ) + ], + critics=[ + SimilarityCritic(critic_field="subject", weight=0.3), + SimilarityCritic(critic_field="body", weight=0.3), + BinaryCritic(critic_field="to_recipients", weight=0.4), + ], + ) + + suite.add_case( + name="Update draft email", + user_message=( + "forward the draft AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4_x_WeUwAAAIBDwAAAFuxokOLZRtDncM4_x_WeUwAAAAC-dpvAAAA " # noqa: E501 + ), + expected_tool_calls=[ + ExpectedToolCall( + func=send_draft_email, + args={ + "message_id": "AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4_x_WeUwAAAIBDwAAAFuxokOLZRtDncM4_x_WeUwAAAAC-dpvAAAA", # noqa: E501 + }, + ) + ], + critics=[ + BinaryCritic(critic_field="message_id", weight=1), + ], + ) + + suite.add_case( + name="Reply all to email", + user_message=("Reply to everyone - 'sounds good to me'"), + expected_tool_calls=[ + ExpectedToolCall( + func=reply_to_email, + args={ + "message_id": "AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4_x_WeUwAAAIBDAAAAFuxokOLZRtDncM4_x_WeUwAAAABc_ezAAAA", # noqa: E501 + "body": "sounds good to me", + "reply_type": ReplyType.REPLY_ALL, + }, + ) + ], + critics=[ + BinaryCritic(critic_field="message_id", weight=1 / 3), + SimilarityCritic(critic_field="body", weight=1 / 3), + BinaryCritic(critic_field="reply_type", weight=1 / 3), + ], + additional_messages=list_emails_with_pagination_token_additional_messages, + ) + + suite.add_case( + name="Reply to email", + user_message=("Reply to the account security team - 'sounds good to me'"), + expected_tool_calls=[ + ExpectedToolCall( + func=reply_to_email, + args={ + "message_id": "AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4_x_WeUwAAAIBDAAAAFuxokOLZRtDncM4_x_WeUwAAAABc_ezAAAA", # noqa: E501 + "body": "sounds good to me", + "reply_type": ReplyType.REPLY, + }, + ) + ], + critics=[ + BinaryCritic(critic_field="message_id", weight=1 / 3), + SimilarityCritic(critic_field="body", weight=1 / 3), + BinaryCritic(critic_field="reply_type", weight=1 / 3), + ], + additional_messages=list_emails_with_pagination_token_additional_messages, + ) + + return suite diff --git a/toolkits/microsoft/evals/outlook_mail/eval_write.py b/toolkits/microsoft/evals/outlook_mail/eval_write.py new file mode 100644 index 00000000..49209df4 --- /dev/null +++ b/toolkits/microsoft/evals/outlook_mail/eval_write.py @@ -0,0 +1,104 @@ +from arcade.sdk import ToolCatalog +from arcade.sdk.eval import ( + BinaryCritic, + EvalRubric, + EvalSuite, + ExpectedToolCall, + tool_eval, +) +from arcade.sdk.eval.critic import SimilarityCritic + +from arcade_microsoft.outlook_mail import create_draft_email, update_draft_email +from evals.outlook_mail.additional_messages import ( + update_draft_email_additional_messages, +) + +# Evaluation rubric +rubric = EvalRubric( + fail_threshold=0.9, + warn_threshold=0.95, +) + + +catalog = ToolCatalog() +catalog.add_tool(create_draft_email, "Microsoft") +catalog.add_tool(update_draft_email, "Microsoft") + + +@tool_eval() +def outlook_mail_write_eval_suite() -> EvalSuite: + """Create an evaluation suite for Outlook Mail tools.""" + suite = EvalSuite( + name="Outlook Mail Write Evaluation", + system_message=("You are an AI that has access to tools to send, read, and write emails."), + catalog=catalog, + rubric=rubric, + ) + + suite.add_case( + name="Create draft email", + user_message=( + "create a new draft email with subject 'Hello friends' and body " + "'I've gathered you all here to celebrate the launch of the new Arcade platform." + "address it to e@arcade.dev and z@arcade.dev. also carbon copy to j@arcade.dev, " + "f@arcade.dev, k@arcade.dev and finally to m@arcade.dev. also bcc to r@arcade.dev" + ), + expected_tool_calls=[ + ExpectedToolCall( + func=create_draft_email, + args={ + "subject": "Hello friends", + "body": "I've gathered you all here to celebrate the launch of the new Arcade platform.", # noqa: E501 + "to_recipients": ["e@arcade.dev", "z@arcade.dev"], + "cc_recipients": [ + "j@arcade.dev", + "f@arcade.dev", + "k@arcade.dev", + "m@arcade.dev", + ], + "bcc_recipients": ["r@arcade.dev"], + }, + ) + ], + critics=[ + SimilarityCritic(critic_field="subject", weight=0.2), + SimilarityCritic(critic_field="body", weight=0.2), + BinaryCritic(critic_field="to_recipients", weight=0.2), + BinaryCritic(critic_field="cc_recipients", weight=0.2), + BinaryCritic(critic_field="bcc_recipients", weight=0.2), + ], + ) + + suite.add_case( + name="Update draft email", + user_message=( + "oh wait i think i messed up on some emails. I meant 'z', not 'e'. " + "Also, I forgot to bcc y@arcade.dev. Also, replace the period with an " + "exclamation point since I want to convey excitement. Oh I almost forgot, " + "Don't cc anyone." + ), + expected_tool_calls=[ + ExpectedToolCall( + func=update_draft_email, + args={ + "message_id": "AQMkADAwATM0MDAAMi04Y2Y1LTQ3MTEALTAwAi0wMAoARgAAAyXxSd3UxTpCkDpGouEg0JMHAFuxokOLZRtDncM4_x_WeUwAAAIBDwAAAFuxokOLZRtDncM4_x_WeUwAAAAC-dpvAAAA", # noqa: E501 + "body": "I've gathered you all here to celebrate the launch of the new Arcade platform!", # noqa: E501 + "to_add": ["z@arcade.dev"], + "to_remove": ["e@arcade.dev"], + "cc_remove": ["j@arcade.dev", "f@arcade.dev", "k@arcade.dev", "m@arcade.dev"], + "bcc_add": ["y@arcade.dev"], + }, + ) + ], + critics=[ + BinaryCritic(critic_field="message_id", weight=1 / 6), + BinaryCritic(critic_field="body", weight=1 / 6), + BinaryCritic(critic_field="to_add", weight=1 / 6), + BinaryCritic(critic_field="to_remove", weight=1 / 6), + BinaryCritic(critic_field="cc_remove", weight=1 / 6), + BinaryCritic(critic_field="bcc_add", weight=1 / 6), + ], + additional_messages=update_draft_email_additional_messages, + ) + + return suite diff --git a/toolkits/microsoft/pyproject.toml b/toolkits/microsoft/pyproject.toml new file mode 100644 index 00000000..8b2b5ff5 --- /dev/null +++ b/toolkits/microsoft/pyproject.toml @@ -0,0 +1,41 @@ +[tool.poetry] +name = "arcade_microsoft" +version = "0.1.0" +description = "Arcade.dev LLM tools for Outlook Mail" +authors = ["Arcade "] + +[tool.poetry.dependencies] +python = "^3.10" +arcade-ai = "^1.3.2" +msgraph-sdk = "^1.28.0" +beautifulsoup4 = "^4.10.0" + +[tool.poetry.dev-dependencies] +pytest = "^8.3.0" +pytest-cov = "^4.0.0" +mypy = "^1.5.1" +pre-commit = "^3.4.0" +tox = "^4.11.1" +ruff = "^0.7.4" + +[build-system] +requires = ["poetry-core>=1.0.0,<2.0.0"] +build-backend = "poetry.core.masonry.api" + +[tool.mypy] +files = ["arcade_microsoft/**/*.py"] +python_version = "3.10" +disallow_untyped_defs = "True" +disallow_any_unimported = "True" +no_implicit_optional = "True" +check_untyped_defs = "True" +warn_return_any = "True" +warn_unused_ignores = "True" +show_error_codes = "True" +ignore_missing_imports = "True" + +[tool.pytest.ini_options] +testpaths = ["tests"] + +[tool.coverage.report] +skip_empty = true diff --git a/toolkits/microsoft/tests/__init__.py b/toolkits/microsoft/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/toolkits/microsoft/tests/outlook_mail/__init__.py b/toolkits/microsoft/tests/outlook_mail/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/toolkits/microsoft/tests/outlook_mail/test_message.py b/toolkits/microsoft/tests/outlook_mail/test_message.py new file mode 100644 index 00000000..c189c40d --- /dev/null +++ b/toolkits/microsoft/tests/outlook_mail/test_message.py @@ -0,0 +1,249 @@ +import pytest +from msgraph.generated.models.email_address import EmailAddress as GraphEmailAddress +from msgraph.generated.models.message import Message as GraphMessage +from msgraph.generated.models.recipient import Recipient as GraphRecipient + +from arcade_microsoft.outlook_mail.message import Message, Recipient + + +# Dummy classes to simulate SDK objects +class DummyBody: + def __init__(self, content): + self.content = content + + +class DummyFlagStatus: + def __init__(self, value): + self.value = value + + +class DummyImportance: + def __init__(self, value): + self.value = value + + +class DummyDueDateTime: + def __init__(self, date_time): + self.date_time = date_time + + +class DummyFlag: + def __init__(self, flag_status, due_date_time): + self.flag_status = DummyFlagStatus(flag_status) + self.due_date_time = DummyDueDateTime(due_date_time) + + +class DummyDateTime: + def __init__(self, date_str): + self.date_str = date_str + + def isoformat(self): + return self.date_str + + +def make_graph_recipient(rec_data): + recipient = GraphRecipient() + recipient.email_address = GraphEmailAddress() + recipient.email_address.address = rec_data["email_address"] + recipient.email_address.name = rec_data.get("name", "") + return recipient + + +@pytest.mark.parametrize( + "input_data, expected", + [ + ( + { + "body_content": "

Hello world

", + "subject": "Test subject", + "conversation_id": "conv-1", + "conversation_index": "conv-index", + "flag_status": "flagged", + "due_date_time": "2021-01-01T10:00:00", + "has_attachments": False, + "importance": "high", + "is_read": True, + "received_date_time": "2021-01-02T00:00:00", + "web_link": "http://example.com", + "is_draft": False, + "message_id": "1234", + "to_recipients": [{"email_address": "to@example.com", "name": "ToName"}], + "cc_recipients": [{"email_address": "cc@example.com", "name": "CcName"}], + "bcc_recipients": [{"email_address": "bcc@example.com", "name": "BccName"}], + "reply_to": [{"email_address": "reply@example.com", "name": "ReplyName"}], + "from_": {"email_address": "from@example.com", "name": "FromName"}, + "conversation_index_bytes": False, + }, + { + "body": "Hello world", + "subject": "Test subject", + "conversation_id": "conv-1", + "conversation_index": "conv-index", + "flag": {"flag_status": "flagged", "due_date_time": "2021-01-01T10:00:00"}, + "has_attachments": False, + "importance": "high", + "is_read": True, + "received_date_time": "2021-01-02T00:00:00", + "web_link": "http://example.com", + "is_draft": False, + "message_id": "1234", + "to_recipients": [{"email_address": "to@example.com", "name": "ToName"}], + "cc_recipients": [{"email_address": "cc@example.com", "name": "CcName"}], + "bcc_recipients": [{"email_address": "bcc@example.com", "name": "BccName"}], + "reply_to": [{"email_address": "reply@example.com", "name": "ReplyName"}], + "from_": {"email_address": "from@example.com", "name": "FromName"}, + }, + ), + ( + { + "body_content": "

Sample email message

", + "subject": "Another subject", + "conversation_id": "conv-2", + "conversation_index": b"byte-index", + "flag_status": "notFlaged", + "due_date_time": "", + "has_attachments": False, + "importance": "low", + "is_read": False, + "received_date_time": "", + "web_link": "", + "is_draft": True, + "message_id": "5678", + "to_recipients": [{"email_address": "user1@example.com", "name": "User1"}], + "cc_recipients": [], + "bcc_recipients": [], + "reply_to": [], + "from_": {"email_address": "sender@example.com", "name": "Sender"}, + "conversation_index_bytes": True, + }, + { + "body": "Sample email message", + "subject": "Another subject", + "conversation_id": "conv-2", + "conversation_index": "byte-index", + "flag": {"flag_status": "notFlaged", "due_date_time": ""}, + "has_attachments": False, + "importance": "low", + "is_read": False, + "received_date_time": "", + "web_link": "", + "is_draft": True, + "message_id": "5678", + "to_recipients": [{"email_address": "user1@example.com", "name": "User1"}], + "cc_recipients": [], + "bcc_recipients": [], + "reply_to": [], + "from_": {"email_address": "sender@example.com", "name": "Sender"}, + }, + ), + ], +) +def test_message_conversion(input_data, expected): + # Set up sdk message + sdk_message = GraphMessage() + sdk_message.body = ( + DummyBody(input_data["body_content"]) if "body_content" in input_data else None + ) + sdk_message.subject = input_data["subject"] + sdk_message.conversation_id = input_data["conversation_id"] + sdk_message.conversation_index = input_data["conversation_index"] + sdk_message.flag = ( + DummyFlag(input_data["flag_status"], input_data["due_date_time"]) + if "flag_status" in input_data + else None + ) + sdk_message.has_attachments = input_data["has_attachments"] + sdk_message.importance = DummyImportance(input_data["importance"]) + sdk_message.is_read = input_data["is_read"] + sdk_message.received_date_time = ( + DummyDateTime(input_data["received_date_time"]) + if input_data["received_date_time"] + else None + ) + sdk_message.web_link = input_data["web_link"] + sdk_message.is_draft = input_data["is_draft"] + sdk_message.id = input_data["message_id"] + sdk_message.to_recipients = [make_graph_recipient(r) for r in input_data["to_recipients"]] + sdk_message.cc_recipients = [make_graph_recipient(r) for r in input_data["cc_recipients"]] + sdk_message.bcc_recipients = [make_graph_recipient(r) for r in input_data["bcc_recipients"]] + sdk_message.reply_to = [make_graph_recipient(r) for r in input_data["reply_to"]] + sdk_message.from_ = make_graph_recipient(input_data["from_"]) + + # Convert to Arcade Message type + message = Message.from_sdk(sdk_message) + + # Ensure conversion is correct + assert message.body == expected["body"], "Body conversion mismatch" + assert message.subject == expected["subject"] + assert message.conversation_id == expected["conversation_id"] + assert message.conversation_index == expected["conversation_index"] + assert message.flag == expected["flag"] + assert message.has_attachments == expected["has_attachments"] + assert message.importance == expected["importance"] + assert message.is_read == expected["is_read"] + assert message.received_date_time == expected["received_date_time"] + assert message.web_link == expected["web_link"] + assert message.is_draft == expected["is_draft"] + assert message.message_id == expected["message_id"] + assert message.from_.email_address == expected["from_"]["email_address"] + assert message.from_.name == expected["from_"]["name"] + + def check_recipient_list(actual, exp_list): + assert len(actual) == len(exp_list) + for rec, exp in zip(actual, exp_list, strict=False): + assert rec.email_address == exp["email_address"] + assert rec.name == exp["name"] + + check_recipient_list(message.to_recipients, expected["to_recipients"]) + check_recipient_list(message.cc_recipients, expected["cc_recipients"]) + check_recipient_list(message.bcc_recipients, expected["bcc_recipients"]) + check_recipient_list(message.reply_to, expected["reply_to"]) + + +@pytest.mark.parametrize( + "initial, add_params, expected_to_recipients", + [ + # Add a "To" recipient + ( + {"to_recipients": []}, + {"to_add": ["new@example.com"]}, + [{"email_address": "new@example.com", "name": ""}], + ), + # Add a "To" recipient that already exists + ( + {"to_recipients": [{"email_address": "dup@example.com", "name": ""}]}, + {"to_add": ["dup@example.com"]}, + [ + {"email_address": "dup@example.com", "name": ""}, + ], + ), + # Remove a "To" recipient + ( + { + "to_recipients": [ + {"email_address": "a@example.com", "name": "A"}, + {"email_address": "b@example.com", "name": "B"}, + ] + }, + {"to_remove": ["a@example.com"]}, + [{"email_address": "b@example.com", "name": "B"}], + ), + # Add and remove a "To" recipient + ( + {"to_recipients": [{"email_address": "c@example.com", "name": "C"}]}, + {"to_add": ["d@example.com", "c@example.com"], "to_remove": ["c@example.com"]}, + [{"email_address": "d@example.com", "name": ""}], + ), + ], +) +def test_update_recipient_lists(initial, add_params, expected_to_recipients): + msg = Message() + msg.to_recipients = [ + Recipient(email_address=r["email_address"], name=r.get("name", "")) + for r in initial.get("to_recipients", []) + ] + msg.update_recipient_lists( + to_add=add_params.get("to_add"), to_remove=add_params.get("to_remove") + ) + result = [r.to_dict() for r in msg.to_recipients] + assert result == expected_to_recipients, f"Expected {expected_to_recipients}, got {result}" diff --git a/toolkits/microsoft/tests/outlook_mail/test_recipient.py b/toolkits/microsoft/tests/outlook_mail/test_recipient.py new file mode 100644 index 00000000..8dcb021f --- /dev/null +++ b/toolkits/microsoft/tests/outlook_mail/test_recipient.py @@ -0,0 +1,43 @@ +import pytest +from msgraph.generated.models.email_address import EmailAddress as GraphEmailAddress +from msgraph.generated.models.recipient import Recipient as GraphRecipient + +from arcade_microsoft.outlook_mail.message import Recipient + + +@pytest.mark.parametrize( + "input_sdk_recipient, expected_email, expected_name", + [ + ( + GraphRecipient(email_address=GraphEmailAddress(address="dev@arcade.dev", name="Dev")), + "dev@arcade.dev", + "Dev", + ), + ( + GraphRecipient(email_address=GraphEmailAddress(address="dev@arcade.dev")), + "dev@arcade.dev", + "", + ), + (GraphRecipient(email_address=GraphEmailAddress(name="Dev")), "", "Dev"), + (GraphRecipient(email_address=GraphEmailAddress()), "", ""), + (GraphRecipient(), "", ""), + ], +) +def test_recipient(input_sdk_recipient, expected_email, expected_name): + recipient = Recipient.from_sdk(input_sdk_recipient) + assert ( + recipient.email_address == expected_email + ), "SDK conversion didn't set email_address correctly" + assert recipient.name == expected_name, "SDK conversion didn't set name correctly" + + recipient_dict = recipient.to_dict() + expected_dict = {"email_address": expected_email, "name": expected_name} + assert recipient_dict == expected_dict, "to_dict conversion did not produce expected dictionary" + + actual_sdk_recipient = recipient.to_sdk() + assert ( + actual_sdk_recipient.email_address.address == expected_email + ), "to_sdk conversion produced wrong email address" + assert ( + actual_sdk_recipient.email_address.name == expected_name + ), "to_sdk conversion produced wrong name"