Slack tools to retrieve messages & metadata from multi-person DM conversation (#254)

toolkits/slack/arcade_slack/exceptions.py

@@ -17,8 +17,9 @@ class ItemNotFoundError(SlackToolkitError):
 class UsernameNotFoundError(SlackToolkitError):
     """Raised when a user is not found by the username searched"""
 
-    def __init__(self, usernames_found: list[str]) -> None:
+    def __init__(self, usernames_found: list[str], username_not_found: str) -> None:
         self.usernames_found = usernames_found
+        self.username_not_found = username_not_found
 
 
 class ConversationNotFoundError(SlackToolkitError):

toolkits/slack/arcade_slack/tools/chat.py

@@ -491,7 +491,7 @@ async def get_messages_in_direct_message_conversation_by_username(
     dict,
     (
         "The messages in a direct message conversation and next cursor for paginating results "
-        "(when there are additional messages to retrieve)."
+        "when there are additional messages to retrieve."
     ),
 ]:
     """Get the messages in a direct conversation by the user's name.
@@ -526,6 +526,79 @@ async def get_messages_in_direct_message_conversation_by_username(
     )
 
 
+@tool(requires_auth=Slack(scopes=["im:history", "im:read"]))
+async def get_messages_in_multi_person_dm_conversation_by_usernames(
+    context: ToolContext,
+    usernames: Annotated[list[str], "The usernames of the users to get messages from"],
+    oldest_relative: Annotated[
+        Optional[str],
+        (
+            "The oldest message to include in the results, specified as a time offset from the "
+            "current time in the format 'DD:HH:MM'"
+        ),
+    ] = None,
+    latest_relative: Annotated[
+        Optional[str],
+        (
+            "The latest message to include in the results, specified as a time offset from the "
+            "current time in the format 'DD:HH:MM'"
+        ),
+    ] = None,
+    oldest_datetime: Annotated[
+        Optional[str],
+        (
+            "The oldest message to include in the results, specified as a datetime object in the "
+            "format 'YYYY-MM-DD HH:MM:SS'"
+        ),
+    ] = None,
+    latest_datetime: Annotated[
+        Optional[str],
+        (
+            "The latest message to include in the results, specified as a datetime object in the "
+            "format 'YYYY-MM-DD HH:MM:SS'"
+        ),
+    ] = None,
+    limit: Annotated[Optional[int], "The maximum number of messages to return."] = None,
+    next_cursor: Annotated[Optional[str], "The cursor to use for pagination."] = None,
+) -> Annotated[
+    dict,
+    (
+        "The messages in a multi-person direct message conversation and next cursor for "
+        "paginating results (when there are additional messages to retrieve)."
+    ),
+]:
+    """Get the messages in a multi-person direct message conversation by the usernames.
+
+    To filter messages by an absolute datetime, use 'oldest_datetime' and/or 'latest_datetime'. If
+    only 'oldest_datetime' is provided, it will return messages from the oldest_datetime to the
+    current time. If only 'latest_datetime' is provided, it will return messages since the
+    beginning of the conversation to the latest_datetime.
+
+    To filter messages by a relative datetime (e.g. 3 days ago, 1 hour ago, etc.), use
+    'oldest_relative' and/or 'latest_relative'. If only 'oldest_relative' is provided, it will
+    return messages from the oldest_relative to the current time. If only 'latest_relative' is
+    provided, it will return messages from the current time to the latest_relative.
+
+    Do not provide both 'oldest_datetime' and 'oldest_relative' or both 'latest_datetime' and
+    'latest_relative'.
+
+    Leave all arguments with the default None to get messages without date/time filtering"""
+    direct_conversation = await get_multi_person_dm_conversation_metadata_by_usernames(
+        context=context, usernames=usernames
+    )
+
+    return await get_messages_in_conversation_by_id(  # type: ignore[no-any-return]
+        context=context,
+        conversation_id=direct_conversation["id"],
+        oldest_relative=oldest_relative,
+        latest_relative=latest_relative,
+        oldest_datetime=oldest_datetime,
+        latest_datetime=latest_datetime,
+        limit=limit,
+        next_cursor=next_cursor,
+    )
+
+
 @tool(
     requires_auth=Slack(
         scopes=["channels:read", "groups:read", "im:read", "mpim:read"],
@@ -535,7 +608,10 @@ async def get_conversation_metadata_by_id(
     context: ToolContext,
     conversation_id: Annotated[str, "The ID of the conversation to get metadata for"],
 ) -> Annotated[dict, "The conversation metadata"]:
-    """Get the metadata of a conversation in Slack searching by its ID."""
+    """Get the metadata of a conversation in Slack searching by its ID.
+
+    This tool does not return the messages in a conversation. To get the messages, use the
+    `get_messages_in_conversation_by_id` tool."""
     token = (
         context.authorization.token if context.authorization and context.authorization.token else ""
     )
@@ -577,7 +653,10 @@ async def get_channel_metadata_by_name(
         "The cursor to use for pagination, if continuing from a previous search.",
     ] = None,
 ) -> Annotated[dict, "The channel metadata"]:
-    """Get the metadata of a channel in Slack searching by its name."""
+    """Get the metadata of a channel in Slack searching by its name.
+
+    This tool does not return the messages in a channel. To get the messages, use the
+    `get_messages_in_channel_by_name` tool."""
     channel_names: list[str] = []
 
     async def find_channel() -> dict:
@@ -639,7 +718,10 @@ async def get_direct_message_conversation_metadata_by_username(
     Optional[dict],
     "The direct message conversation metadata.",
 ]:
-    """Get the metadata of a direct message conversation in Slack by the username."""
+    """Get the metadata of a direct message conversation in Slack by the username.
+
+    This tool does not return the messages in a conversation. To get the messages, use the
+    `get_messages_in_direct_message_conversation_by_username` tool."""
     try:
         token = (
             context.authorization.token
@@ -669,8 +751,73 @@ async def get_direct_message_conversation_metadata_by_username(
 
     except UsernameNotFoundError as e:
         raise RetryableToolError(
-            f"Username '{username}' not found",
+            f"Username '{e.username_not_found}' not found",
-            developer_message=f"User with username '{username}' not found.",
+            developer_message=f"User with username '{e.username_not_found}' not found.",
+            additional_prompt_content=f"Available users: {e.usernames_found}",
+            retry_after_ms=500,
+        )
+
+
+@tool(requires_auth=Slack(scopes=["im:read"]))
+async def get_multi_person_dm_conversation_metadata_by_usernames(
+    context: ToolContext,
+    usernames: Annotated[list[str], "The usernames of the users/people to get messages with"],
+    next_cursor: Annotated[
+        Optional[str],
+        "The cursor to use for pagination, if continuing from a previous search.",
+    ] = None,
+) -> Annotated[
+    Optional[dict],
+    "The multi-person direct message conversation metadata.",
+]:
+    """Get the metadata of a multi-person direct message conversation in Slack by the usernames.
+
+    This tool does not return the messages in a conversation. To get the messages, use the
+    `get_messages_in_multi_person_dm_conversation_by_usernames` tool.
+    """
+    try:
+        token = (
+            context.authorization.token
+            if context.authorization and context.authorization.token
+            else ""
+        )
+        slack_client = AsyncWebClient(token=token)
+
+        current_user, list_users_response = await asyncio.gather(
+            slack_client.auth_test(), list_users(context)
+        )
+
+        other_users = [
+            get_user_by_username(username, list_users_response["users"]) for username in usernames
+        ]
+
+        conversations_found = await retrieve_conversations_by_user_ids(
+            list_conversations_func=list_conversations_metadata,
+            get_members_in_conversation_func=get_members_in_conversation_by_id,
+            context=context,
+            conversation_types=[ConversationType.MULTI_PERSON_DIRECT_MESSAGE],
+            user_ids=[
+                current_user["user_id"],
+                *[user["id"] for user in other_users if user["id"] != current_user["user_id"]],
+            ],
+            exact_match=True,
+            limit=1,
+            next_cursor=next_cursor,
+        )
+
+        if not conversations_found:
+            raise RetryableToolError(
+                "Conversation not found with the usernames provided",
+                developer_message="Conversation not found with the usernames provided",
+                retry_after_ms=500,
+            )
+
+        return conversations_found[0]
+
+    except UsernameNotFoundError as e:
+        raise RetryableToolError(
+            f"Username '{e.username_not_found}' not found",
+            developer_message=f"User with username '{e.username_not_found}' not found.",
             additional_prompt_content=f"Available users: {e.usernames_found}",
             retry_after_ms=500,
         )

toolkits/slack/arcade_slack/utils.py

@@ -103,7 +103,7 @@ def get_user_by_username(username: str, users_list: list[dict]) -> SlackUser:
         if username.lower() == username_found.lower():
             return SlackUser(**user)
 
-    raise UsernameNotFoundError(usernames_found)
+    raise UsernameNotFoundError(usernames_found=usernames_found, username_not_found=username)
 
 
 def convert_conversation_type_to_slack_name(

toolkits/slack/evals/eval_chat.py

@@ -23,6 +23,7 @@ from arcade_slack.tools.chat import (
     get_messages_in_channel_by_name,
     get_messages_in_conversation_by_id,
     get_messages_in_direct_message_conversation_by_username,
+    get_messages_in_multi_person_dm_conversation_by_usernames,
     list_conversations_metadata,
     list_direct_message_conversations_metadata,
     list_group_direct_message_conversations_metadata,
@@ -1086,3 +1087,82 @@ def get_messages_in_direct_message_eval_suite() -> EvalSuite:
     )
 
     return suite
+
+
+@tool_eval()
+def get_messages_in_multi_person_direct_message_eval_suite() -> EvalSuite:
+    """Create an evaluation suite for tools getting messages in multi-person direct messages."""
+    suite = EvalSuite(
+        name="Slack Chat Tools Evaluation",
+        system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.",
+        catalog=catalog,
+        rubric=rubric,
+    )
+
+    no_arguments_user_messages_by_username = [
+        "what are the latest messages I exchanged together with the usernames john, ryan, and jennifer",
+        "show the messages in the multi person dm with the usernames john, ryan, and jennifer on Slack",
+        "list the messages I exchanged together with the usernames john, ryan, and jennifer",
+        "list the message history together with the usernames john, ryan, and jennifer",
+    ]
+
+    for i, user_message in enumerate(no_arguments_user_messages_by_username):
+        suite.add_case(
+            name=f"{user_message} [{i}]",
+            user_message=user_message,
+            expected_tool_calls=[
+                ExpectedToolCall(
+                    func=get_messages_in_multi_person_dm_conversation_by_usernames,
+                    args={
+                        "usernames": ["john", "ryan", "jennifer"],
+                    },
+                ),
+            ],
+            critics=[
+                BinaryCritic(critic_field="usernames", weight=1.0),
+            ],
+        )
+
+    suite.add_case(
+        name="get messages in direct conversation by username (on a specific date)",
+        user_message="get the messages I exchanged together with the usernames john, ryan, and jennifer on 2025-01-31",
+        expected_tool_calls=[
+            ExpectedToolCall(
+                func=get_messages_in_multi_person_dm_conversation_by_usernames,
+                args={
+                    "usernames": ["john", "ryan", "jennifer"],
+                    "oldest_datetime": "2025-01-31 00:00:00",
+                    "latest_datetime": "2025-01-31 23:59:59",
+                },
+            ),
+        ],
+        critics=[
+            BinaryCritic(critic_field="usernames", weight=1 / 3),
+            DatetimeCritic(
+                critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2)
+            ),
+            DatetimeCritic(
+                critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2)
+            ),
+        ],
+    )
+
+    suite.add_case(
+        name="Get conversation history oldest relative by username (2 days ago)",
+        user_message="Get the messages I exchanged together with the usernames john, ryan, and jennifer starting 2 days ago",
+        expected_tool_calls=[
+            ExpectedToolCall(
+                func=get_messages_in_multi_person_dm_conversation_by_usernames,
+                args={
+                    "usernames": ["john", "ryan", "jennifer"],
+                    "oldest_relative": "02:00:00",
+                },
+            ),
+        ],
+        critics=[
+            BinaryCritic(critic_field="usernames", weight=0.5),
+            RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5),
+        ],
+    )
+
+    return suite

toolkits/slack/tests/test_chat.py

@@ -18,6 +18,8 @@ from arcade_slack.tools.chat import (
     get_messages_in_channel_by_name,
     get_messages_in_conversation_by_id,
     get_messages_in_direct_message_conversation_by_username,
+    get_messages_in_multi_person_dm_conversation_by_usernames,
+    get_multi_person_dm_conversation_metadata_by_usernames,
     list_conversations_metadata,
     list_direct_message_conversations_metadata,
     list_group_direct_message_conversations_metadata,
@@ -818,3 +820,135 @@ async def test_get_messages_in_direct_conversation_by_username_not_found(
         await get_messages_in_direct_message_conversation_by_username(
             context=mock_context, username="user2"
         )
+
+
+@pytest.mark.asyncio
+@patch("arcade_slack.tools.chat.retrieve_conversations_by_user_ids")
+async def test_get_multi_person_direct_message_conversation_metadata_by_username(
+    mock_retrieve_conversations_by_user_ids,
+    mock_context,
+    mock_chat_slack_client,
+    mock_users_slack_client,
+):
+    mock_chat_slack_client.auth_test.return_value = {
+        "ok": True,
+        "user_id": "U1",
+        "team_id": "T1",
+        "user": "user1",
+    }
+
+    mock_users_slack_client.users_list.return_value = {
+        "ok": True,
+        "members": [
+            {"id": "U1", "name": "user1"},
+            {"id": "U2", "name": "user2"},
+            {"id": "U3", "name": "user3"},
+            {"id": "U4", "name": "user4"},
+            {"id": "U5", "name": "user5"},
+        ],
+        "response_metadata": {"next_cursor": None},
+    }
+
+    conversation = {
+        "id": "C12345",
+        "type": ConversationTypeSlackName.MPIM.value,
+        "is_mpim": True,
+        "members": ["U1", "U4", "U5"],
+    }
+
+    mock_retrieve_conversations_by_user_ids.return_value = [conversation]
+
+    response = await get_multi_person_dm_conversation_metadata_by_usernames(
+        context=mock_context, usernames=["user1", "user4", "user5"]
+    )
+
+    assert response == conversation
+    mock_retrieve_conversations_by_user_ids.assert_called_once_with(
+        list_conversations_func=list_conversations_metadata,
+        get_members_in_conversation_func=get_members_in_conversation_by_id,
+        context=mock_context,
+        conversation_types=[ConversationType.MULTI_PERSON_DIRECT_MESSAGE],
+        user_ids=["U1", "U4", "U5"],
+        exact_match=True,
+        limit=1,
+        next_cursor=None,
+    )
+
+
+@pytest.mark.asyncio
+@patch("arcade_slack.tools.chat.retrieve_conversations_by_user_ids")
+async def test_get_multi_person_direct_message_conversation_metadata_by_username_username_not_found(
+    mock_retrieve_conversations_by_user_ids,
+    mock_context,
+    mock_chat_slack_client,
+    mock_users_slack_client,
+):
+    mock_chat_slack_client.users_identity.return_value = {
+        "ok": True,
+        "user": {"id": "U1", "name": "user1"},
+        "team": {"id": "T1", "name": "team1"},
+    }
+
+    mock_users_slack_client.users_list.return_value = {
+        "ok": True,
+        "members": [
+            {"id": "U1", "name": "user1"},
+            {"id": "U2", "name": "user2"},
+        ],
+        "response_metadata": {"next_cursor": None},
+    }
+
+    mock_retrieve_conversations_by_user_ids.side_effect = TimeoutError()
+
+    with pytest.raises(RetryableToolError):
+        await get_multi_person_dm_conversation_metadata_by_usernames(
+            context=mock_context, usernames=["user999", "user1", "user2"]
+        )
+
+
+@pytest.mark.asyncio
+@patch("arcade_slack.tools.chat.get_messages_in_conversation_by_id")
+@patch("arcade_slack.tools.chat.get_multi_person_dm_conversation_metadata_by_usernames")
+async def test_get_messages_in_multi_person_dm_conversation_by_usernames(
+    mock_get_multi_person_dm_conversation_metadata_by_usernames,
+    mock_get_messages_in_conversation_by_id,
+    mock_context,
+):
+    mock_get_multi_person_dm_conversation_metadata_by_usernames.return_value = {
+        "id": "C12345",
+    }
+
+    response = await get_messages_in_multi_person_dm_conversation_by_usernames(
+        context=mock_context, usernames=["user1", "user4", "user5"]
+    )
+
+    assert response == mock_get_messages_in_conversation_by_id.return_value
+
+    mock_get_multi_person_dm_conversation_metadata_by_usernames.assert_called_once_with(
+        context=mock_context, usernames=["user1", "user4", "user5"]
+    )
+
+    mock_get_messages_in_conversation_by_id.assert_called_once_with(
+        context=mock_context,
+        conversation_id="C12345",
+        oldest_relative=None,
+        latest_relative=None,
+        oldest_datetime=None,
+        latest_datetime=None,
+        limit=None,
+        next_cursor=None,
+    )
+
+
+@pytest.mark.asyncio
+@patch("arcade_slack.tools.chat.get_multi_person_dm_conversation_metadata_by_usernames")
+async def test_get_messages_in_multi_person_dm_conversation_by_usernames_not_found(
+    mock_get_multi_person_dm_conversation_metadata_by_usernames,
+    mock_context,
+):
+    mock_get_multi_person_dm_conversation_metadata_by_usernames.return_value = None
+
+    with pytest.raises(ToolExecutionError):
+        await get_messages_in_direct_message_conversation_by_username(
+            context=mock_context, username="user2"
+        )