From 8efa9a51dfd28d2c1bcdb0adbfc0171ef034941d Mon Sep 17 00:00:00 2001 From: Renato Byrro Date: Wed, 19 Feb 2025 16:51:45 -0300 Subject: [PATCH] Slack tools to retrieve messages & metadata from multi-person DM conversation (#254) --- toolkits/slack/arcade_slack/exceptions.py | 3 +- toolkits/slack/arcade_slack/tools/chat.py | 159 +++++++++++++++++++++- toolkits/slack/arcade_slack/utils.py | 2 +- toolkits/slack/evals/eval_chat.py | 80 +++++++++++ toolkits/slack/tests/test_chat.py | 134 ++++++++++++++++++ 5 files changed, 370 insertions(+), 8 deletions(-) diff --git a/toolkits/slack/arcade_slack/exceptions.py b/toolkits/slack/arcade_slack/exceptions.py index 49e40a75..8f4ff55a 100644 --- a/toolkits/slack/arcade_slack/exceptions.py +++ b/toolkits/slack/arcade_slack/exceptions.py @@ -17,8 +17,9 @@ class ItemNotFoundError(SlackToolkitError): class UsernameNotFoundError(SlackToolkitError): """Raised when a user is not found by the username searched""" - def __init__(self, usernames_found: list[str]) -> None: + def __init__(self, usernames_found: list[str], username_not_found: str) -> None: self.usernames_found = usernames_found + self.username_not_found = username_not_found class ConversationNotFoundError(SlackToolkitError): diff --git a/toolkits/slack/arcade_slack/tools/chat.py b/toolkits/slack/arcade_slack/tools/chat.py index 9daf5e11..3ce610ec 100644 --- a/toolkits/slack/arcade_slack/tools/chat.py +++ b/toolkits/slack/arcade_slack/tools/chat.py @@ -491,7 +491,7 @@ async def get_messages_in_direct_message_conversation_by_username( dict, ( "The messages in a direct message conversation and next cursor for paginating results " - "(when there are additional messages to retrieve)." + "when there are additional messages to retrieve." ), ]: """Get the messages in a direct conversation by the user's name. @@ -526,6 +526,79 @@ async def get_messages_in_direct_message_conversation_by_username( ) +@tool(requires_auth=Slack(scopes=["im:history", "im:read"])) +async def get_messages_in_multi_person_dm_conversation_by_usernames( + context: ToolContext, + usernames: Annotated[list[str], "The usernames of the users to get messages from"], + oldest_relative: Annotated[ + Optional[str], + ( + "The oldest message to include in the results, specified as a time offset from the " + "current time in the format 'DD:HH:MM'" + ), + ] = None, + latest_relative: Annotated[ + Optional[str], + ( + "The latest message to include in the results, specified as a time offset from the " + "current time in the format 'DD:HH:MM'" + ), + ] = None, + oldest_datetime: Annotated[ + Optional[str], + ( + "The oldest message to include in the results, specified as a datetime object in the " + "format 'YYYY-MM-DD HH:MM:SS'" + ), + ] = None, + latest_datetime: Annotated[ + Optional[str], + ( + "The latest message to include in the results, specified as a datetime object in the " + "format 'YYYY-MM-DD HH:MM:SS'" + ), + ] = None, + limit: Annotated[Optional[int], "The maximum number of messages to return."] = None, + next_cursor: Annotated[Optional[str], "The cursor to use for pagination."] = None, +) -> Annotated[ + dict, + ( + "The messages in a multi-person direct message conversation and next cursor for " + "paginating results (when there are additional messages to retrieve)." + ), +]: + """Get the messages in a multi-person direct message conversation by the usernames. + + To filter messages by an absolute datetime, use 'oldest_datetime' and/or 'latest_datetime'. If + only 'oldest_datetime' is provided, it will return messages from the oldest_datetime to the + current time. If only 'latest_datetime' is provided, it will return messages since the + beginning of the conversation to the latest_datetime. + + To filter messages by a relative datetime (e.g. 3 days ago, 1 hour ago, etc.), use + 'oldest_relative' and/or 'latest_relative'. If only 'oldest_relative' is provided, it will + return messages from the oldest_relative to the current time. If only 'latest_relative' is + provided, it will return messages from the current time to the latest_relative. + + Do not provide both 'oldest_datetime' and 'oldest_relative' or both 'latest_datetime' and + 'latest_relative'. + + Leave all arguments with the default None to get messages without date/time filtering""" + direct_conversation = await get_multi_person_dm_conversation_metadata_by_usernames( + context=context, usernames=usernames + ) + + return await get_messages_in_conversation_by_id( # type: ignore[no-any-return] + context=context, + conversation_id=direct_conversation["id"], + oldest_relative=oldest_relative, + latest_relative=latest_relative, + oldest_datetime=oldest_datetime, + latest_datetime=latest_datetime, + limit=limit, + next_cursor=next_cursor, + ) + + @tool( requires_auth=Slack( scopes=["channels:read", "groups:read", "im:read", "mpim:read"], @@ -535,7 +608,10 @@ async def get_conversation_metadata_by_id( context: ToolContext, conversation_id: Annotated[str, "The ID of the conversation to get metadata for"], ) -> Annotated[dict, "The conversation metadata"]: - """Get the metadata of a conversation in Slack searching by its ID.""" + """Get the metadata of a conversation in Slack searching by its ID. + + This tool does not return the messages in a conversation. To get the messages, use the + `get_messages_in_conversation_by_id` tool.""" token = ( context.authorization.token if context.authorization and context.authorization.token else "" ) @@ -577,7 +653,10 @@ async def get_channel_metadata_by_name( "The cursor to use for pagination, if continuing from a previous search.", ] = None, ) -> Annotated[dict, "The channel metadata"]: - """Get the metadata of a channel in Slack searching by its name.""" + """Get the metadata of a channel in Slack searching by its name. + + This tool does not return the messages in a channel. To get the messages, use the + `get_messages_in_channel_by_name` tool.""" channel_names: list[str] = [] async def find_channel() -> dict: @@ -639,7 +718,10 @@ async def get_direct_message_conversation_metadata_by_username( Optional[dict], "The direct message conversation metadata.", ]: - """Get the metadata of a direct message conversation in Slack by the username.""" + """Get the metadata of a direct message conversation in Slack by the username. + + This tool does not return the messages in a conversation. To get the messages, use the + `get_messages_in_direct_message_conversation_by_username` tool.""" try: token = ( context.authorization.token @@ -669,8 +751,73 @@ async def get_direct_message_conversation_metadata_by_username( except UsernameNotFoundError as e: raise RetryableToolError( - f"Username '{username}' not found", - developer_message=f"User with username '{username}' not found.", + f"Username '{e.username_not_found}' not found", + developer_message=f"User with username '{e.username_not_found}' not found.", + additional_prompt_content=f"Available users: {e.usernames_found}", + retry_after_ms=500, + ) + + +@tool(requires_auth=Slack(scopes=["im:read"])) +async def get_multi_person_dm_conversation_metadata_by_usernames( + context: ToolContext, + usernames: Annotated[list[str], "The usernames of the users/people to get messages with"], + next_cursor: Annotated[ + Optional[str], + "The cursor to use for pagination, if continuing from a previous search.", + ] = None, +) -> Annotated[ + Optional[dict], + "The multi-person direct message conversation metadata.", +]: + """Get the metadata of a multi-person direct message conversation in Slack by the usernames. + + This tool does not return the messages in a conversation. To get the messages, use the + `get_messages_in_multi_person_dm_conversation_by_usernames` tool. + """ + try: + token = ( + context.authorization.token + if context.authorization and context.authorization.token + else "" + ) + slack_client = AsyncWebClient(token=token) + + current_user, list_users_response = await asyncio.gather( + slack_client.auth_test(), list_users(context) + ) + + other_users = [ + get_user_by_username(username, list_users_response["users"]) for username in usernames + ] + + conversations_found = await retrieve_conversations_by_user_ids( + list_conversations_func=list_conversations_metadata, + get_members_in_conversation_func=get_members_in_conversation_by_id, + context=context, + conversation_types=[ConversationType.MULTI_PERSON_DIRECT_MESSAGE], + user_ids=[ + current_user["user_id"], + *[user["id"] for user in other_users if user["id"] != current_user["user_id"]], + ], + exact_match=True, + limit=1, + next_cursor=next_cursor, + ) + + if not conversations_found: + raise RetryableToolError( + "Conversation not found with the usernames provided", + developer_message="Conversation not found with the usernames provided", + retry_after_ms=500, + ) + + return conversations_found[0] + + except UsernameNotFoundError as e: + raise RetryableToolError( + f"Username '{e.username_not_found}' not found", + developer_message=f"User with username '{e.username_not_found}' not found.", additional_prompt_content=f"Available users: {e.usernames_found}", retry_after_ms=500, ) diff --git a/toolkits/slack/arcade_slack/utils.py b/toolkits/slack/arcade_slack/utils.py index 159cf9bc..b2a38b4c 100644 --- a/toolkits/slack/arcade_slack/utils.py +++ b/toolkits/slack/arcade_slack/utils.py @@ -103,7 +103,7 @@ def get_user_by_username(username: str, users_list: list[dict]) -> SlackUser: if username.lower() == username_found.lower(): return SlackUser(**user) - raise UsernameNotFoundError(usernames_found) + raise UsernameNotFoundError(usernames_found=usernames_found, username_not_found=username) def convert_conversation_type_to_slack_name( diff --git a/toolkits/slack/evals/eval_chat.py b/toolkits/slack/evals/eval_chat.py index 55b39e91..3784a80c 100644 --- a/toolkits/slack/evals/eval_chat.py +++ b/toolkits/slack/evals/eval_chat.py @@ -23,6 +23,7 @@ from arcade_slack.tools.chat import ( get_messages_in_channel_by_name, get_messages_in_conversation_by_id, get_messages_in_direct_message_conversation_by_username, + get_messages_in_multi_person_dm_conversation_by_usernames, list_conversations_metadata, list_direct_message_conversations_metadata, list_group_direct_message_conversations_metadata, @@ -1086,3 +1087,82 @@ def get_messages_in_direct_message_eval_suite() -> EvalSuite: ) return suite + + +@tool_eval() +def get_messages_in_multi_person_direct_message_eval_suite() -> EvalSuite: + """Create an evaluation suite for tools getting messages in multi-person direct messages.""" + suite = EvalSuite( + name="Slack Chat Tools Evaluation", + system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", + catalog=catalog, + rubric=rubric, + ) + + no_arguments_user_messages_by_username = [ + "what are the latest messages I exchanged together with the usernames john, ryan, and jennifer", + "show the messages in the multi person dm with the usernames john, ryan, and jennifer on Slack", + "list the messages I exchanged together with the usernames john, ryan, and jennifer", + "list the message history together with the usernames john, ryan, and jennifer", + ] + + for i, user_message in enumerate(no_arguments_user_messages_by_username): + suite.add_case( + name=f"{user_message} [{i}]", + user_message=user_message, + expected_tool_calls=[ + ExpectedToolCall( + func=get_messages_in_multi_person_dm_conversation_by_usernames, + args={ + "usernames": ["john", "ryan", "jennifer"], + }, + ), + ], + critics=[ + BinaryCritic(critic_field="usernames", weight=1.0), + ], + ) + + suite.add_case( + name="get messages in direct conversation by username (on a specific date)", + user_message="get the messages I exchanged together with the usernames john, ryan, and jennifer on 2025-01-31", + expected_tool_calls=[ + ExpectedToolCall( + func=get_messages_in_multi_person_dm_conversation_by_usernames, + args={ + "usernames": ["john", "ryan", "jennifer"], + "oldest_datetime": "2025-01-31 00:00:00", + "latest_datetime": "2025-01-31 23:59:59", + }, + ), + ], + critics=[ + BinaryCritic(critic_field="usernames", weight=1 / 3), + DatetimeCritic( + critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) + ), + DatetimeCritic( + critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) + ), + ], + ) + + suite.add_case( + name="Get conversation history oldest relative by username (2 days ago)", + user_message="Get the messages I exchanged together with the usernames john, ryan, and jennifer starting 2 days ago", + expected_tool_calls=[ + ExpectedToolCall( + func=get_messages_in_multi_person_dm_conversation_by_usernames, + args={ + "usernames": ["john", "ryan", "jennifer"], + "oldest_relative": "02:00:00", + }, + ), + ], + critics=[ + BinaryCritic(critic_field="usernames", weight=0.5), + RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), + ], + ) + + return suite diff --git a/toolkits/slack/tests/test_chat.py b/toolkits/slack/tests/test_chat.py index c1fe3ca5..07f8b166 100644 --- a/toolkits/slack/tests/test_chat.py +++ b/toolkits/slack/tests/test_chat.py @@ -18,6 +18,8 @@ from arcade_slack.tools.chat import ( get_messages_in_channel_by_name, get_messages_in_conversation_by_id, get_messages_in_direct_message_conversation_by_username, + get_messages_in_multi_person_dm_conversation_by_usernames, + get_multi_person_dm_conversation_metadata_by_usernames, list_conversations_metadata, list_direct_message_conversations_metadata, list_group_direct_message_conversations_metadata, @@ -818,3 +820,135 @@ async def test_get_messages_in_direct_conversation_by_username_not_found( await get_messages_in_direct_message_conversation_by_username( context=mock_context, username="user2" ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.retrieve_conversations_by_user_ids") +async def test_get_multi_person_direct_message_conversation_metadata_by_username( + mock_retrieve_conversations_by_user_ids, + mock_context, + mock_chat_slack_client, + mock_users_slack_client, +): + mock_chat_slack_client.auth_test.return_value = { + "ok": True, + "user_id": "U1", + "team_id": "T1", + "user": "user1", + } + + mock_users_slack_client.users_list.return_value = { + "ok": True, + "members": [ + {"id": "U1", "name": "user1"}, + {"id": "U2", "name": "user2"}, + {"id": "U3", "name": "user3"}, + {"id": "U4", "name": "user4"}, + {"id": "U5", "name": "user5"}, + ], + "response_metadata": {"next_cursor": None}, + } + + conversation = { + "id": "C12345", + "type": ConversationTypeSlackName.MPIM.value, + "is_mpim": True, + "members": ["U1", "U4", "U5"], + } + + mock_retrieve_conversations_by_user_ids.return_value = [conversation] + + response = await get_multi_person_dm_conversation_metadata_by_usernames( + context=mock_context, usernames=["user1", "user4", "user5"] + ) + + assert response == conversation + mock_retrieve_conversations_by_user_ids.assert_called_once_with( + list_conversations_func=list_conversations_metadata, + get_members_in_conversation_func=get_members_in_conversation_by_id, + context=mock_context, + conversation_types=[ConversationType.MULTI_PERSON_DIRECT_MESSAGE], + user_ids=["U1", "U4", "U5"], + exact_match=True, + limit=1, + next_cursor=None, + ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.retrieve_conversations_by_user_ids") +async def test_get_multi_person_direct_message_conversation_metadata_by_username_username_not_found( + mock_retrieve_conversations_by_user_ids, + mock_context, + mock_chat_slack_client, + mock_users_slack_client, +): + mock_chat_slack_client.users_identity.return_value = { + "ok": True, + "user": {"id": "U1", "name": "user1"}, + "team": {"id": "T1", "name": "team1"}, + } + + mock_users_slack_client.users_list.return_value = { + "ok": True, + "members": [ + {"id": "U1", "name": "user1"}, + {"id": "U2", "name": "user2"}, + ], + "response_metadata": {"next_cursor": None}, + } + + mock_retrieve_conversations_by_user_ids.side_effect = TimeoutError() + + with pytest.raises(RetryableToolError): + await get_multi_person_dm_conversation_metadata_by_usernames( + context=mock_context, usernames=["user999", "user1", "user2"] + ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.get_messages_in_conversation_by_id") +@patch("arcade_slack.tools.chat.get_multi_person_dm_conversation_metadata_by_usernames") +async def test_get_messages_in_multi_person_dm_conversation_by_usernames( + mock_get_multi_person_dm_conversation_metadata_by_usernames, + mock_get_messages_in_conversation_by_id, + mock_context, +): + mock_get_multi_person_dm_conversation_metadata_by_usernames.return_value = { + "id": "C12345", + } + + response = await get_messages_in_multi_person_dm_conversation_by_usernames( + context=mock_context, usernames=["user1", "user4", "user5"] + ) + + assert response == mock_get_messages_in_conversation_by_id.return_value + + mock_get_multi_person_dm_conversation_metadata_by_usernames.assert_called_once_with( + context=mock_context, usernames=["user1", "user4", "user5"] + ) + + mock_get_messages_in_conversation_by_id.assert_called_once_with( + context=mock_context, + conversation_id="C12345", + oldest_relative=None, + latest_relative=None, + oldest_datetime=None, + latest_datetime=None, + limit=None, + next_cursor=None, + ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.get_multi_person_dm_conversation_metadata_by_usernames") +async def test_get_messages_in_multi_person_dm_conversation_by_usernames_not_found( + mock_get_multi_person_dm_conversation_metadata_by_usernames, + mock_context, +): + mock_get_multi_person_dm_conversation_metadata_by_usernames.return_value = None + + with pytest.raises(ToolExecutionError): + await get_messages_in_direct_message_conversation_by_username( + context=mock_context, username="user2" + )