From a00bd4734e0b4036778f3012bd9e0c78386bc1ae Mon Sep 17 00:00:00 2001 From: Renato Byrro Date: Mon, 10 Feb 2025 09:01:04 -0300 Subject: [PATCH] Tool to retrieve Slack messages from a DM conversation with a given username (#241) Currently, retrieving DMs with a given username requires several actions: first get the current user's ID; list all users and find the ID of the username; then scan all DM conversations and find the one with the current user's ID and the username's ID, to finally retrieve the messages using that conversation ID. This tool abstracts all that in a single call. PS: we'll implement a similar tool for multi-person DM conversations in a subsequent PR. --- toolkits/slack/arcade_slack/exceptions.py | 15 + toolkits/slack/arcade_slack/tools/chat.py | 209 ++++- toolkits/slack/arcade_slack/utils.py | 105 ++- toolkits/slack/conftest.py | 12 + toolkits/slack/evals/eval_chat.py | 133 ++- toolkits/slack/evals/eval_slack_chat.py | 1020 --------------------- toolkits/slack/tests/test_chat.py | 306 +++++-- toolkits/slack/tests/test_utils.py | 501 +++++++++- 8 files changed, 1128 insertions(+), 1173 deletions(-) delete mode 100644 toolkits/slack/evals/eval_slack_chat.py diff --git a/toolkits/slack/arcade_slack/exceptions.py b/toolkits/slack/arcade_slack/exceptions.py index 106ca01e..49e40a75 100644 --- a/toolkits/slack/arcade_slack/exceptions.py +++ b/toolkits/slack/arcade_slack/exceptions.py @@ -12,3 +12,18 @@ class PaginationTimeoutError(SlackToolkitError): class ItemNotFoundError(SlackToolkitError): """Raised when an item is not found.""" + + +class UsernameNotFoundError(SlackToolkitError): + """Raised when a user is not found by the username searched""" + + def __init__(self, usernames_found: list[str]) -> None: + self.usernames_found = usernames_found + + +class ConversationNotFoundError(SlackToolkitError): + """Raised when a conversation is not found""" + + +class DirectMessageConversationNotFoundError(ConversationNotFoundError): + """Raised when a direct message conversation searched is not found""" diff --git a/toolkits/slack/arcade_slack/tools/chat.py b/toolkits/slack/arcade_slack/tools/chat.py index 3c826392..9daf5e11 100644 --- a/toolkits/slack/arcade_slack/tools/chat.py +++ b/toolkits/slack/arcade_slack/tools/chat.py @@ -9,9 +9,15 @@ from slack_sdk.errors import SlackApiError from slack_sdk.web.async_client import AsyncWebClient from arcade_slack.constants import MAX_PAGINATION_TIMEOUT_SECONDS -from arcade_slack.exceptions import ItemNotFoundError -from arcade_slack.models import ConversationType, SlackUserList -from arcade_slack.tools.users import get_user_info_by_id +from arcade_slack.exceptions import ( + ItemNotFoundError, + UsernameNotFoundError, +) +from arcade_slack.models import ( + ConversationType, + SlackUserList, +) +from arcade_slack.tools.users import get_user_info_by_id, list_users from arcade_slack.utils import ( async_paginate, convert_conversation_type_to_slack_name, @@ -21,6 +27,8 @@ from arcade_slack.utils import ( extract_conversation_metadata, format_conversations_as_csv, format_users, + get_user_by_username, + retrieve_conversations_by_user_ids, ) @@ -218,20 +226,18 @@ async def get_members_in_conversation_by_id( ], ) ) -async def get_members_in_conversation_by_name( +async def get_members_in_channel_by_name( context: ToolContext, - conversation_name: Annotated[str, "The name of the conversation to get members for"], + channel_name: Annotated[str, "The name of the channel to get members for"], limit: Annotated[Optional[int], "The maximum number of members to return."] = None, next_cursor: Annotated[Optional[str], "The cursor to use for pagination."] = None, -) -> Annotated[dict, "The conversation members' IDs and Names"]: +) -> Annotated[dict, "The channel members' IDs and Names"]: """Get the members of a conversation in Slack by the conversation's name.""" - conversation_metadata = await get_conversation_metadata_by_name( - context=context, conversation_name=conversation_name, next_cursor=next_cursor - ) + channel = await get_channel_metadata_by_name(context=context, channel_name=channel_name) return await get_members_in_conversation_by_id( # type: ignore[no-any-return] context=context, - conversation_id=conversation_metadata["id"], + conversation_id=channel["id"], limit=limit, next_cursor=next_cursor, ) @@ -433,12 +439,84 @@ async def get_messages_in_channel_by_name( 'latest_relative'. Leave all arguments with the default None to get messages without date/time filtering""" - conversation_metadata = await get_conversation_metadata_by_name( - context=context, conversation_name=channel_name - ) + channel = await get_channel_metadata_by_name(context=context, channel_name=channel_name) + return await get_messages_in_conversation_by_id( # type: ignore[no-any-return] context=context, - conversation_id=conversation_metadata["id"], + conversation_id=channel["id"], + oldest_relative=oldest_relative, + latest_relative=latest_relative, + oldest_datetime=oldest_datetime, + latest_datetime=latest_datetime, + limit=limit, + next_cursor=next_cursor, + ) + + +@tool(requires_auth=Slack(scopes=["im:history", "im:read"])) +async def get_messages_in_direct_message_conversation_by_username( + context: ToolContext, + username: Annotated[str, "The username of the user to get messages from"], + oldest_relative: Annotated[ + Optional[str], + ( + "The oldest message to include in the results, specified as a time offset from the " + "current time in the format 'DD:HH:MM'" + ), + ] = None, + latest_relative: Annotated[ + Optional[str], + ( + "The latest message to include in the results, specified as a time offset from the " + "current time in the format 'DD:HH:MM'" + ), + ] = None, + oldest_datetime: Annotated[ + Optional[str], + ( + "The oldest message to include in the results, specified as a datetime object in the " + "format 'YYYY-MM-DD HH:MM:SS'" + ), + ] = None, + latest_datetime: Annotated[ + Optional[str], + ( + "The latest message to include in the results, specified as a datetime object in the " + "format 'YYYY-MM-DD HH:MM:SS'" + ), + ] = None, + limit: Annotated[Optional[int], "The maximum number of messages to return."] = None, + next_cursor: Annotated[Optional[str], "The cursor to use for pagination."] = None, +) -> Annotated[ + dict, + ( + "The messages in a direct message conversation and next cursor for paginating results " + "(when there are additional messages to retrieve)." + ), +]: + """Get the messages in a direct conversation by the user's name. + + To filter messages by an absolute datetime, use 'oldest_datetime' and/or 'latest_datetime'. If + only 'oldest_datetime' is provided, it will return messages from the oldest_datetime to the + current time. If only 'latest_datetime' is provided, it will return messages since the + beginning of the conversation to the latest_datetime. + + To filter messages by a relative datetime (e.g. 3 days ago, 1 hour ago, etc.), use + 'oldest_relative' and/or 'latest_relative'. If only 'oldest_relative' is provided, it will + return messages from the oldest_relative to the current time. If only 'latest_relative' is + provided, it will return messages from the current time to the latest_relative. + + Do not provide both 'oldest_datetime' and 'oldest_relative' or both 'latest_datetime' and + 'latest_relative'. + + Leave all arguments with the default None to get messages without date/time filtering""" + direct_conversation = await get_direct_message_conversation_metadata_by_username( + context=context, username=username + ) + + return await get_messages_in_conversation_by_id( # type: ignore[no-any-return] + context=context, + conversation_id=direct_conversation["id"], oldest_relative=oldest_relative, latest_relative=latest_relative, oldest_datetime=oldest_datetime, @@ -490,39 +568,33 @@ async def get_conversation_metadata_by_id( return dict(**extract_conversation_metadata(response["channel"])) -@tool( - requires_auth=Slack( - scopes=["channels:read", "groups:read", "im:read", "mpim:read"], - ) -) -async def get_conversation_metadata_by_name( +@tool(requires_auth=Slack(scopes=["channels:read"])) +async def get_channel_metadata_by_name( context: ToolContext, - conversation_name: Annotated[str, "The name of the conversation to get metadata for"], + channel_name: Annotated[str, "The name of the channel to get metadata for"], next_cursor: Annotated[ Optional[str], "The cursor to use for pagination, if continuing from a previous search.", ] = None, -) -> Annotated[dict, "The conversation metadata"]: - """Get the metadata of a conversation in Slack searching by its name.""" - conversation_names: list[str] = [] +) -> Annotated[dict, "The channel metadata"]: + """Get the metadata of a channel in Slack searching by its name.""" + channel_names: list[str] = [] - async def find_conversation() -> dict: - nonlocal conversation_names, conversation_name, next_cursor + async def find_channel() -> dict: + nonlocal channel_names, channel_name, next_cursor should_continue = True while should_continue: response = await list_conversations_metadata(context, next_cursor=next_cursor) - next_cursor = response["next_cursor"] + next_cursor = response.get("next_cursor") - for conversation in response["conversations"]: - response_conversation_name = ( - "" - if not isinstance(conversation.get("name"), str) - else conversation["name"].lower() + for channel in response["conversations"]: + response_channel_name = ( + "" if not isinstance(channel.get("name"), str) else channel["name"].lower() ) - if response_conversation_name == conversation_name.lower(): - return conversation # type: ignore[no-any-return] - conversation_names.append(conversation["name"]) + if response_channel_name == channel_name.lower(): + return channel # type: ignore[no-any-return] + channel_names.append(channel["name"]) if not next_cursor: should_continue = False @@ -530,31 +602,80 @@ async def get_conversation_metadata_by_name( raise ItemNotFoundError() try: - return await asyncio.wait_for(find_conversation(), timeout=MAX_PAGINATION_TIMEOUT_SECONDS) + return await asyncio.wait_for(find_channel(), timeout=MAX_PAGINATION_TIMEOUT_SECONDS) except ItemNotFoundError: raise RetryableToolError( - "Conversation not found", - developer_message=f"Conversation with name '{conversation_name}' not found.", - additional_prompt_content=f"Available conversation names: {conversation_names}", + "Channel not found", + developer_message=f"Channel with name '{channel_name}' not found.", + additional_prompt_content=f"Available channel names: {channel_names}", retry_after_ms=500, ) except TimeoutError: raise RetryableToolError( - "Conversation not found, search timed out.", + "Channel not found, search timed out.", developer_message=( - f"Conversation with name '{conversation_name}' not found. " + f"Channel with name '{channel_name}' not found. " f"Search timed out after {MAX_PAGINATION_TIMEOUT_SECONDS} seconds." ), additional_prompt_content=( - f"Other conversation names found are: {conversation_names}. " + f"Other channel names found are: {channel_names}. " "The list is potentially non-exhaustive, since the search process timed out. " - f"Use the '{list_conversations_metadata.__name__}' tool to get a comprehensive " - "list of conversations." + f"Use the '{list_conversations_metadata.__tool_name__}' tool to get" + "a comprehensive list of channels." ), retry_after_ms=500, ) +@tool(requires_auth=Slack(scopes=["im:read"])) +async def get_direct_message_conversation_metadata_by_username( + context: ToolContext, + username: Annotated[str, "The username of the user/person to get messages with"], + next_cursor: Annotated[ + Optional[str], + "The cursor to use for pagination, if continuing from a previous search.", + ] = None, +) -> Annotated[ + Optional[dict], + "The direct message conversation metadata.", +]: + """Get the metadata of a direct message conversation in Slack by the username.""" + try: + token = ( + context.authorization.token + if context.authorization and context.authorization.token + else "" + ) + slack_client = AsyncWebClient(token=token) + + current_user, list_users_response = await asyncio.gather( + slack_client.auth_test(), list_users(context) + ) + + other_user = get_user_by_username(username, list_users_response["users"]) + + conversations_found = await retrieve_conversations_by_user_ids( + list_conversations_func=list_conversations_metadata, + get_members_in_conversation_func=get_members_in_conversation_by_id, + context=context, + conversation_types=[ConversationType.DIRECT_MESSAGE], + user_ids=[current_user["user_id"], other_user["id"]], + exact_match=True, + limit=1, + next_cursor=next_cursor, + ) + + return None if not conversations_found else conversations_found[0] + + except UsernameNotFoundError as e: + raise RetryableToolError( + f"Username '{username}' not found", + developer_message=f"User with username '{username}' not found.", + additional_prompt_content=f"Available users: {e.usernames_found}", + retry_after_ms=500, + ) + + @tool( requires_auth=Slack( scopes=["channels:read", "groups:read", "im:read", "mpim:read"], diff --git a/toolkits/slack/arcade_slack/utils.py b/toolkits/slack/arcade_slack/utils.py index 6b82b711..159cf9bc 100644 --- a/toolkits/slack/arcade_slack/utils.py +++ b/toolkits/slack/arcade_slack/utils.py @@ -2,11 +2,15 @@ import asyncio from datetime import datetime, timezone from typing import Any, Callable, Optional +from arcade.sdk import ToolContext from arcade.sdk.errors import RetryableToolError from arcade_slack.constants import MAX_PAGINATION_SIZE_LIMIT, MAX_PAGINATION_TIMEOUT_SECONDS from arcade_slack.custom_types import SlackPaginationNextCursor -from arcade_slack.exceptions import PaginationTimeoutError +from arcade_slack.exceptions import ( + PaginationTimeoutError, + UsernameNotFoundError, +) from arcade_slack.models import ( BasicUserInfo, ConversationMetadata, @@ -90,6 +94,18 @@ def get_slack_conversation_type_as_str(channel: SlackConversation) -> str: raise ValueError(f"Invalid conversation type in channel {channel.get('name')}") +def get_user_by_username(username: str, users_list: list[dict]) -> SlackUser: + usernames_found = [] + for user in users_list: + if isinstance(user.get("name"), str): + usernames_found.append(user["name"]) + username_found = user.get("name") or "" + if username.lower() == username_found.lower(): + return SlackUser(**user) + + raise UsernameNotFoundError(usernames_found) + + def convert_conversation_type_to_slack_name( conversation_type: ConversationType, ) -> ConversationTypeSlackName: @@ -174,6 +190,93 @@ def extract_basic_user_info(user_info: SlackUser) -> BasicUserInfo: ) +async def associate_members_of_multiple_conversations( + get_members_in_conversation_func: Callable, + conversations: list[dict], + context: ToolContext, +) -> list[dict]: + """Associate members to each conversation, returning the updated list.""" + return await asyncio.gather(*[ + associate_members_of_conversation(get_members_in_conversation_func, context, conv) + for conv in conversations + ]) + + +async def associate_members_of_conversation( + get_members_in_conversation_func: Callable, + context: ToolContext, + conversation: dict, +) -> dict: + response = await get_members_in_conversation_func(context, conversation["id"]) + conversation["members"] = response["members"] + return conversation + + +async def retrieve_conversations_by_user_ids( + list_conversations_func: Callable, + get_members_in_conversation_func: Callable, + context: ToolContext, + conversation_types: list[ConversationType], + user_ids: list[str], + exact_match: bool = False, + limit: Optional[int] = None, + next_cursor: Optional[str] = None, +) -> list[dict]: + """ + Retrieve conversations filtered by the given user IDs. Includes pagination support + and optionally limits the number of returned conversations. + """ + conversations_found: list[dict] = [] + + response = await list_conversations_func( + context=context, + conversation_types=conversation_types, + next_cursor=next_cursor, + ) + + # Associate members to each conversation + conversations_with_members = await associate_members_of_multiple_conversations( + get_members_in_conversation_func, response["conversations"], context + ) + + conversations_found.extend( + filter_conversations_by_user_ids(conversations_with_members, user_ids, exact_match) + ) + + return conversations_found[:limit] + + +def filter_conversations_by_user_ids( + conversations: list[dict], + user_ids: list[str], + exact_match: bool = False, +) -> list[dict]: + """ + Filter conversations by the members' user IDs. + + Args: + conversations: The list of conversations to filter. + user_ids: The user IDs to filter conversations for. + exact_match: Whether to match the exact number of members in the conversations. + + Returns: + The list of conversations found. + """ + matches = [] + for conversation in conversations: + member_ids = [member["id"] for member in conversation["members"]] + if exact_match: + same_length = len(user_ids) == len(member_ids) + has_all_members = all(user_id in member_ids for user_id in user_ids) + if same_length and has_all_members: + matches.append(conversation) + else: + if all(user_id in member_ids for user_id in user_ids): + matches.append(conversation) + + return matches + + def is_user_a_bot(user: SlackUser) -> bool: """Check if a Slack user represents a bot. diff --git a/toolkits/slack/conftest.py b/toolkits/slack/conftest.py index fb7c1c32..acda0bea 100644 --- a/toolkits/slack/conftest.py +++ b/toolkits/slack/conftest.py @@ -6,3 +6,15 @@ from arcade.sdk import ToolAuthorizationContext, ToolContext def mock_context(): mock_auth = ToolAuthorizationContext(token="fake-token") # noqa: S106 return ToolContext(authorization=mock_auth) + + +@pytest.fixture +def mock_chat_slack_client(mocker): + mock_client = mocker.patch("arcade_slack.tools.chat.AsyncWebClient", autospec=True) + return mock_client.return_value + + +@pytest.fixture +def mock_users_slack_client(mocker): + mock_client = mocker.patch("arcade_slack.tools.users.AsyncWebClient", autospec=True) + return mock_client.return_value diff --git a/toolkits/slack/evals/eval_chat.py b/toolkits/slack/evals/eval_chat.py index e83f1aeb..55b39e91 100644 --- a/toolkits/slack/evals/eval_chat.py +++ b/toolkits/slack/evals/eval_chat.py @@ -15,12 +15,14 @@ from arcade.sdk.eval import ( import arcade_slack from arcade_slack.critics import RelativeTimeBinaryCritic from arcade_slack.tools.chat import ( + get_channel_metadata_by_name, get_conversation_metadata_by_id, - get_conversation_metadata_by_name, + get_direct_message_conversation_metadata_by_username, + get_members_in_channel_by_name, get_members_in_conversation_by_id, - get_members_in_conversation_by_name, get_messages_in_channel_by_name, get_messages_in_conversation_by_id, + get_messages_in_direct_message_conversation_by_username, list_conversations_metadata, list_direct_message_conversations_metadata, list_group_direct_message_conversations_metadata, @@ -433,19 +435,19 @@ def get_conversations_metadata_eval_suite() -> EvalSuite: """Create an evaluation suite for tools getting conversations metadata.""" suite = EvalSuite( name="Slack Tools Evaluation", - system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", + system_message="You are an AI assistant that can interact with Slack to get information from conversations, users, etc.", catalog=catalog, rubric=rubric, ) suite.add_case( - name="Get conversation metadata by name", + name="Get channel metadata by name", user_message="Get the metadata of the #general channel", expected_tool_calls=[ ExpectedToolCall( - func=get_conversation_metadata_by_name, + func=get_channel_metadata_by_name, args={ - "conversation_name": "general", + "channel_name": "general", }, ), ], @@ -470,6 +472,26 @@ def get_conversations_metadata_eval_suite() -> EvalSuite: ], ) + get_metadata_by_username_user_messages = [ + "get the metadata of the direct message conversation with the user 'jane.doe'" + "get data about my private conversation with the user 'jane.doe'", + "get data about my IM conversation with the 'jane.doe'", + ] + + for i, user_message in enumerate(get_metadata_by_username_user_messages): + suite.add_case( + name=f"Get direct message conversation metadata by username {i}", + user_message=user_message, + expected_tool_calls=[ + ExpectedToolCall( + func=get_direct_message_conversation_metadata_by_username, + args={ + "username": "jane.doe", + }, + ), + ], + ) + return suite @@ -496,13 +518,13 @@ def get_conversations_members_eval_suite() -> EvalSuite: for user_message in user_messages: suite.add_case( - name=f"Get conversation members by name: {user_message}", + name=f"Get channel members by name: {user_message}", user_message=user_message, expected_tool_calls=[ ExpectedToolCall( - func=get_members_in_conversation_by_name, + func=get_members_in_channel_by_name, args={ - "conversation_name": "general", + "channel_name": "general", }, ), ], @@ -531,8 +553,8 @@ def get_conversations_members_eval_suite() -> EvalSuite: @tool_eval() -def get_conversation_history_eval_suite() -> EvalSuite: - """Create an evaluation suite for tools getting conversations history.""" +def get_messages_in_channel_eval_suite() -> EvalSuite: + """Create an evaluation suite for tools getting messages in channels.""" suite = EvalSuite( name="Slack Chat Tools Evaluation", system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", @@ -540,16 +562,16 @@ def get_conversation_history_eval_suite() -> EvalSuite: rubric=rubric, ) - no_arguments_user_messages_by_conversation_name = [ - "Get the history of the #general channel", - "Get the history of the general channel", + no_arguments_user_messages_by_channel_name = [ + "what are the latest messages in the #general channel", + "show me the messages in the general channel", "list the messages in the #general channel", "list the messages in the general channel", ] - for user_message in no_arguments_user_messages_by_conversation_name: + for i, user_message in enumerate(no_arguments_user_messages_by_channel_name): suite.add_case( - name=f"Get conversation history by name: '{user_message}'", + name=f"Get messages in conversation by name {i}: '{user_message}'", user_message=user_message, expected_tool_calls=[ ExpectedToolCall( @@ -985,3 +1007,82 @@ def get_conversation_history_eval_suite() -> EvalSuite: ) return suite + + +@tool_eval() +def get_messages_in_direct_message_eval_suite() -> EvalSuite: + """Create an evaluation suite for tools getting messages in direct messages.""" + suite = EvalSuite( + name="Slack Chat Tools Evaluation", + system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", + catalog=catalog, + rubric=rubric, + ) + + no_arguments_user_messages_by_username = [ + "what are the latest messages I exchanged with jane.doe", + "show my messages with jane.doe on Slack", + "list the messages I exchanged with jane.doe", + "list the message history with jane.doe", + ] + + for i, user_message in enumerate(no_arguments_user_messages_by_username): + suite.add_case( + name=f"{user_message} [{i}]", + user_message=user_message, + expected_tool_calls=[ + ExpectedToolCall( + func=get_messages_in_direct_message_conversation_by_username, + args={ + "username": "jane.doe", + }, + ), + ], + critics=[ + BinaryCritic(critic_field="username", weight=1.0), + ], + ) + + suite.add_case( + name="get messages in direct conversation by username (on a specific date)", + user_message="get the messages I exchanged with jane.doe on 2025-01-31", + expected_tool_calls=[ + ExpectedToolCall( + func=get_messages_in_direct_message_conversation_by_username, + args={ + "username": "jane.doe", + "oldest_datetime": "2025-01-31 00:00:00", + "latest_datetime": "2025-01-31 23:59:59", + }, + ), + ], + critics=[ + BinaryCritic(critic_field="username", weight=1 / 3), + DatetimeCritic( + critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) + ), + DatetimeCritic( + critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) + ), + ], + ) + + suite.add_case( + name="Get conversation history oldest relative by username (2 days ago)", + user_message="Get the messages I exchanged with jane.doe starting 2 days ago", + expected_tool_calls=[ + ExpectedToolCall( + func=get_messages_in_direct_message_conversation_by_username, + args={ + "username": "jane.doe", + "oldest_relative": "02:00:00", + }, + ), + ], + critics=[ + BinaryCritic(critic_field="username", weight=0.5), + RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), + ], + ) + + return suite diff --git a/toolkits/slack/evals/eval_slack_chat.py b/toolkits/slack/evals/eval_slack_chat.py deleted file mode 100644 index 0cfc7a65..00000000 --- a/toolkits/slack/evals/eval_slack_chat.py +++ /dev/null @@ -1,1020 +0,0 @@ -import json -from datetime import timedelta -from typing import Any - -from arcade.sdk import ToolCatalog -from arcade.sdk.eval import ( - BinaryCritic, - DatetimeCritic, - EvalRubric, - EvalSuite, - ExpectedToolCall, - SimilarityCritic, - tool_eval, -) - -import arcade_slack -from arcade_slack.tools.chat import ( - get_conversation_metadata_by_id, - get_conversation_metadata_by_name, - get_members_in_conversation_by_id, - get_members_in_conversation_by_name, - get_messages_in_channel_by_name, - get_messages_in_conversation_by_id, - list_conversations_metadata, - list_direct_message_conversations_metadata, - list_group_direct_message_conversations_metadata, - list_private_channels_metadata, - list_public_channels_metadata, - send_dm_to_user, - send_message_to_channel, -) - - -class RelativeTimeBinaryCritic(BinaryCritic): - def evaluate(self, expected: Any, actual: Any) -> dict[str, float | bool]: - """ - Evaluates whether the expected and actual relative time strings are equivalent after casting. - - Args: - expected: The expected value. - actual: The actual value to compare, cast to the type of expected. - - Returns: - dict: A dictionary containing the match status and score. - """ - # Cast actual to the type of expected - try: - actual_casted = self.cast_actual(expected, actual) - # TODO log or something better here - except TypeError: - actual_casted = actual - - expected_parts = tuple(map(int, expected.split(":"))) - actual_parts = tuple(map(int, actual_casted.split(":"))) - - if len(expected_parts) != 3 or len(actual_parts) != 3: - return {"match": False, "score": 0.0} - - exp_days, exp_hours, exp_minutes = expected_parts - act_days, act_hours, act_minutes = actual_parts - - match = exp_days == act_days and exp_hours == act_hours and exp_minutes == act_minutes - return {"match": match, "score": self.weight if match else 0.0} - - -# Evaluation rubric -rubric = EvalRubric( - fail_threshold=0.8, - warn_threshold=0.9, -) - - -catalog = ToolCatalog() -# Register the Slack tools -catalog.add_module(arcade_slack) - - -@tool_eval() -def send_message_eval_suite() -> EvalSuite: - """Create an evaluation suite for Slack messaging tools.""" - suite = EvalSuite( - name="Slack Messaging Tools Evaluation", - system_message="You are an AI assistant that can send direct messages and post messages to channels in Slack using the provided tools.", - catalog=catalog, - rubric=rubric, - ) - - # Send DM to User Scenarios - suite.add_case( - name="Send DM to user with clear username", - user_message="Send a direct message to johndoe saying 'Hello, can we meet at 3 PM?'", - expected_tool_calls=[ - ExpectedToolCall( - func=send_dm_to_user, - args={ - "user_name": "johndoe", - "message": "Hello, can we meet at 3 PM?", - }, - ) - ], - critics=[ - BinaryCritic(critic_field="user_name", weight=0.5), - SimilarityCritic(critic_field="message", weight=0.5, similarity_threshold=0.9), - ], - ) - - suite.add_case( - name="Send DM with ambiguous username", - user_message="ask him for an update on the project", - expected_tool_calls=[ - ExpectedToolCall( - func=send_dm_to_user, - args={ - "user_name": "john", - "message": "Hi John, could you please provide an update on the Acme project?", - }, - ) - ], - critics=[ - BinaryCritic(critic_field="user_name", weight=0.75), - SimilarityCritic(critic_field="message", weight=0.25, similarity_threshold=0.6), - ], - additional_messages=[ - {"role": "user", "content": "Message John about the Acme project deadline"}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "Slack_ListUsers", - "arguments": '{"exclude_bots":true}', - }, - } - ], - }, - { - "role": "tool", - "content": json.dumps({ - "users": [ - { - "display_name": "john", - "email": "john@randomtech.com", - "id": "abc123", - "is_bot": False, - "name": "john", - "real_name": "John Doe", - "timezone": "America/Los_Angeles", - }, - { - "display_name": "jack", - "email": "jack@randomtech.com", - "id": "def456", - "is_bot": False, - "name": "jack", - "real_name": "Jack Doe", - "timezone": "America/Los_Angeles", - }, - ] - }), - "tool_call_id": "call_1", - "name": "Slack_ListUsers", - }, - { - "role": "assistant", - "content": "What would you like to include in the message to John about the Acme project deadline?", - }, - ], - ) - - suite.add_case( - name="Send DM with username in different format", - user_message="yes, send it", - expected_tool_calls=[ - ExpectedToolCall( - func=send_dm_to_user, - args={ - "user_name": "jane.doe", - "message": "Hi Jane, I need to reschedule our meeting. When are you available?", - }, - ) - ], - critics=[ - BinaryCritic(critic_field="user_name", weight=0.75), - SimilarityCritic(critic_field="message", weight=0.25, similarity_threshold=0.6), - ], - additional_messages=[ - {"role": "user", "content": "Message Jane.Doe asking to reschedule our meeting"}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "Slack_ListUsers", - "arguments": '{"exclude_bots":true}', - }, - } - ], - }, - { - "role": "tool", - "content": json.dumps({ - "users": [ - { - "display_name": "jane.doe", - "email": "jane@randomtech.com", - "id": "abc123", - "is_bot": False, - "name": "jane.doe", - "real_name": "Jane Doe", - "timezone": "America/Los_Angeles", - }, - { - "display_name": "jack", - "email": "jack@randomtech.com", - "id": "def456", - "is_bot": False, - "name": "jack", - "real_name": "Jack Doe", - "timezone": "America/Los_Angeles", - }, - ] - }), - "tool_call_id": "call_1", - "name": "Slack_ListUsers", - }, - { - "role": "assistant", - "content": "I found a user with the name 'jane.doe'. Would you like to send a message to them?", - }, - ], - ) - - # Send Message to Channel Scenarios - suite.add_case( - name="Send message to channel with clear name", - user_message="Post 'The new feature is now live!' in the #announcements channel", - expected_tool_calls=[ - ExpectedToolCall( - func=send_message_to_channel, - args={ - "channel_name": "announcements", - "message": "The new feature is now live!", - }, - ) - ], - critics=[ - BinaryCritic(critic_field="channel_name", weight=0.5), - SimilarityCritic(critic_field="message", weight=0.5), - ], - ) - - suite.add_case( - name="Send message to channel with ambiguous name", - user_message="Inform the team in the general channel about the upcoming maintenance", - expected_tool_calls=[ - ExpectedToolCall( - func=send_message_to_channel, - args={ - "channel_name": "general", - "message": "Attention team: There will be upcoming maintenance. Please save your work and expect some downtime.", - }, - ) - ], - critics=[ - SimilarityCritic(critic_field="channel_name", weight=0.8), - SimilarityCritic(critic_field="message", weight=0.2, similarity_threshold=0.6), - ], - ) - - # Adversarial Scenarios - suite.add_case( - name="Ambiguous between DM and channel message", - user_message="general", - expected_tool_calls=[ - ExpectedToolCall( - func=send_message_to_channel, - args={ - "channel_name": "general", - "message": "Great job on the presentation!", - }, - ) - ], - critics=[ - SimilarityCritic(critic_field="channel_name", weight=0.4), - SimilarityCritic(critic_field="message", weight=0.6), - ], - additional_messages=[ - {"role": "user", "content": "Send 'Great job on the presentation!' to the team"}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "Slack_ListPublicChannelsMetadata", - "arguments": '{"limit":20}', - }, - } - ], - }, - { - "role": "tool", - "content": json.dumps({ - "conversations": [ - { - "conversation_type": "public_channel", - "id": "channel1", - "is_archived": False, - "is_member": True, - "is_private": False, - "name": "random", - "num_members": 999, - "purpose": "Random stuff", - }, - { - "conversation_type": "public_channel", - "id": "channel2", - "is_archived": False, - "is_member": True, - "is_private": False, - "name": "general", - "num_members": 999, - "purpose": "Just a general channel", - }, - ], - "next_cursor": "", - }), - "tool_call_id": "call_1", - "name": "Slack_ListPublicChannelsMetadata", - }, - { - "role": "assistant", - "content": 'To send the message "Great job on the presentation!" to the team, please let me know which Slack channel you\'d like to use:\n\n1. #random\n2. #general\n\nPlease let me know your choice!', - }, - ], - ) - - # Multiple recipients in DM request - suite.add_case( - name="Multiple recipients in DM request", - user_message="Send DMs to the users 'alice' and 'bob' about pushing the meeting tomorrow. I have to much work to do.", - expected_tool_calls=[ - ExpectedToolCall( - func=send_dm_to_user, - args={ - "user_name": "alice", - "message": "Hi Alice, about our meeting tomorrow, let's reschedule? I am swamped with work.", - }, - ), - ExpectedToolCall( - func=send_dm_to_user, - args={ - "user_name": "bob", - "message": "Hi Bob, about our meeting tomorrow, let's reschedule? I am swamped with work.", - }, - ), - ], - critics=[ - SimilarityCritic(critic_field="user_name", weight=0.75), - SimilarityCritic(critic_field="message", weight=0.25, similarity_threshold=0.5), - ], - ) - - suite.add_case( - name="Channel name similar to username", - user_message="Post 'sounds great!' in john-project channel", - expected_tool_calls=[ - ExpectedToolCall( - func=send_message_to_channel, - args={ - "channel_name": "john-project", - "message": "Sounds great!", - }, - ) - ], - critics=[ - BinaryCritic(critic_field="channel_name", weight=0.5), - SimilarityCritic(critic_field="message", weight=0.5), - ], - ) - - return suite - - -@tool_eval() -def list_conversations_eval_suite() -> EvalSuite: - """Create an evaluation suite for tools listing conversations.""" - suite = EvalSuite( - name="Slack Messaging Tools Evaluation", - system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", - catalog=catalog, - rubric=rubric, - ) - - cases = [ - ( - "List my conversations", - "List all conversations I am a member of", - list_conversations_metadata, - ), - ( - "List public channels", - "List all public channels", - list_public_channels_metadata, - ), - ( - "List private channels", - "List all private channels", - list_private_channels_metadata, - ), - ( - "List group direct message channels", - "List all group direct message channels", - list_group_direct_message_conversations_metadata, - ), - ( - "List individual direct message channels", - "List all individual direct message channels", - list_direct_message_conversations_metadata, - ), - ( - "List direct message channels", - "List all direct message channels", - list_direct_message_conversations_metadata, - ), - ( - "List public and private channels", - "List public and private channels I am a member of", - list_public_channels_metadata, - list_private_channels_metadata, - ), - ( - "List public channels and direct message conversations", - "List public channels and direct message conversations I am a member of", - list_public_channels_metadata, - list_direct_message_conversations_metadata, - ), - ] - - for name, user_message, *expect_called_tool_functions in cases: - suite.add_case( - name=name, - user_message=user_message, - expected_tool_calls=[ - ExpectedToolCall( - func=tool_function, - args={}, - ) - for tool_function in expect_called_tool_functions - ], - ) - - return suite - - -@tool_eval() -def get_conversations_metadata_eval_suite() -> EvalSuite: - """Create an evaluation suite for tools getting conversations metadata.""" - suite = EvalSuite( - name="Slack Tools Evaluation", - system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", - catalog=catalog, - rubric=rubric, - ) - - suite.add_case( - name="Get conversation metadata by name", - user_message="Get the metadata of the #general channel", - expected_tool_calls=[ - ExpectedToolCall( - func=get_conversation_metadata_by_name, - args={ - "conversation_name": "general", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1.0), - ], - ) - - suite.add_case( - name="Get conversation metadata by id", - user_message="Get the metadata of the conversation with id '1234567890'", - expected_tool_calls=[ - ExpectedToolCall( - func=get_conversation_metadata_by_id, - args={ - "conversation_id": "1234567890", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1.0), - ], - ) - - return suite - - -@tool_eval() -def get_conversations_members_eval_suite() -> EvalSuite: - """Create an evaluation suite for tools getting conversations members.""" - suite = EvalSuite( - name="Slack Tools Evaluation", - system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", - catalog=catalog, - rubric=rubric, - ) - - user_messages = [ - "Get the members of the #general channel", - "Get the members of the general channel", - "Get a list of people in the #general channel", - "Get a list of people in the general channel", - "Show me who's in the #general channel", - "Show me who's in the general channel", - "Who is in the #general channel?", - "Who is in the general channel?", - ] - - for user_message in user_messages: - suite.add_case( - name=f"Get conversation members by name: {user_message}", - user_message=user_message, - expected_tool_calls=[ - ExpectedToolCall( - func=get_members_in_conversation_by_name, - args={ - "conversation_name": "general", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1.0), - ], - ) - - suite.add_case( - name="Get conversation members by id", - user_message="Get the members of the conversation with id '1234567890'", - expected_tool_calls=[ - ExpectedToolCall( - func=get_members_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1.0), - ], - ) - - return suite - - -@tool_eval() -def get_conversation_history_eval_suite() -> EvalSuite: - """Create an evaluation suite for tools getting conversations history.""" - suite = EvalSuite( - name="Slack Tools Evaluation", - system_message="You are an AI assistant that can interact with Slack to send messages and get information from conversations, users, etc.", - catalog=catalog, - rubric=rubric, - ) - - no_arguments_user_messages_by_conversation_name = [ - "Get the history of the #general channel", - "Get the history of the general channel", - "list the messages in the #general channel", - "list the messages in the general channel", - ] - - for user_message in no_arguments_user_messages_by_conversation_name: - suite.add_case( - name=f"Get conversation history by name: '{user_message}'", - user_message=user_message, - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1.0), - ], - ) - - no_arguments_user_messages_by_conversation_id = [ - "Get the history of the conversation with id '1234567890'", - "Get the history of the conversation with id '1234567890'", - "list the messages in the conversation with id '1234567890'", - "list the messages in the conversation with id '1234567890'", - ] - - for user_message in no_arguments_user_messages_by_conversation_id: - suite.add_case( - name=f"Get conversation history by id: '{user_message}'", - user_message=user_message, - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1.0), - ], - ) - - suite.add_case( - name="Get conversation history with limit by name", - user_message="Get the last 10 messages in the #general channel", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "limit": 10, - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1.0), - ], - ) - - suite.add_case( - name="Get conversation history with limit by id", - user_message="Get the last 25 messages in the conversation with id '1234567890'", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "limit": 25, - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1.0), - ], - ) - - # Relative time eval cases by id - - suite.add_case( - name="Get conversation history oldest relative by id (2 days ago)", - user_message="Get the messages in the conversation with id '1234567890' starting 2 days ago", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_relative": "02:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - suite.add_case( - name="Get conversation history oldest and latest relative by id", - user_message="Get the messages in the conversation with id '1234567890' from 2 days ago to 3 hours ago", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_relative": "02:00:00", - "latest_relative": "00:03:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1 / 3), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=1 / 3), - RelativeTimeBinaryCritic(critic_field="latest_relative", weight=1 / 3), - ], - ) - - suite.add_case( - name="Get conversation history oldest relative by id (1 week ago)", - user_message="Get the messages in the conversation with id '1234567890' starting 1 week ago", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_relative": "07:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - suite.add_case( - name="Get conversation history oldest relative by id (yesterday)", - user_message="Get the messages in the conversation with id '1234567890' from yesterday", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_relative": "01:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - # Relative time eval cases by name - - suite.add_case( - name="Get conversation history oldest relative by name (2 days ago)", - user_message="Get the messages in the #general channel starting 2 days ago", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_relative": "02:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - suite.add_case( - name="Get conversation history oldest and latest relative by name", - user_message="Get the messages in the #general channel from 2 days ago to 3 hours ago", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_relative": "02:00:00", - "latest_relative": "00:03:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1 / 3), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=1 / 3), - RelativeTimeBinaryCritic(critic_field="latest_relative", weight=1 / 3), - ], - ) - - suite.add_case( - name="Get conversation history oldest relative by name (yesterday)", - user_message="Get the messages in the #general channel from yesterday", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_relative": "01:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - suite.add_case( - name="Get conversation history oldest relative by name (last week)", - user_message="Get the messages in the #general channel from last week", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_relative": "07:00:00", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=0.5), - RelativeTimeBinaryCritic(critic_field="oldest_relative", weight=0.5), - ], - ) - - # Absolute time eval cases by id - - suite.add_case( - name="Get conversation history oldest absolute by id (on a specific date)", - user_message="Get the messages in the conversation with id '1234567890' from 2025-01-20", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_datetime": "2025-01-20 00:00:00", - "latest_datetime": "2025-01-20 23:59:59", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1 / 3), - DatetimeCritic( - critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) - ), - DatetimeCritic( - critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) - ), - ], - ) - - suite.add_case( - name="Get conversation history oldest absolute by id (between a date range)", - user_message="Get the messages in the conversation with id '1234567890' from 2025-01-20 to 2025-01-25", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_conversation_by_id, - args={ - "conversation_id": "1234567890", - "oldest_datetime": "2025-01-20 00:00:00", - "latest_datetime": "2025-01-25 23:59:59", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_id", weight=1 / 3), - DatetimeCritic( - critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) - ), - DatetimeCritic( - critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(minutes=2) - ), - ], - ) - - suite.add_case( - name="Get conversation history oldest absolute by name (on a specific date)", - user_message="Get the messages in the #general channel from 2025-01-20", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_datetime": "2025-01-20 00:00:00", - "latest_datetime": "2025-01-20 23:59:59", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1 / 3), - # We use a timedelta of 10 seconds because sometimes the LLM will select the limit - # date at 23:59:59, other times it'll select the next day at 00:00:00. - DatetimeCritic( - critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(seconds=10) - ), - DatetimeCritic( - critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(seconds=10) - ), - ], - ) - - suite.add_case( - name="Get conversation history oldest absolute by name (between a date range)", - user_message="Get the messages in the #general channel from 2025-01-20 to 2025-01-25", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "oldest_datetime": "2025-01-20 00:00:00", - "latest_datetime": "2025-01-25 23:59:59", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1 / 3), - # We use a timedelta of 10 seconds because sometimes the LLM will select the limit - # date at 23:59:59, other times it'll select the next day at 00:00:00. - DatetimeCritic( - critic_field="oldest_datetime", weight=1 / 3, max_difference=timedelta(seconds=10) - ), - DatetimeCritic( - critic_field="latest_datetime", weight=1 / 3, max_difference=timedelta(seconds=10) - ), - ], - ) - - # Eval case for pagination - - suite.add_case( - name="Get conversation history with pagination", - user_message="get the next 5 messages", - expected_tool_calls=[ - ExpectedToolCall( - func=get_messages_in_channel_by_name, - args={ - "conversation_name": "general", - "limit": 5, - "cursor": "cursor_xyz", - }, - ), - ], - critics=[ - BinaryCritic(critic_field="conversation_name", weight=1 / 3), - BinaryCritic(critic_field="cursor", weight=1 / 3), - BinaryCritic(critic_field="limit", weight=1 / 3), - ], - additional_messages=[ - {"role": "user", "content": "Get the last 2 messages on the general channel"}, - { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "Slack_GetConversationHistoryByName", - "arguments": json.dumps({ - "conversation_name": "general", - "limit": 2, - }), - }, - } - ], - }, - { - "role": "tool", - "content": json.dumps({ - "messages": [ - { - "blocks": [ - { - "block_id": "abc123", - "elements": [ - { - "elements": [ - { - "text": "Almost there, Boss, need to get some evals in!", - "type": "text", - } - ], - "type": "rich_text_section", - } - ], - "type": "rich_text", - } - ], - "client_msg_id": "msg_id_0", - "datetime_timestamp": "2025-01-21 16:59:55", - "team": "617263616465207465616D20697320617420626F7373206C6576656C", - "text": "Almost there, Boss, need to get some evals in!", - "ts": "1737507595.598529", - "type": "message", - "user": "77686F2069732074686520626F73733F", - }, - { - "blocks": [ - { - "block_id": "xyz456", - "elements": [ - { - "elements": [ - { - "text": "hey, are the Slack Tools ready yet?", - "type": "text", - } - ], - "type": "rich_text_section", - } - ], - "type": "rich_text", - } - ], - "client_msg_id": "msg_id_1", - "datetime_timestamp": "2025-01-21 16:57:35", - "team": "617263616465207465616D20697320617420626F7373206C6576656C", - "text": "hey, are the Slack Tools ready yet?", - "ts": "1737507595.598529", - "type": "message", - "user": "73616D2069732074686520626F7373", - }, - ], - "next_cursor": "cursor_xyz", - }), - "tool_call_id": "call_1", - "name": "Slack_GetConversationHistoryByName", - }, - { - "role": "assistant", - "content": 'Here are the last 2 messages from the general channel:\n\n1. **User:** 77686F2069732074686520626F73733F \n **Message:** "Almost there, Boss, need to get some evals in!" \n **Timestamp:** 2025-01-21 16:59:55\n\n2. **User:** 73616D2069732074686520626F7373 \n **Message:** "hey, are the Slack Tools ready yet?" \n **Timestamp:** 2025-01-21 16:57:35', - }, - ], - ) - - return suite diff --git a/toolkits/slack/tests/test_chat.py b/toolkits/slack/tests/test_chat.py index 19111b4d..c1fe3ca5 100644 --- a/toolkits/slack/tests/test_chat.py +++ b/toolkits/slack/tests/test_chat.py @@ -10,12 +10,14 @@ from slack_sdk.web.async_slack_response import AsyncSlackResponse from arcade_slack.constants import MAX_PAGINATION_SIZE_LIMIT from arcade_slack.models import ConversationType, ConversationTypeSlackName from arcade_slack.tools.chat import ( + get_channel_metadata_by_name, get_conversation_metadata_by_id, - get_conversation_metadata_by_name, + get_direct_message_conversation_metadata_by_username, + get_members_in_channel_by_name, get_members_in_conversation_by_id, - get_members_in_conversation_by_name, get_messages_in_channel_by_name, get_messages_in_conversation_by_id, + get_messages_in_direct_message_conversation_by_username, list_conversations_metadata, list_direct_message_conversations_metadata, list_group_direct_message_conversations_metadata, @@ -37,37 +39,31 @@ def mock_channel_info() -> dict: return {"name": "general", "id": "C12345", "is_member": True, "is_channel": True} -@pytest.fixture -def mock_slack_client(mocker): - mock_client = mocker.patch("arcade_slack.tools.chat.AsyncWebClient", autospec=True) - return mock_client.return_value - - @pytest.mark.asyncio -async def test_send_dm_to_user(mock_context, mock_slack_client): - mock_slack_client.users_list.return_value = { +async def test_send_dm_to_user(mock_context, mock_chat_slack_client): + mock_chat_slack_client.users_list.return_value = { "ok": True, "members": [{"name": "testuser", "id": "U12345"}], } - mock_slack_client.conversations_open.return_value = { + mock_chat_slack_client.conversations_open.return_value = { "ok": True, "channel": {"id": "D12345"}, } mock_slack_response = Mock(spec=AsyncSlackResponse) mock_slack_response.data = {"ok": True} - mock_slack_client.chat_postMessage.return_value = mock_slack_response + mock_chat_slack_client.chat_postMessage.return_value = mock_slack_response response = await send_dm_to_user(mock_context, "testuser", "Hello!") assert response["response"]["ok"] is True - mock_slack_client.users_list.assert_called_once() - mock_slack_client.conversations_open.assert_called_once_with(users=["U12345"]) - mock_slack_client.chat_postMessage.assert_called_once_with(channel="D12345", text="Hello!") + mock_chat_slack_client.users_list.assert_called_once() + mock_chat_slack_client.conversations_open.assert_called_once_with(users=["U12345"]) + mock_chat_slack_client.chat_postMessage.assert_called_once_with(channel="D12345", text="Hello!") @pytest.mark.asyncio -async def test_send_dm_to_inexistent_user(mock_context, mock_slack_client): - mock_slack_client.users_list.return_value = { +async def test_send_dm_to_inexistent_user(mock_context, mock_chat_slack_client): + mock_chat_slack_client.users_list.return_value = { "ok": True, "members": [{"name": "testuser", "id": "U12345"}], } @@ -75,33 +71,33 @@ async def test_send_dm_to_inexistent_user(mock_context, mock_slack_client): with pytest.raises(RetryableToolError): await send_dm_to_user(mock_context, "inexistent_user", "Hello!") - mock_slack_client.users_list.assert_called_once() - mock_slack_client.conversations_open.assert_not_called() - mock_slack_client.chat_postMessage.assert_not_called() + mock_chat_slack_client.users_list.assert_called_once() + mock_chat_slack_client.conversations_open.assert_not_called() + mock_chat_slack_client.chat_postMessage.assert_not_called() @pytest.mark.asyncio -async def test_send_message_to_channel(mock_context, mock_slack_client): - mock_slack_client.conversations_list.return_value = { +async def test_send_message_to_channel(mock_context, mock_chat_slack_client): + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [{"id": "C12345", "name": "general"}], } mock_slack_response = Mock(spec=AsyncSlackResponse) mock_slack_response.data = {"ok": True} - mock_slack_client.chat_postMessage.return_value = mock_slack_response + mock_chat_slack_client.chat_postMessage.return_value = mock_slack_response response = await send_message_to_channel(mock_context, "general", "Hello, channel!") assert response["response"]["ok"] is True - mock_slack_client.conversations_list.assert_called_once() - mock_slack_client.chat_postMessage.assert_called_once_with( + mock_chat_slack_client.conversations_list.assert_called_once() + mock_chat_slack_client.chat_postMessage.assert_called_once_with( channel="C12345", text="Hello, channel!" ) @pytest.mark.asyncio -async def test_send_message_to_inexistent_channel(mock_context, mock_slack_client): - mock_slack_client.conversations_list.return_value = { +async def test_send_message_to_inexistent_channel(mock_context, mock_chat_slack_client): + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [], } @@ -109,15 +105,15 @@ async def test_send_message_to_inexistent_channel(mock_context, mock_slack_clien with pytest.raises(RetryableToolError): await send_message_to_channel(mock_context, "inexistent_channel", "Hello!") - mock_slack_client.conversations_list.assert_called_once() - mock_slack_client.chat_postMessage.assert_not_called() + mock_chat_slack_client.conversations_list.assert_called_once() + mock_chat_slack_client.chat_postMessage.assert_not_called() @pytest.mark.asyncio async def test_list_conversations_metadata_with_default_args( - mock_context, mock_slack_client, mock_channel_info + mock_context, mock_chat_slack_client, mock_channel_info ): - mock_slack_client.conversations_list.return_value = { + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [mock_channel_info], } @@ -127,7 +123,7 @@ async def test_list_conversations_metadata_with_default_args( assert response["conversations"] == [extract_conversation_metadata(mock_channel_info)] assert response["next_cursor"] is None - mock_slack_client.conversations_list.assert_called_once_with( + mock_chat_slack_client.conversations_list.assert_called_once_with( types=",".join([conv_type.value for conv_type in ConversationTypeSlackName]), exclude_archived=True, limit=MAX_PAGINATION_SIZE_LIMIT, @@ -137,9 +133,9 @@ async def test_list_conversations_metadata_with_default_args( @pytest.mark.asyncio async def test_list_conversations_metadata_filtering_single_conversation_type( - mock_context, mock_slack_client, mock_channel_info + mock_context, mock_chat_slack_client, mock_channel_info ): - mock_slack_client.conversations_list.return_value = { + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [mock_channel_info], } @@ -151,7 +147,7 @@ async def test_list_conversations_metadata_filtering_single_conversation_type( assert response["conversations"] == [extract_conversation_metadata(mock_channel_info)] assert response["next_cursor"] is None - mock_slack_client.conversations_list.assert_called_once_with( + mock_chat_slack_client.conversations_list.assert_called_once_with( types=ConversationTypeSlackName.PUBLIC_CHANNEL.value, exclude_archived=True, limit=MAX_PAGINATION_SIZE_LIMIT, @@ -161,9 +157,9 @@ async def test_list_conversations_metadata_filtering_single_conversation_type( @pytest.mark.asyncio async def test_list_conversations_metadata_filtering_multiple_conversation_types( - mock_context, mock_slack_client, mock_channel_info + mock_context, mock_chat_slack_client, mock_channel_info ): - mock_slack_client.conversations_list.return_value = { + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [mock_channel_info], } @@ -179,7 +175,7 @@ async def test_list_conversations_metadata_filtering_multiple_conversation_types assert response["conversations"] == [extract_conversation_metadata(mock_channel_info)] assert response["next_cursor"] is None - mock_slack_client.conversations_list.assert_called_once_with( + mock_chat_slack_client.conversations_list.assert_called_once_with( types=f"{ConversationTypeSlackName.PUBLIC_CHANNEL.value},{ConversationTypeSlackName.PRIVATE_CHANNEL.value}", exclude_archived=True, limit=MAX_PAGINATION_SIZE_LIMIT, @@ -189,9 +185,9 @@ async def test_list_conversations_metadata_filtering_multiple_conversation_types @pytest.mark.asyncio async def test_list_conversations_metadata_with_custom_pagination_args( - mock_context, mock_slack_client, mock_channel_info + mock_context, mock_chat_slack_client, mock_channel_info ): - mock_slack_client.conversations_list.return_value = { + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [mock_channel_info] * 3, "response_metadata": {"next_cursor": "456"}, @@ -204,7 +200,7 @@ async def test_list_conversations_metadata_with_custom_pagination_args( ] assert response["next_cursor"] == "456" - mock_slack_client.conversations_list.assert_called_once_with( + mock_chat_slack_client.conversations_list.assert_called_once_with( types=",".join([conv_type.value for conv_type in ConversationTypeSlackName]), exclude_archived=True, limit=3, @@ -221,9 +217,9 @@ async def test_list_conversations_metadata_with_custom_pagination_args( ], ) async def test_tools_with_slack_error( - mock_context, mock_slack_client, faulty_slack_function_name, tool_function, tool_args + mock_context, mock_chat_slack_client, faulty_slack_function_name, tool_function, tool_args ): - getattr(mock_slack_client, faulty_slack_function_name).side_effect = SlackApiError( + getattr(mock_chat_slack_client, faulty_slack_function_name).side_effect = SlackApiError( message="test_slack_error", response={"ok": False, "error": "test_slack_error"}, ) @@ -262,8 +258,10 @@ async def test_list_channels_metadata( @pytest.mark.asyncio -async def test_get_conversation_metadata_by_id(mock_context, mock_slack_client, mock_channel_info): - mock_slack_client.conversations_info.return_value = { +async def test_get_conversation_metadata_by_id( + mock_context, mock_chat_slack_client, mock_channel_info +): + mock_chat_slack_client.conversations_info.return_value = { "ok": True, "channel": mock_channel_info, } @@ -271,7 +269,7 @@ async def test_get_conversation_metadata_by_id(mock_context, mock_slack_client, response = await get_conversation_metadata_by_id(mock_context, "C12345") assert response == extract_conversation_metadata(mock_channel_info) - mock_slack_client.conversations_info.assert_called_once_with( + mock_chat_slack_client.conversations_info.assert_called_once_with( channel="C12345", include_locale=True, include_num_members=True, @@ -281,14 +279,14 @@ async def test_get_conversation_metadata_by_id(mock_context, mock_slack_client, @pytest.mark.asyncio @patch("arcade_slack.tools.chat.list_conversations_metadata") async def test_get_conversation_metadata_by_id_slack_api_error( - mock_list_conversations_metadata, mock_context, mock_slack_client, mock_channel_info + mock_list_conversations_metadata, mock_context, mock_chat_slack_client, mock_channel_info ): mock_channel_info["name"] = "whatever_conversation_should_be_present_in_additional_prompt" mock_list_conversations_metadata.return_value = { "conversations": [extract_conversation_metadata(mock_channel_info)], "response_metadata": {"next_cursor": None}, } - mock_slack_client.conversations_info.side_effect = SlackApiError( + mock_chat_slack_client.conversations_info.side_effect = SlackApiError( message="channel_not_found", response={"ok": False, "error": "channel_not_found"}, ) @@ -301,7 +299,7 @@ async def test_get_conversation_metadata_by_id_slack_api_error( in e.additional_prompt_content ) - mock_slack_client.conversations_info.assert_called_once_with( + mock_chat_slack_client.conversations_info.assert_called_once_with( channel="C12345", include_locale=True, include_num_members=True, @@ -319,34 +317,34 @@ async def test_get_conversation_metadata_by_name( "next_cursor": None, } - response = await get_conversation_metadata_by_name(mock_context, sample_conversation["name"]) + response = await get_channel_metadata_by_name(mock_context, sample_conversation["name"]) assert response == sample_conversation mock_list_conversations_metadata.assert_called_once_with(mock_context, next_cursor=None) @pytest.mark.asyncio -async def test_get_conversation_metadata_by_name_triggering_pagination( +async def test_get_channel_metadata_by_name_triggering_pagination( mock_context, mock_list_conversations_metadata, mock_channel_info ): - target_conversation = extract_conversation_metadata(mock_channel_info) - another_conversation = extract_conversation_metadata(mock_channel_info) - another_conversation["name"] = "another_conversation" + target_channel = extract_conversation_metadata(mock_channel_info) + another_channel = extract_conversation_metadata(mock_channel_info) + another_channel["name"] = "another_channel" mock_list_conversations_metadata.side_effect = [ { - "conversations": [another_conversation], + "conversations": [another_channel], "next_cursor": "123", }, { - "conversations": [target_conversation], + "conversations": [target_channel], "next_cursor": None, }, ] - response = await get_conversation_metadata_by_name(mock_context, target_conversation["name"]) + response = await get_channel_metadata_by_name(mock_context, target_channel["name"]) - assert response == target_conversation + assert response == target_channel assert mock_list_conversations_metadata.call_count == 2 mock_list_conversations_metadata.assert_has_calls([ call(mock_context, next_cursor=None), @@ -355,26 +353,26 @@ async def test_get_conversation_metadata_by_name_triggering_pagination( @pytest.mark.asyncio -async def test_get_conversation_metadata_by_name_not_found( +async def test_get_channel_metadata_by_name_not_found( mock_context, mock_list_conversations_metadata, mock_channel_info ): - first_conversation = extract_conversation_metadata(mock_channel_info) - second_conversation = extract_conversation_metadata(mock_channel_info) - second_conversation["name"] = "second_conversation" + first_channel = extract_conversation_metadata(mock_channel_info) + second_channel = extract_conversation_metadata(mock_channel_info) + second_channel["name"] = "second_channel" mock_list_conversations_metadata.side_effect = [ { - "conversations": [second_conversation], + "conversations": [second_channel], "next_cursor": "123", }, { - "conversations": [first_conversation], + "conversations": [first_channel], "next_cursor": None, }, ] with pytest.raises(RetryableToolError): - await get_conversation_metadata_by_name(mock_context, "inexistent_conversation") + await get_channel_metadata_by_name(mock_context, "inexistent_channel") assert mock_list_conversations_metadata.call_count == 2 mock_list_conversations_metadata.assert_has_calls([ @@ -387,7 +385,7 @@ async def test_get_conversation_metadata_by_name_not_found( @patch("arcade_slack.tools.chat.async_paginate") @patch("arcade_slack.tools.chat.get_user_info_by_id") async def test_get_members_from_conversation_id( - mock_get_user_info_by_id, mock_async_paginate, mock_context, mock_slack_client + mock_get_user_info_by_id, mock_async_paginate, mock_context, mock_chat_slack_client ): member1 = {"id": "U123", "name": "testuser123"} member1_info = extract_basic_user_info(member1) @@ -406,7 +404,7 @@ async def test_get_members_from_conversation_id( "next_cursor": "token123", } mock_async_paginate.assert_called_once_with( - mock_slack_client.conversations_members, + mock_chat_slack_client.conversations_members, "members", limit=2, next_cursor=None, @@ -427,7 +425,7 @@ async def test_get_members_from_conversation_id_channel_not_found( mock_get_user_info_by_id, mock_async_paginate, mock_context, - mock_slack_client, + mock_chat_slack_client, mock_channel_info, ): conversations = [extract_conversation_metadata(mock_channel_info)] * 2 @@ -451,7 +449,7 @@ async def test_get_members_from_conversation_id_channel_not_found( await get_members_in_conversation_by_id(mock_context, conversation_id="C12345", limit=2) mock_async_paginate.assert_called_once_with( - mock_slack_client.conversations_members, + mock_chat_slack_client.conversations_members, "members", limit=2, next_cursor=None, @@ -463,7 +461,7 @@ async def test_get_members_from_conversation_id_channel_not_found( @pytest.mark.asyncio @patch("arcade_slack.tools.chat.list_conversations_metadata") @patch("arcade_slack.tools.chat.get_members_in_conversation_by_id") -async def test_get_members_in_conversation_by_name( +async def test_get_members_in_channel_by_name( mock_get_members_in_conversation_by_id, mock_list_conversations_metadata, mock_context, @@ -474,7 +472,7 @@ async def test_get_members_in_conversation_by_name( "next_cursor": None, } - response = await get_members_in_conversation_by_name( + response = await get_members_in_channel_by_name( mock_context, mock_channel_info["name"], limit=2 ) @@ -491,7 +489,7 @@ async def test_get_members_in_conversation_by_name( @pytest.mark.asyncio @patch("arcade_slack.tools.chat.list_conversations_metadata") @patch("arcade_slack.tools.chat.get_members_in_conversation_by_id") -async def test_get_members_in_conversation_by_name_triggering_pagination( +async def test_get_members_in_channel_by_name_triggering_pagination( mock_get_members_in_conversation_by_id, mock_list_conversations_metadata, mock_context, @@ -513,9 +511,7 @@ async def test_get_members_in_conversation_by_name_triggering_pagination( }, ] - response = await get_members_in_conversation_by_name( - mock_context, conversation2["name"], limit=2 - ) + response = await get_members_in_channel_by_name(mock_context, conversation2["name"], limit=2) assert response == mock_get_members_in_conversation_by_id.return_value mock_list_conversations_metadata.assert_has_calls([ @@ -531,8 +527,8 @@ async def test_get_members_in_conversation_by_name_triggering_pagination( @pytest.mark.asyncio -async def test_get_conversation_history_by_id(mock_context, mock_slack_client): - mock_slack_client.conversations_history.return_value = { +async def test_get_conversation_history_by_id(mock_context, mock_chat_slack_client): + mock_chat_slack_client.conversations_history.return_value = { "ok": True, "messages": [{"text": "Hello, world!"}], } @@ -540,7 +536,7 @@ async def test_get_conversation_history_by_id(mock_context, mock_slack_client): response = await get_messages_in_conversation_by_id(mock_context, "C12345", limit=1) assert response == {"messages": [{"text": "Hello, world!"}], "next_cursor": None} - mock_slack_client.conversations_history.assert_called_once_with( + mock_chat_slack_client.conversations_history.assert_called_once_with( channel="C12345", include_all_metadata=True, inclusive=True, @@ -555,9 +551,12 @@ async def test_get_conversation_history_by_id(mock_context, mock_slack_client): @patch("arcade_slack.tools.chat.convert_relative_datetime_to_unix_timestamp") @patch("arcade_slack.tools.chat.datetime") async def test_get_conversation_history_by_id_with_relative_datetime_args( - mock_datetime, mock_convert_relative_datetime_to_unix_timestamp, mock_context, mock_slack_client + mock_datetime, + mock_convert_relative_datetime_to_unix_timestamp, + mock_context, + mock_chat_slack_client, ): - mock_slack_client.conversations_history.return_value = { + mock_chat_slack_client.conversations_history.return_value = { "ok": True, "messages": [{"text": "Hello, world!"}], } @@ -584,7 +583,7 @@ async def test_get_conversation_history_by_id_with_relative_datetime_args( call("01:00:00", expected_current_unix_timestamp), call("02:00:00", expected_current_unix_timestamp), ]) - mock_slack_client.conversations_history.assert_called_once_with( + mock_chat_slack_client.conversations_history.assert_called_once_with( channel="C12345", include_all_metadata=True, inclusive=True, @@ -600,9 +599,9 @@ async def test_get_conversation_history_by_id_with_relative_datetime_args( @pytest.mark.asyncio @patch("arcade_slack.tools.chat.convert_datetime_to_unix_timestamp") async def test_get_conversation_history_by_id_with_absolute_datetime_args( - mock_convert_datetime_to_unix_timestamp, mock_context, mock_slack_client + mock_convert_datetime_to_unix_timestamp, mock_context, mock_chat_slack_client ): - mock_slack_client.conversations_history.return_value = { + mock_chat_slack_client.conversations_history.return_value = { "ok": True, "messages": [{"text": "Hello, world!"}], } @@ -631,7 +630,7 @@ async def test_get_conversation_history_by_id_with_absolute_datetime_args( call("2025-01-02 00:00:00"), call("2025-01-01 00:00:00"), ]) - mock_slack_client.conversations_history.assert_called_once_with( + mock_chat_slack_client.conversations_history.assert_called_once_with( channel="C12345", include_all_metadata=True, inclusive=True, @@ -644,7 +643,7 @@ async def test_get_conversation_history_by_id_with_absolute_datetime_args( @pytest.mark.asyncio async def test_get_conversation_history_by_id_with_messed_oldest_args( - mock_context, mock_slack_client + mock_context, mock_chat_slack_client ): with pytest.raises(ToolExecutionError): await get_messages_in_conversation_by_id( @@ -657,7 +656,7 @@ async def test_get_conversation_history_by_id_with_messed_oldest_args( @pytest.mark.asyncio async def test_get_conversation_history_by_id_with_messed_latest_args( - mock_context, mock_slack_client + mock_context, mock_chat_slack_client ): with pytest.raises(ToolExecutionError): await get_messages_in_conversation_by_id( @@ -669,8 +668,8 @@ async def test_get_conversation_history_by_id_with_messed_latest_args( @pytest.mark.asyncio -async def test_get_conversation_history_by_name(mock_context, mock_slack_client): - mock_slack_client.conversations_list.return_value = { +async def test_get_conversation_history_by_name(mock_context, mock_chat_slack_client): + mock_chat_slack_client.conversations_list.return_value = { "ok": True, "channels": [ { @@ -681,7 +680,7 @@ async def test_get_conversation_history_by_name(mock_context, mock_slack_client) } ], } - mock_slack_client.conversations_history.return_value = { + mock_chat_slack_client.conversations_history.return_value = { "ok": True, "messages": [{"text": "Hello, world!"}], } @@ -689,6 +688,133 @@ async def test_get_conversation_history_by_name(mock_context, mock_slack_client) response = await get_messages_in_channel_by_name(mock_context, "general", limit=1) assert response == {"messages": [{"text": "Hello, world!"}], "next_cursor": None} - mock_slack_client.conversations_history.assert_called_once_with( + mock_chat_slack_client.conversations_history.assert_called_once_with( channel="C12345", include_all_metadata=True, inclusive=True, limit=1, cursor=None ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.retrieve_conversations_by_user_ids") +async def test_get_direct_message_conversation_metadata_by_username( + mock_retrieve_conversations_by_user_ids, + mock_context, + mock_chat_slack_client, + mock_users_slack_client, +): + mock_chat_slack_client.auth_test.return_value = { + "ok": True, + "user_id": "U1", + "team_id": "T1", + "user": "user1", + } + + mock_users_slack_client.users_list.return_value = { + "ok": True, + "members": [ + {"id": "U1", "name": "user1"}, + {"id": "U2", "name": "user2"}, + ], + "response_metadata": {"next_cursor": None}, + } + + conversation = { + "id": "C12345", + "type": ConversationTypeSlackName.IM.value, + "is_im": True, + "members": ["U1", "U2"], + } + + mock_retrieve_conversations_by_user_ids.return_value = [conversation] + + response = await get_direct_message_conversation_metadata_by_username( + context=mock_context, username="user2" + ) + + assert response == conversation + mock_retrieve_conversations_by_user_ids.assert_called_once_with( + list_conversations_func=list_conversations_metadata, + get_members_in_conversation_func=get_members_in_conversation_by_id, + context=mock_context, + conversation_types=[ConversationType.DIRECT_MESSAGE], + user_ids=["U1", "U2"], + exact_match=True, + limit=1, + next_cursor=None, + ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.retrieve_conversations_by_user_ids") +async def test_get_direct_message_conversation_metadata_by_username_username_not_found( + mock_retrieve_conversations_by_user_ids, + mock_context, + mock_chat_slack_client, + mock_users_slack_client, +): + mock_chat_slack_client.users_identity.return_value = { + "ok": True, + "user": {"id": "U1", "name": "user1"}, + "team": {"id": "T1", "name": "team1"}, + } + + mock_users_slack_client.users_list.return_value = { + "ok": True, + "members": [ + {"id": "U1", "name": "user1"}, + {"id": "U2", "name": "user2"}, + ], + "response_metadata": {"next_cursor": None}, + } + + mock_retrieve_conversations_by_user_ids.side_effect = TimeoutError() + + with pytest.raises(RetryableToolError): + await get_direct_message_conversation_metadata_by_username( + context=mock_context, username="user999" + ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.get_messages_in_conversation_by_id") +@patch("arcade_slack.tools.chat.get_direct_message_conversation_metadata_by_username") +async def test_get_messages_in_direct_conversation_by_username( + mock_get_direct_message_conversation_metadata_by_username, + mock_get_messages_in_conversation_by_id, + mock_context, +): + mock_get_direct_message_conversation_metadata_by_username.return_value = { + "id": "C12345", + } + + response = await get_messages_in_direct_message_conversation_by_username( + context=mock_context, username="user2" + ) + + assert response == mock_get_messages_in_conversation_by_id.return_value + mock_get_direct_message_conversation_metadata_by_username.assert_called_once_with( + context=mock_context, username="user2" + ) + mock_get_messages_in_conversation_by_id.assert_called_once_with( + context=mock_context, + conversation_id="C12345", + oldest_relative=None, + latest_relative=None, + oldest_datetime=None, + latest_datetime=None, + limit=None, + next_cursor=None, + ) + + +@pytest.mark.asyncio +@patch("arcade_slack.tools.chat.get_direct_message_conversation_metadata_by_username") +async def test_get_messages_in_direct_conversation_by_username_not_found( + mock_get_direct_message_conversation_metadata_by_username, + mock_context, +): + mock_get_direct_message_conversation_metadata_by_username.return_value = None + + with pytest.raises(ToolExecutionError): + await get_messages_in_direct_message_conversation_by_username( + context=mock_context, username="user2" + ) diff --git a/toolkits/slack/tests/test_utils.py b/toolkits/slack/tests/test_utils.py index f7c83ec2..c569e0aa 100644 --- a/toolkits/slack/tests/test_utils.py +++ b/toolkits/slack/tests/test_utils.py @@ -1,12 +1,26 @@ import asyncio -from unittest.mock import AsyncMock, call, patch +from unittest.mock import AsyncMock, MagicMock, call, patch import pytest +from arcade.sdk import ToolContext from slack_sdk.errors import SlackApiError from slack_sdk.web.async_client import AsyncWebClient from arcade_slack.exceptions import PaginationTimeoutError -from arcade_slack.utils import async_paginate +from arcade_slack.models import ( + ConversationType, + ConversationTypeSlackName, +) +from arcade_slack.tools.chat import ( + get_members_in_conversation_by_id, + list_conversations_metadata, +) +from arcade_slack.utils import ( + async_paginate, + convert_conversation_type_to_slack_name, + filter_conversations_by_user_ids, + retrieve_conversations_by_user_ids, +) @pytest.mark.asyncio @@ -185,3 +199,486 @@ async def test_async_paginate_with_timeout(): str(exc_info.value) == f"The pagination process timed out after {max_pagination_timeout_seconds} seconds." ) + + +def test_filter_conversations_by_user_ids(): + conversations = [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}, {"id": "user3"}]}, + {"id": "456", "members": [{"id": "user2"}, {"id": "user3"}]}, + ] + response = filter_conversations_by_user_ids( + conversations=conversations, + user_ids=["user1", "user2"], + exact_match=False, + ) + assert response == [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}, {"id": "user3"}]}, + ] + + +def test_filter_conversations_by_user_ids_empty_response(): + conversations = [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user3"}, {"id": "user4"}]}, + {"id": "456", "members": [{"id": "user2"}, {"id": "user3"}, {"id": "user4"}]}, + ] + response = filter_conversations_by_user_ids( + conversations=conversations, + user_ids=["user1", "user2"], + exact_match=False, + ) + assert response == [] + + +def test_filter_conversations_by_user_ids_multiple_matches(): + conversations = [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}, {"id": "user3"}]}, + {"id": "456", "members": [{"id": "user2"}, {"id": "user3"}]}, + { + "id": "789", + "members": [{"id": "user4"}, {"id": "user1"}, {"id": "user2"}, {"id": "user3"}], + }, + ] + response = filter_conversations_by_user_ids( + conversations=conversations, + user_ids=["user1", "user2"], + exact_match=False, + ) + assert response == [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}, {"id": "user3"}]}, + { + "id": "789", + "members": [{"id": "user4"}, {"id": "user1"}, {"id": "user2"}, {"id": "user3"}], + }, + ] + + +def test_filter_conversations_by_user_ids_exact_match(): + conversations = [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}]}, + {"id": "456", "members": [{"id": "user2"}, {"id": "user3"}]}, + ] + response = filter_conversations_by_user_ids( + conversations=conversations, + user_ids=["user1", "user2"], + exact_match=True, + ) + assert response == [{"id": "123", "members": [{"id": "user1"}, {"id": "user2"}]}] + + +def test_filter_conversations_by_user_ids_exact_match_empty_response(): + conversations = [ + {"id": "123", "members": [{"id": "user1"}, {"id": "user2"}, {"id": "user3"}]}, + {"id": "456", "members": [{"id": "user2"}, {"id": "user3"}]}, + ] + response = filter_conversations_by_user_ids( + conversations=conversations, + user_ids=["user1", "user2"], + exact_match=True, + ) + assert response == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "search_user_ids, conversation_types, exact_match, limit, expected_conversation_ids", + [ + (["U1", "U2"], [ConversationType.DIRECT_MESSAGE], False, 1, ["C1"]), + (["U1", "U2"], [ConversationType.DIRECT_MESSAGE], True, 1, ["C1"]), + (["U1", "U2", "U3"], [ConversationType.DIRECT_MESSAGE], False, 1, []), + ( + ["U1", "U2"], + [ConversationType.DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + False, + 10, + ["C1", "C3", "C4"], + ), + ( + ["U1", "U2"], + [ConversationType.DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + True, + 10, + ["C1", "C3"], + ), + ], +) +async def test_retrieve_conversations_by_user_ids( + mock_chat_slack_client, + mock_users_slack_client, + search_user_ids, + conversation_types, + exact_match, + limit, + expected_conversation_ids, +): + context = MagicMock(spec=ToolContext) + context.authorization = MagicMock() + context.authorization.token = MagicMock() + + conversation_types_slack_name_str = [ + convert_conversation_type_to_slack_name(conv_type).value + for conv_type in conversation_types or ConversationType + ] + + conversations = [ + { + "conversation": { + "id": "C1", + "type": ConversationTypeSlackName.IM.value, + "name": "im-1", + "is_channel": False, + "is_im": True, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + ], + }, + { + "conversation": { + "id": "C2", + "type": ConversationTypeSlackName.IM.value, + "name": "im-2", + "is_channel": False, + "is_im": True, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U2", "U3"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + ], + }, + { + "conversation": { + "id": "C3", + "type": ConversationTypeSlackName.PUBLIC_CHANNEL.value, + "name": "general", + "is_channel": True, + "is_im": False, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + ], + }, + { + "conversation": { + "id": "C4", + "type": ConversationTypeSlackName.PUBLIC_CHANNEL.value, + "name": "random", + "is_channel": True, + "is_im": False, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2", "U3", "U4"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + {"ok": True, "user": {"id": "U4", "team_id": "T123", "name": "user4"}}, + ], + }, + ] + + conversations_listed = [ + conversation + for conversation in conversations + if conversation["conversation"]["type"] in conversation_types_slack_name_str + ] + + mock_chat_slack_client.conversations_list.return_value = { + "ok": True, + "channels": [conversation["conversation"] for conversation in conversations_listed], + "response_metadata": {"next_cursor": None}, + } + + mock_chat_slack_client.conversations_members.side_effect = [ + conversation["members"] for conversation in conversations_listed + ] + + mock_users_slack_client.users_info.side_effect = [ + user for conversation in conversations_listed for user in conversation["users"] + ] + + conversations_found = await retrieve_conversations_by_user_ids( + list_conversations_func=list_conversations_metadata, + get_members_in_conversation_func=get_members_in_conversation_by_id, + context=context, + conversation_types=conversation_types, + user_ids=search_user_ids, + exact_match=exact_match, + limit=limit, + next_cursor=None, + ) + + assert [conversation["id"] for conversation in conversations_found] == expected_conversation_ids + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ( + "search_user_ids, conversation_types, exact_match, limit, " + "expected_conversation_ids, expected_conversations_list_calls" + ), + [ + ( + ["U1", "U2", "U3"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE], + False, + None, + ["C1", "C3"], + 2, + ), + ( + ["U1", "U2", "U3"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE], + True, + None, + ["C1"], + 2, + ), + (["U1", "U2", "U99"], [ConversationType.MULTI_PERSON_DIRECT_MESSAGE], False, None, [], 2), + ( + ["U1", "U2"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + False, + None, + ["C1", "C3", "C4", "C6"], + 2, + ), + ( + ["U1", "U2"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + False, + 1, + ["C1"], + 2, + ), + ( + ["U1", "U2"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + False, + 3, + ["C1", "C3", "C4"], + 2, + ), + ( + ["U1", "U2"], + [ConversationType.MULTI_PERSON_DIRECT_MESSAGE, ConversationType.PUBLIC_CHANNEL], + True, + None, + ["C4"], + 2, + ), + ], +) +async def test_retrieve_conversations_by_user_ids_with_pagination( + mock_chat_slack_client, + mock_users_slack_client, + search_user_ids, + conversation_types, + exact_match, + limit, + expected_conversation_ids, + expected_conversations_list_calls, +): + context = MagicMock(spec=ToolContext) + context.authorization = MagicMock() + context.authorization.token = MagicMock() + + conversation_types_slack_name_str = [ + convert_conversation_type_to_slack_name(conv_type).value + for conv_type in conversation_types or ConversationType + ] + + conversations = [ + { + "conversation": { + "id": "C1", + "type": ConversationTypeSlackName.MPIM.value, + "name": "mpim-1", + "is_channel": False, + "is_im": False, + "is_mpim": True, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2", "U3"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + ], + }, + { + "conversation": { + "id": "C2", + "type": ConversationTypeSlackName.MPIM.value, + "name": "mpim-2", + "is_channel": False, + "is_im": False, + "is_mpim": True, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U2", "U3"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + ], + }, + { + "conversation": { + "id": "C3", + "type": ConversationTypeSlackName.MPIM.value, + "name": "mpim-3", + "is_channel": False, + "is_im": False, + "is_mpim": True, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2", "U3", "U4"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + {"ok": True, "user": {"id": "U4", "team_id": "T123", "name": "user4"}}, + ], + }, + { + "conversation": { + "id": "C4", + "type": ConversationTypeSlackName.PUBLIC_CHANNEL.value, + "name": "channel-4", + "is_channel": True, + "is_im": False, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + ], + }, + { + "conversation": { + "id": "C5", + "type": ConversationTypeSlackName.PUBLIC_CHANNEL.value, + "name": "channel-5", + "is_channel": True, + "is_im": False, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U2", "U3", "U4"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + {"ok": True, "user": {"id": "U4", "team_id": "T123", "name": "user4"}}, + ], + }, + { + "conversation": { + "id": "C6", + "type": ConversationTypeSlackName.PUBLIC_CHANNEL.value, + "name": "channel-6", + "is_channel": True, + "is_im": False, + "is_member": True, + }, + "members": { + "ok": True, + "members": ["U1", "U2", "U3", "U4"], + "response_metadata": {"next_cursor": None}, + }, + "users": [ + {"ok": True, "user": {"id": "U1", "team_id": "T123", "name": "user1"}}, + {"ok": True, "user": {"id": "U2", "team_id": "T123", "name": "user2"}}, + {"ok": True, "user": {"id": "U3", "team_id": "T123", "name": "user3"}}, + {"ok": True, "user": {"id": "U4", "team_id": "T123", "name": "user4"}}, + ], + }, + ] + + conversations_listed = [ + conversation + for conversation in conversations + if conversation["conversation"]["type"] in conversation_types_slack_name_str + ] + + split_size = len(conversations_listed) // 2 + + conversations_listed_1 = conversations_listed[:split_size] + conversations_listed_2 = conversations_listed[split_size:] + + mock_chat_slack_client.conversations_list.side_effect = [ + { + "ok": True, + "channels": [conversation["conversation"] for conversation in conversations_listed_1], + "response_metadata": {"next_cursor": "cursor_1"}, + }, + { + "ok": True, + "channels": [conversation["conversation"] for conversation in conversations_listed_2], + "response_metadata": {"next_cursor": None}, + }, + ] + + mock_chat_slack_client.conversations_members.side_effect = [ + conversation["members"] for conversation in conversations_listed + ] + + mock_users_slack_client.users_info.side_effect = [ + user for conversation in conversations_listed for user in conversation["users"] + ] + + conversations_found = await retrieve_conversations_by_user_ids( + list_conversations_func=list_conversations_metadata, + get_members_in_conversation_func=get_members_in_conversation_by_id, + context=context, + conversation_types=conversation_types, + user_ids=search_user_ids, + exact_match=exact_match, + limit=limit, + next_cursor=None, + ) + + assert [conversation["id"] for conversation in conversations_found] == expected_conversation_ids + assert mock_chat_slack_client.conversations_list.call_count == expected_conversations_list_calls